From 06e140fb914181af672ff949b936a4b4af906875 Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Tue, 20 Oct 2026 14:58:24 +0800 Subject: [PATCH 01/30] refactor table --- docs/source/user_guide/format_table.rst | 28 +- include/paimon/catalog/catalog.h | 33 +- include/paimon/catalog/table.h | 47 ++- include/paimon/commit_context.h | 2 +- include/paimon/read_context.h | 2 +- include/paimon/scan_context.h | 2 +- include/paimon/table/format/format_table.h | 74 ++-- include/paimon/write_context.h | 2 +- src/paimon/CMakeLists.txt | 1 - src/paimon/core/catalog/catalog.cpp | 17 - src/paimon/core/catalog/catalog_utils.cpp | 20 - src/paimon/core/catalog/catalog_utils.h | 25 +- .../core/catalog/file_system_catalog.cpp | 30 +- src/paimon/core/catalog/file_system_catalog.h | 7 - .../core/catalog/file_system_catalog_test.cpp | 21 +- src/paimon/core/table/file_store_table.h | 65 +++ src/paimon/core/table/format/format_table.cpp | 195 ++++++++- .../core/table/format/format_table_loader.cpp | 87 ---- .../core/table/format/format_table_loader.h | 71 ---- .../core/table/format/format_table_test.cpp | 155 +++++++ src/paimon/core/table/table.cpp | 389 +++++++----------- src/paimon/core/table/table_test.cpp | 18 +- src/paimon/rest/rest_catalog.cpp | 37 +- src/paimon/rest/rest_catalog.h | 6 - src/paimon/rest/rest_catalog_test.cpp | 45 +- src/paimon/testing/mock/mock_catalog.h | 3 +- 26 files changed, 755 insertions(+), 627 deletions(-) create mode 100644 src/paimon/core/table/file_store_table.h delete mode 100644 src/paimon/core/table/format/format_table_loader.cpp delete mode 100644 src/paimon/core/table/format/format_table_loader.h diff --git a/docs/source/user_guide/format_table.rst b/docs/source/user_guide/format_table.rst index 492b5f0d0..4be21a1dd 100644 --- a/docs/source/user_guide/format_table.rst +++ b/docs/source/user_guide/format_table.rst @@ -40,8 +40,9 @@ format table is an insert. Reading and writing ------------------- -A format table is not served through :cpp:func:`Catalog::GetTable`, which describes a managed -table; use :cpp:func:`Catalog::GetFormatTable` instead. +A format table is served through the same :cpp:func:`Catalog::GetTable` as every other table: +its ``type`` option says what it is, and the call hands back the ``FormatTable`` it describes, +through the ``Table`` interface the two kinds of table share. Reading and writing go through the entry points every other table uses: ``TableScan::Create``, ``TableRead::Create``, ``FileStoreWrite::Create`` and ``FileStoreCommit::Create``, each from its @@ -58,8 +59,11 @@ a constructor taking one - which is what Java does through ``FormatTable.newRead .. code-block:: cpp - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, - catalog->GetFormatTable(paimon::Identifier("db", "tbl"))); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr loaded, + catalog->GetTable(paimon::Identifier("db", "tbl"))); + // The table's `type` option is `format-table`, so this is a `FormatTable`. + std::shared_ptr table = + std::dynamic_pointer_cast(loaded); paimon::ScanContextBuilder scan_builder(table); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_builder.Finish()); @@ -71,6 +75,12 @@ from one refuses ``SetTableSchema()``, ``WithFileSystem()``, ``WithFileSystemSchemeToIdentifierMap()``, ``WithCatalog()`` and a branch. Options given at the call still win over the ones the schema stored, as they do everywhere else. +``FormatTable`` is a derived class of ``Table``, so the table serves the same calls itself: a +caller holding one through a ``std::shared_ptr`` calls ``NewScan()``, +``NewRead()``, ``NewWrite()`` or ``NewCommit()`` on it, and the operators come back for the table +it is, whether or not the context carries one. This is the C++ side of what Java serves through +one ``Table.newReadBuilder()`` for both kinds of table. + A batch comes back in the table's column order, or the projection's when the read names one, behind the ``_VALUE_KIND`` field every ``BatchReader`` carries. Which columns a *file* is asked for and in what order is its own business: they are matched to the table's by name, as Java matches @@ -89,12 +99,10 @@ outside the projection is never read, which is how Java scopes the same checks. named here only to describe what happens. A caller never needs them: a plan comes back as ``Plan``, a split as ``Split``, and a commit message as ``CommitMessage``. -A catalog that can load a format table itself overrides ``Catalog::LoadFormatTable()``, the -protected hook :cpp:func:`Catalog::GetFormatTable` calls. The file system catalog uses it to say -its metadata lives under the table location, and the REST catalog to take the location and the -schema from one response instead of two that could disagree. A catalog that does not override it -still serves format tables, by reading the location and the schema through the virtuals every -catalog has. +Each catalog serves a format table from its own ``GetTable``. The file system catalog reads the +schema under the table location, so it says the location carries the table's own metadata and a +scan leaves the ``schema`` and ``branch`` directories below it alone; the REST catalog takes the +location and the schema from one response, so the two cannot disagree. ``TableScan::ListPartitions()`` lists the partitions a scan can see. A format table answers it by listing directories; every other table type returns ``NotImplemented`` for now. diff --git a/include/paimon/catalog/catalog.h b/include/paimon/catalog/catalog.h index 69eb6187c..d501bc7e3 100644 --- a/include/paimon/catalog/catalog.h +++ b/include/paimon/catalog/catalog.h @@ -40,7 +40,6 @@ namespace paimon { using Instant = std::variant; class Database; -class FormatTable; class Table; class View; class Schema; @@ -148,6 +147,11 @@ class PAIMON_EXPORT Catalog { /// Gets a table. /// + /// The table's `type` option decides what is handed back: a `format-table` comes back as + /// the `FormatTable` it is - a directory of data files, with no snapshot and no manifest - + /// and every other implemented type as the managed table it is. Both go through this one + /// `Table` interface: `NewScan()`, `NewRead()`, `NewWrite()` and `NewCommit()` serve either. + /// /// @param identifier Identifier of the table to get. /// @return A result containing the table, or an error status. virtual Result> GetTable(const Identifier& identifier) const = 0; @@ -246,33 +250,6 @@ class PAIMON_EXPORT Catalog { /// @note Implementations returning true must also implement the internal /// `VersionManagedCatalog`. virtual bool SupportsVersionManagement() const; - - /// Gets a format table: a directory of data files laid out like a standard Hive table. - /// - /// A format table carries no snapshots and no manifests, so it is loaded through its own - /// method rather than `GetTable()`. Reading and writing it go through the same `TableScan`, - /// `TableRead`, `FileStoreWrite` and `FileStoreCommit` entry points every other table uses: - /// hand the table back to `ScanContextBuilder`, `ReadContextBuilder`, `WriteContextBuilder` or - /// `CommitContextBuilder`, each of which takes one. - /// - /// @param identifier Identifier of the table to get. - /// @return A result containing the format table, or an error status if the table does not - /// exist or its `type` option is not `format-table`. - Result> GetFormatTable(const Identifier& identifier) const; - - protected: - /// Loads `identifier` as a format table, which is what `GetFormatTable()` hands back. - /// - /// The default reads the location and the schema through the virtuals above - two requests - /// that can disagree - and treats every directory below the location as table content. An - /// override answers both questions directly: whether location and schema come back in one - /// round trip, and whether this catalog keeps the table's metadata under the table path. - /// - /// @param identifier Identifier of the table to load. - /// @return A result containing the format table, or an error status if the table does not - /// exist or its `type` option is not `format-table`. - virtual Result> LoadFormatTable( - const Identifier& identifier) const; }; } // namespace paimon diff --git a/include/paimon/catalog/table.h b/include/paimon/catalog/table.h index e542c9aee..d18bd07b8 100644 --- a/include/paimon/catalog/table.h +++ b/include/paimon/catalog/table.h @@ -33,20 +33,25 @@ namespace paimon { -class TableScan; +class CommitContext; +class FileStoreCommit; +class FileStoreWrite; +class ReadContext; +class ScanContext; class TableRead; +class TableScan; +class WriteContext; /// A table provides basic abstraction for table type. +/// +/// The state a table of every kind carries - file system, root path, schema and identifier - lives +/// here, and each kind of table fills the operations in: the operators below create this table's +/// scan, read, write or commit, from a context the caller finishes through its builder. What the +/// context asks for that this table cannot honour is refused by name by the implementation rather +/// than silently dropped. class PAIMON_EXPORT Table { public: - /// @param file_system File system the table's data is accessed through. - /// @param root_path Root path of the table in the file system. - /// @param uuid Metastore UUID, or empty if unavailable. - Table(const std::shared_ptr& file_system, const std::string& root_path, - const std::shared_ptr& schema, const Identifier& identifier, - const std::string& uuid = ""); - - ~Table() = default; + virtual ~Table() = default; /// Returns the file system the table was opened with, which serves the table's /// files. A catalog that hands out per-table file systems opens a table with the @@ -99,33 +104,45 @@ class PAIMON_EXPORT Table { /// that `TableScan::Create()` also builds through. The scan is derived from the context. /// @param context The finished scan context. /// @return The scan, or the error building it reports. - Result> NewScan(std::unique_ptr context) const; + virtual Result> NewScan( + std::unique_ptr context) const = 0; /// Creates a read of this table from a finished read context, the table-first entry point /// that `TableRead::Create()` also builds through. The read is derived from the context. /// @param context The finished read context. /// @return The read, or the error building it reports. - Result> NewRead(std::unique_ptr context) const; + virtual Result> NewRead( + std::unique_ptr context) const = 0; /// Creates a write of this table from a finished write context, the table-first entry point /// that `FileStoreWrite::Create()` also builds through. The write is derived from the context. /// @param context The finished write context. /// @return The write, or the error building it reports. - Result> NewWrite(std::unique_ptr context) const; + virtual Result> NewWrite( + std::unique_ptr context) const = 0; /// Creates a commit of this table from a finished commit context, the table-first entry point /// that `FileStoreCommit::Create()` also builds through. The commit is derived from the /// context. /// @param context The finished commit context. /// @return The commit, or the error building it reports. - Result> NewCommit( - std::unique_ptr context) const; + virtual Result> NewCommit( + std::unique_ptr context) const = 0; + + protected: + /// @param file_system File system the table's data is accessed through. + /// @param root_path Root path of the table in the file system. + /// @param uuid Metastore UUID, or empty if unavailable. + Table(const std::shared_ptr& file_system, const std::string& root_path, + const std::shared_ptr& schema, const Identifier& identifier, + const std::string& uuid = ""); - private: std::shared_ptr file_system_; std::string root_path_; std::shared_ptr schema_; Identifier identifier_; + + private: std::string uuid_; }; diff --git a/include/paimon/commit_context.h b/include/paimon/commit_context.h index ffd40ac07..1ec784f76 100644 --- a/include/paimon/commit_context.h +++ b/include/paimon/commit_context.h @@ -151,7 +151,7 @@ class PAIMON_EXPORT CommitContextBuilder { /// /// There is no commit user: a format table keeps no snapshot to record one in. /// - /// @param table The format table to commit to, as `Catalog::GetFormatTable()` hands it back. + /// @param table The format table to commit to, as `Catalog::GetTable()` hands it back. explicit CommitContextBuilder(const std::shared_ptr& table); ~CommitContextBuilder(); diff --git a/include/paimon/read_context.h b/include/paimon/read_context.h index e62386c65..b9c534065 100644 --- a/include/paimon/read_context.h +++ b/include/paimon/read_context.h @@ -246,7 +246,7 @@ class PAIMON_EXPORT ReadContextBuilder { /// `SetTableSchema()`, `WithFileSystem()`, `WithFileSystemSchemeToIdentifierMap()` and a /// branch are refused here rather than ignored. /// - /// @param table The format table to read, as `Catalog::GetFormatTable()` hands it back. + /// @param table The format table to read, as `Catalog::GetTable()` hands it back. explicit ReadContextBuilder(const std::shared_ptr& table); ~ReadContextBuilder(); diff --git a/include/paimon/scan_context.h b/include/paimon/scan_context.h index 6fbb006f0..5be7c9c4e 100644 --- a/include/paimon/scan_context.h +++ b/include/paimon/scan_context.h @@ -183,7 +183,7 @@ class PAIMON_EXPORT ScanContextBuilder { /// table a REST catalog serves. The table carries what such a location does not say, so /// `SetTableSchema()` and `WithFileSystem()` are refused here rather than ignored. /// - /// @param table The format table to scan, as `Catalog::GetFormatTable()` hands it back. + /// @param table The format table to scan, as `Catalog::GetTable()` hands it back. explicit ScanContextBuilder(const std::shared_ptr& table); ~ScanContextBuilder(); diff --git a/include/paimon/table/format/format_table.h b/include/paimon/table/format/format_table.h index a4c310254..d2db6aa4a 100644 --- a/include/paimon/table/format/format_table.h +++ b/include/paimon/table/format/format_table.h @@ -24,6 +24,7 @@ #include #include "paimon/catalog/identifier.h" +#include "paimon/catalog/table.h" #include "paimon/result.h" #include "paimon/schema/schema.h" #include "paimon/status.h" @@ -47,13 +48,18 @@ class FileSystem; /// Writes only insert - there is nowhere to record an update or a delete - so a read fills /// `_VALUE_KIND` with inserts throughout. /// -/// Reading and writing go through the entry points every other table uses: `TableScan`, -/// `TableRead`, `FileStoreWrite` and `FileStoreCommit`. Each recognises a format table from the -/// schema under the table path, or takes one already loaded through the `FormatTable` constructor -/// every context builder has, which is how a table whose schema lives in a metastore is reached. +/// A format table is a `Table`, so the operators it inherits create its scan, read, write and +/// commit from this table, with the context's options applied on top of the ones it carries. The +/// entry points every other table uses remain open too - `TableScan::Create()` and friends - each +/// of which recognises a format table from the schema under the table path, or takes one already +/// loaded through the `FormatTable` constructor every context builder has, which is how a table +/// whose schema lives in a metastore is reached. +/// +/// Every instance lives in a `shared_ptr`, which is what its operators hand to +/// `FormatTable::Copy()`. /// /// See `docs/source/user_guide/format_table.rst` for what is not supported yet. -class PAIMON_EXPORT FormatTable { +class PAIMON_EXPORT FormatTable : public Table, public std::enable_shared_from_this { public: /// Formats a format table's files can be in. enum class Format { @@ -72,7 +78,7 @@ class PAIMON_EXPORT FormatTable { /// /// This needs a schema file under the table directory, which a table created through /// `SchemaManager` or a file system catalog has. A table whose schema lives in a metastore has - /// none, and is loaded through `Catalog::GetFormatTable()` instead. + /// none, and is loaded through `Catalog::GetTable()` instead. /// /// @param file_system File system holding the table directory. /// @param table_path Root path of the table, which is also its data location. @@ -119,7 +125,7 @@ class PAIMON_EXPORT FormatTable { /// Directory the data files live in. const std::string& Location() const { - return location_; + return GetRootPath(); } /// Format of every data file in the directory. @@ -155,27 +161,16 @@ class PAIMON_EXPORT FormatTable { return options_; } - /// A name to identify this table. - std::string Name() const { - return identifier_.GetObjectName(); - } - - /// Full name of the table, database.tableName. - std::string FullName() const; - - /// Schema of the table, including its partition fields. + /// Schema of the table, including its partition fields. It hides `Table::LatestSchema()`, + /// which returns the same schema as the base every table sees; this one hands it back as the + /// `DataSchema` a format table is always created with. std::shared_ptr LatestSchema() const { - return schema_; + return std::static_pointer_cast(schema_); } /// Schema of the table as an arrow schema, including its partition fields. Result> GetArrowSchema() const; - /// File system holding the table directory. - std::shared_ptr GetFileSystem() const { - return file_system_; - } - /// Whether this table's own metadata lives under its location, as told by whoever loaded it. /// /// Only then are the `schema` and `branch` directories below the location table metadata @@ -185,6 +180,28 @@ class PAIMON_EXPORT FormatTable { return location_carries_paimon_metadata_; } + /// Creates a scan of this table from a finished scan context, refusing by name what a format + /// table cannot honour. The context's options are applied on top of the ones this table + /// carries, the precedence every context builder promises. + Result> NewScan(std::unique_ptr context) const override; + + /// Creates a read of this table from a finished read context, refusing by name what a format + /// table cannot honour. The context's options are applied on top of the ones this table + /// carries, the precedence every context builder promises. + Result> NewRead(std::unique_ptr context) const override; + + /// Creates a write of this table from a finished write context, refusing by name what a format + /// table cannot honour. The context's options are applied on top of the ones this table + /// carries, the precedence every context builder promises. + Result> NewWrite( + std::unique_ptr context) const override; + + /// Creates a commit of this table from a finished commit context, refusing by name what a + /// format table cannot honour. The context's options are applied on top of the ones this + /// table carries, the precedence every context builder promises. + Result> NewCommit( + std::unique_ptr context) const override; + private: FormatTable(const std::shared_ptr& file_system, const std::string& location, const Identifier& identifier, const std::shared_ptr& schema, @@ -192,10 +209,15 @@ class PAIMON_EXPORT FormatTable { const std::string& file_compression, const std::string& partition_default_name, bool partition_only_value_in_path, bool location_carries_paimon_metadata); - std::shared_ptr file_system_; - std::string location_; - Identifier identifier_; - std::shared_ptr schema_; + /// This table as a `shared_ptr`, which the operators above hand to `FormatTable::Copy()`. + /// + /// `shared_from_this()` on a const method hands back a `shared_ptr`, while + /// `Copy()` takes a mutable one; the cast is safe because `Copy()` never writes to the table + /// it is given. + std::shared_ptr SharedSelf() const { + return std::const_pointer_cast(shared_from_this()); + } + std::map options_; Format format_; std::string file_compression_; diff --git a/include/paimon/write_context.h b/include/paimon/write_context.h index e0a940eb0..58c0b3f97 100644 --- a/include/paimon/write_context.h +++ b/include/paimon/write_context.h @@ -178,7 +178,7 @@ class PAIMON_EXPORT WriteContextBuilder { /// /// There is no commit user: a format table keeps no snapshot to record one in. /// - /// @param table The format table to write to, as `Catalog::GetFormatTable()` hands it back. + /// @param table The format table to write to, as `Catalog::GetTable()` hands it back. explicit WriteContextBuilder(const std::shared_ptr& table); ~WriteContextBuilder(); diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index b4ad6e80f..18519223a 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -452,7 +452,6 @@ set(PAIMON_CORE_SRCS core/table/format/format_table_commit.cpp core/table/format/format_table_file_store_commit.cpp core/table/format/format_table_file_store_write.cpp - core/table/format/format_table_loader.cpp core/table/format/format_table_read.cpp core/table/format/format_table_scan.cpp core/table/format/format_table_write.cpp diff --git a/src/paimon/core/catalog/catalog.cpp b/src/paimon/core/catalog/catalog.cpp index 978c20e9d..318125ec6 100644 --- a/src/paimon/core/catalog/catalog.cpp +++ b/src/paimon/core/catalog/catalog.cpp @@ -26,11 +26,9 @@ #include "fmt/format.h" #include "paimon/catalog_options.h" #include "paimon/common/utils/string_utils.h" -#include "paimon/core/catalog/catalog_utils.h" #include "paimon/core/catalog/file_system_catalog.h" #include "paimon/core/core_options.h" #include "paimon/schema/schema.h" -#include "paimon/table/format/format_table.h" #ifdef PAIMON_ENABLE_REST #include "paimon/rest/rest_catalog.h" #endif @@ -74,19 +72,4 @@ bool Catalog::SupportsVersionManagement() const { return false; } -Result> Catalog::GetFormatTable(const Identifier& identifier) const { - // Refused here rather than in each override, so every catalog rejects a system table the same - // way and in the same order: the database first, so that an identifier of "sys" is rejected - // without being parsed as a table name. - PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotSystemTable(identifier, "getFormatTable")); - return LoadFormatTable(identifier); -} - -Result> Catalog::LoadFormatTable(const Identifier& identifier) const { - // The location and the schema are read separately, and every directory below the location is - // table content: a catalog that knows better overrides this and says so. - return CatalogUtils::LoadFormatTableInTwoRequests(*this, identifier, - /*location_carries_paimon_metadata=*/false); -} - } // namespace paimon diff --git a/src/paimon/core/catalog/catalog_utils.cpp b/src/paimon/core/catalog/catalog_utils.cpp index 1f2a3800a..721942a88 100644 --- a/src/paimon/core/catalog/catalog_utils.cpp +++ b/src/paimon/core/catalog/catalog_utils.cpp @@ -31,7 +31,6 @@ #include "paimon/defs.h" #include "paimon/result.h" #include "paimon/schema/schema.h" -#include "paimon/table/format/format_table.h" namespace paimon { @@ -117,12 +116,6 @@ Status CatalogUtils::CheckManagedTableType(const Identifier& identifier, } PAIMON_ASSIGN_OR_RAISE(TableType table_type, TableTypeDefine::FromOptions(data_schema->Options())); - if (table_type == TableType::FORMAT_TABLE) { - return Status::Invalid( - fmt::format("Cannot open format table '{}' as a Table in '{}', please use " - "'Catalog::GetFormatTable' or 'FormatTable::Create'.", - identifier.ToString(), action)); - } // A materialized table is a managed table that also carries the SQL it materializes. if (table_type != TableType::TABLE && table_type != TableType::MATERIALIZED_TABLE) { const std::map& options = data_schema->Options(); @@ -136,17 +129,4 @@ Status CatalogUtils::CheckManagedTableType(const Identifier& identifier, return Status::OK(); } -Result> CatalogUtils::LoadFormatTableInTwoRequests( - const Catalog& catalog, const Identifier& identifier, bool location_carries_paimon_metadata) { - PAIMON_ASSIGN_OR_RAISE(std::string location, catalog.GetTableLocation(identifier)); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, catalog.LoadTableSchema(identifier)); - std::shared_ptr data_schema = std::dynamic_pointer_cast(schema); - if (data_schema == nullptr) { - return Status::Invalid(fmt::format("{} is not a data table, so it cannot be a format table", - identifier.GetFullName())); - } - return FormatTable::Create(catalog.GetFileSystem(), location, identifier, data_schema, - location_carries_paimon_metadata, /*dynamic_options=*/{}); -} - } // namespace paimon diff --git a/src/paimon/core/catalog/catalog_utils.h b/src/paimon/core/catalog/catalog_utils.h index 565a6aef5..23cb77203 100644 --- a/src/paimon/core/catalog/catalog_utils.h +++ b/src/paimon/core/catalog/catalog_utils.h @@ -66,32 +66,19 @@ class CatalogUtils { /// branch name, system table name) cannot be used as a single path component. static Status CheckValidTableName(const Identifier& identifier); - /// Fails when `schema` is not a table a `Table` can describe. + /// Fails when `schema` is not a table a managed `Table` can describe. /// - /// Only a managed table is one: a format table is loaded through `GetFormatTable()`, and the - /// remaining table types are stored differently and are not implemented here, so handing one - /// back as a `Table` would promise snapshots it never had. A schema that is not a data - /// table's, such as a system table's, passes. + /// Only a managed table is one: the remaining table types are stored differently and are + /// not implemented here, so handing one back as a `Table` would promise snapshots it never + /// had. A schema that is not a data table's, such as a system table's, passes. A format + /// table is served by the caller - `GetTable()` hands one back as the `FormatTable` it is - + /// so its schema only reaches this check by mistake and falls to the refusal below. /// /// @param action The call being refused, qualified as a caller would write it, e.g. /// `Catalog::GetTable`. It names the entry point in the message. static Status CheckManagedTableType(const Identifier& identifier, const std::shared_ptr& schema, const std::string& action); - - /// Loads `identifier` as a format table by reading its location and its schema separately. - /// - /// What every catalog can do, so it is what `Catalog::GetFormatTable()` falls back to. A - /// catalog that can get both from one response should not use it, since two requests can - /// disagree. - /// - /// @param location_carries_paimon_metadata Whether this catalog put the table's own metadata - /// under the table path, which is what tells a `schema` or `branch` directory below the - /// location from a partition directory of the same name. See - /// `FormatTable::LocationCarriesPaimonMetadata()`. - static Result> LoadFormatTableInTwoRequests( - const Catalog& catalog, const Identifier& identifier, - bool location_carries_paimon_metadata); }; } // namespace paimon diff --git a/src/paimon/core/catalog/file_system_catalog.cpp b/src/paimon/core/catalog/file_system_catalog.cpp index 87c89de8c..14293021e 100644 --- a/src/paimon/core/catalog/file_system_catalog.cpp +++ b/src/paimon/core/catalog/file_system_catalog.cpp @@ -34,7 +34,9 @@ #include "paimon/common/utils/string_utils.h" #include "paimon/core/catalog/catalog_utils.h" #include "paimon/core/core_options.h" +#include "paimon/core/options/table_type.h" #include "paimon/core/snapshot.h" +#include "paimon/core/table/file_store_table.h" #include "paimon/core/table/system/global_system_tables.h" #include "paimon/core/table/system/system_table.h" #include "paimon/core/table/system/system_table_schema.h" @@ -44,6 +46,7 @@ #include "paimon/fs/file_system.h" #include "paimon/logging.h" #include "paimon/result.h" +#include "paimon/table/format/format_table.h" namespace arrow { class Schema; @@ -343,7 +346,7 @@ Result> FileSystemCatalog::GetTable(const Identifier& ide PAIMON_ASSIGN_OR_RAISE(std::string data_table_name, identifier.GetDataTableName()); Identifier data_identifier(identifier.GetDatabaseName(), data_table_name); PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(data_identifier)); - return std::make_shared(fs_, table_path, schema, identifier); + return std::make_shared(fs_, table_path, schema, identifier); } // The schema is read on the table's own branch: `root/branch/branch-` below the // table path, which `main` resolves to itself. `GetBranchNameOrDefault` folds any @@ -353,14 +356,27 @@ Result> FileSystemCatalog::GetTable(const Identifier& ide if (!latest_schema) { return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); } + PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(identifier)); + // The schema's `type` says what the table is. A format table is the same directory read as + // plain data, so the schema read above is all it takes to serve one: no manifest is looked + // for, and the checks a format table adds run in `FormatTable::Create`. + std::shared_ptr data_schema = checked_pointer_cast(*latest_schema); + PAIMON_ASSIGN_OR_RAISE(bool is_format_table, + TableTypeDefine::IsFormatTable(data_schema->Options())); + if (is_format_table) { + // This catalog keeps a table's own metadata under the table path, so a `schema` or + // `branch` directory there is metadata rather than table content. + return FormatTable::Create(fs_, table_path, identifier, data_schema, + /*location_carries_paimon_metadata=*/true, + /*dynamic_options=*/{}); + } // Only a managed table is a `Table`; describing another type as one would hand back // snapshots and manifests it never had, so the failure belongs to the call that opens the // table rather than at the first read. PAIMON_RETURN_NOT_OK(CatalogUtils::CheckManagedTableType(identifier, *latest_schema, "FileSystemCatalog::GetTable")); - PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(identifier)); - return std::make_shared
(fs_, table_path, checked_pointer_cast(*latest_schema), - identifier); + return std::make_shared( + fs_, table_path, checked_pointer_cast(*latest_schema), identifier); } Status FileSystemCatalog::DropDatabase(const std::string& name, bool ignore_if_not_exists, @@ -579,10 +595,4 @@ Result> FileSystemCatalog::GetLatestSnapshot( return std::optional(snapshot->ToSnapshotInfo()); } -Result> FileSystemCatalog::LoadFormatTable( - const Identifier& identifier) const { - return CatalogUtils::LoadFormatTableInTwoRequests(*this, identifier, - /*location_carries_paimon_metadata=*/true); -} - } // namespace paimon diff --git a/src/paimon/core/catalog/file_system_catalog.h b/src/paimon/core/catalog/file_system_catalog.h index ccc45f666..fd4360755 100644 --- a/src/paimon/core/catalog/file_system_catalog.h +++ b/src/paimon/core/catalog/file_system_catalog.h @@ -71,13 +71,6 @@ class FileSystemCatalog : public Catalog { Result> GetLatestSnapshot(const Identifier& identifier, const std::string& branch) const override; - protected: - /// This catalog keeps a table's schema under the table's own path, so the `schema` and - /// `branch` directories there are metadata rather than table content. Nothing else differs, - /// so the location and the schema are still read separately. - Result> LoadFormatTable( - const Identifier& identifier) const override; - private: /// Fails when `db_name` cannot be used as a single path component, so that the returned /// path always stays under `warehouse`. diff --git a/src/paimon/core/catalog/file_system_catalog_test.cpp b/src/paimon/core/catalog/file_system_catalog_test.cpp index 51e7c642d..cfe361bae 100644 --- a/src/paimon/core/catalog/file_system_catalog_test.cpp +++ b/src/paimon/core/catalog/file_system_catalog_test.cpp @@ -629,7 +629,7 @@ TEST(FileSystemCatalogTest, TestCreateTableWithBlob) { ArrowSchemaRelease(&schema); } -TEST(FileSystemCatalogTest, TestGetTableRejectsFormatTable) { +TEST(FileSystemCatalogTest, TestGetTableServesAFormatTable) { std::map options; options[Options::FILE_SYSTEM] = "local"; options[Options::FILE_FORMAT] = "orc"; @@ -650,11 +650,11 @@ TEST(FileSystemCatalogTest, TestGetTableRejectsFormatTable) { /*ignore_if_exists=*/false)); ArrowSchemaRelease(&schema); - // A format table has no snapshots and no manifests, so handing it back as a Table would - // describe it as something it is not. - ASSERT_NOK_WITH_MSG(catalog.GetTable(Identifier("db1", "fmt")), "Cannot open format table"); - ASSERT_OK_AND_ASSIGN(std::shared_ptr format_table, - catalog.GetFormatTable(Identifier("db1", "fmt"))); + // The table's `type` says what it is, so the one call that opens a table hands a format + // table back as the `FormatTable` it is, through the `Table` interface the two share. + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog.GetTable(Identifier("db1", "fmt"))); + std::shared_ptr format_table = std::dynamic_pointer_cast(table); + ASSERT_NE(nullptr, format_table); ASSERT_EQ(format_table->GetFormat(), FormatTable::Format::PARQUET); ASSERT_EQ(format_table->FullName(), "db1.fmt"); @@ -662,15 +662,6 @@ TEST(FileSystemCatalogTest, TestGetTableRejectsFormatTable) { // `branch` directories there are its metadata and a scan leaves them alone. A catalog that // keeps schemas elsewhere says no, and then every directory below the location is data. ASSERT_TRUE(format_table->LocationCarriesPaimonMetadata()); - - // A system table is refused by `Catalog::GetFormatTable()` itself, so every catalog answers - // alike. The database is checked before the table name is parsed, so a malformed name under - // "sys" is refused as a system table rather than surfacing a parse error. - ASSERT_NOK_WITH_MSG(catalog.GetFormatTable(Identifier("db1", "fmt$snapshots")), - "for system table"); - ASSERT_NOK_WITH_MSG(catalog.GetFormatTable(Identifier("sys", "all_table_options")), - "for system table"); - ASSERT_NOK_WITH_MSG(catalog.GetFormatTable(Identifier("sys", "a$b$c$d")), "for system table"); } TEST(FileSystemCatalogTest, TestInvalidCreateTable) { diff --git a/src/paimon/core/table/file_store_table.h b/src/paimon/core/table/file_store_table.h new file mode 100644 index 000000000..409fefca9 --- /dev/null +++ b/src/paimon/core/table/file_store_table.h @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include + +#include "paimon/catalog/identifier.h" +#include "paimon/catalog/table.h" +#include "paimon/result.h" +#include "paimon/type_fwd.h" + +namespace paimon { + +class Schema; + +/// The managed table a catalog hands out: one with snapshots and manifests, which the schema under +/// its root path describes. A format table - a directory of plain data files - is a `FormatTable`, +/// the other kind of `Table`. +/// +/// The operators create through the context-only entry points every table serves, so a context +/// built without a table reaches the same operators: `TableScan::Create()`, `TableRead::Create()`, +/// `FileStoreWrite::Create()` and `FileStoreCommit::Create()`, which read the table's schema to +/// dispatch on its type. +class FileStoreTable : public Table { + public: + /// The constructor a catalog builds a managed table through, so a `shared_ptr` can hold one; + /// the base constructor is protected, which an inherited constructor would stay. + /// @param file_system File system the table's data is accessed through. + /// @param root_path Root path of the table in the file system. + /// @param uuid Metastore UUID, or empty if unavailable. + FileStoreTable(const std::shared_ptr& file_system, const std::string& root_path, + const std::shared_ptr& schema, const Identifier& identifier, + const std::string& uuid = "") + : Table(file_system, root_path, schema, identifier, uuid) {} + + Result> NewScan(std::unique_ptr context) const override; + + Result> NewRead(std::unique_ptr context) const override; + + Result> NewWrite( + std::unique_ptr context) const override; + + Result> NewCommit( + std::unique_ptr context) const override; +}; + +} // namespace paimon diff --git a/src/paimon/core/table/format/format_table.cpp b/src/paimon/core/table/format/format_table.cpp index a039baa4b..53d53b967 100644 --- a/src/paimon/core/table/format/format_table.cpp +++ b/src/paimon/core/table/format/format_table.cpp @@ -18,9 +18,15 @@ #include "paimon/table/format/format_table.h" +#include +#include #include +#include +#include +#include #include "fmt/format.h" +#include "paimon/commit_context.h" #include "paimon/common/utils/checked_cast.h" #include "paimon/common/utils/options_utils.h" #include "paimon/common/utils/string_utils.h" @@ -29,8 +35,20 @@ #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/schema_validation.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/format/format_table_file_store_commit.h" +#include "paimon/core/table/format/format_table_file_store_write.h" +#include "paimon/core/table/format/format_table_read.h" +#include "paimon/core/table/format/format_table_scan.h" #include "paimon/defs.h" +#include "paimon/file_store_commit.h" +#include "paimon/file_store_write.h" #include "paimon/fs/file_system.h" +#include "paimon/read_context.h" +#include "paimon/scan_context.h" +#include "paimon/status.h" +#include "paimon/table/source/table_read.h" +#include "paimon/table/source/table_scan.h" +#include "paimon/write_context.h" namespace paimon { @@ -45,10 +63,7 @@ FormatTable::FormatTable(const std::shared_ptr& file_system, const std::string& file_compression, const std::string& partition_default_name, bool partition_only_value_in_path, bool location_carries_paimon_metadata) - : file_system_(file_system), - location_(location), - identifier_(identifier), - schema_(schema), + : Table(file_system, location, schema, identifier), options_(options), format_(format), file_compression_(file_compression), @@ -192,20 +207,180 @@ Result> FormatTable::Copy( for (const auto& [key, value] : dynamic_options) { options[key] = value; } - return Create(table->file_system_, table->location_, table->identifier_, table->schema_, + return Create(table->file_system_, table->root_path_, table->identifier_, + std::static_pointer_cast(table->schema_), table->location_carries_paimon_metadata_, options); } const std::vector& FormatTable::PartitionKeys() const { - return schema_->PartitionKeys(); -} - -std::string FormatTable::FullName() const { - return identifier_.GetFullName(); + return LatestSchema()->PartitionKeys(); } Result> FormatTable::GetArrowSchema() const { return schema_->GetArrowSchema(); } +Result> FormatTable::NewScan( + std::unique_ptr context) const { + std::shared_ptr shared_context = std::move(context); + if (shared_context == nullptr) { + return Status::Invalid("scan context is null pointer"); + } + if (shared_context->GetMemoryPool() == nullptr) { + return Status::Invalid("memory pool is null pointer"); + } + if (shared_context->GetExecutor() == nullptr) { + return Status::Invalid("executor is null pointer"); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, + FormatTable::Copy(SharedSelf(), shared_context->GetOptions())); + // Anything a `ScanContext` carries that a format table cannot honour is refused rather than + // silently dropped. + if (shared_context->IsStreamingMode()) { + return Status::NotImplemented( + "a format table has no snapshots, so there is nothing for a streaming scan to follow"); + } + if (shared_context->GetRealtimeContext() != nullptr) { + return Status::NotImplemented( + "a format table has no real-time store to union with what is on disk"); + } + if (shared_context->GetGlobalIndexResult() != nullptr) { + return Status::NotImplemented("a format table carries no global index"); + } + std::map partition_filter; + std::shared_ptr filters = shared_context->GetScanFilters(); + if (filters != nullptr) { + if (filters->GetPredicate() != nullptr) { + return Status::NotImplemented( + "a format table scan does not take a predicate: its files carry no statistics to " + "skip by, and pruning by the partition columns a predicate names is not " + "implemented yet; give the partition values as a partition filter instead"); + } + if (filters->GetBucketFilter()) { + return Status::NotImplemented("a format table has no buckets to filter by"); + } + const std::vector>& partition_filters = + filters->GetPartitionFilters(); + if (partition_filters.size() > 1) { + return Status::NotImplemented("a format table scan takes at most one partition filter"); + } + if (partition_filters.size() == 1) { + partition_filter = partition_filters.front(); + } + } + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr scan, + FormatTableScan::Create(table, partition_filter, shared_context->GetLimit())); + return std::unique_ptr(std::move(scan)); +} + +Result> FormatTable::NewRead( + std::unique_ptr context) const { + std::shared_ptr read_context = std::move(context); + if (read_context == nullptr) { + return Status::Invalid("read context is null pointer"); + } + if (read_context->GetMemoryPool() == nullptr) { + return Status::Invalid("memory pool is null pointer"); + } + if (read_context->GetExecutor() == nullptr) { + return Status::Invalid("executor is null pointer"); + } + if (read_context->HasVectorOrFullTextSearch()) { + return Status::NotImplemented( + "VectorSearch or FullTextSearch is not supported for format tables"); + } + if (PredicateUtils::ContainsFullTextMatch(read_context->GetPredicate())) { + return Status::NotImplemented("FullTextMatch predicate execution is not implemented yet"); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, + FormatTable::Copy(SharedSelf(), read_context->GetOptions())); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, + FormatTableRead::Create(table, read_context)); + return std::unique_ptr(std::move(read)); +} + +Result> FormatTable::NewWrite( + std::unique_ptr context) const { + if (context == nullptr) { + return Status::Invalid("write context is null pointer"); + } + if (context->GetMemoryPool() == nullptr) { + return Status::Invalid("memory pool is null pointer"); + } + if (context->GetExecutor() == nullptr) { + return Status::Invalid("executor is null pointer"); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, + FormatTable::Copy(SharedSelf(), context->GetOptions())); + if (context->GetCatalog() != nullptr) { + return Status::Invalid( + "WithCatalog() requires a native table; use WriteContextBuilder(FormatTable) for a " + "format table"); + } + if (context->IsStreamingMode()) { + return Status::NotImplemented( + "a format table has no snapshots, so there is nothing a streaming write could " + "commit against"); + } + if (context->GetRealtimeContext() != nullptr) { + return Status::NotImplemented("a format table has no real-time store to write into"); + } + if (!context->GetWriteSchema().empty()) { + return Status::NotImplemented( + "a format table write takes the table's own columns; a write schema naming a " + "subset of them is not supported yet"); + } + // A write id prefixes a postpone-bucket writer's files so that one compaction reader can + // put them back in order. A format table has no buckets, so it would identify nothing. + if (context->GetWriteId().has_value()) { + return Status::NotImplemented( + "a format table has no buckets, so a write id would name nothing"); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr format_write, + FormatTableFileStoreWrite::Create(table, context->GetMemoryPool())); + return std::unique_ptr(std::move(format_write)); +} + +Result> FormatTable::NewCommit( + std::unique_ptr context) const { + if (context == nullptr) { + return Status::Invalid("commit context is null pointer"); + } + if (context->GetMemoryPool() == nullptr) { + return Status::Invalid("memory pool is null pointer"); + } + if (context->GetExecutor() == nullptr) { + return Status::Invalid("executor is null pointer"); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr given_table, + FormatTable::Copy(SharedSelf(), context->GetOptions())); + if (!context->IgnoreEmptyCommit()) { + return Status::NotImplemented( + "a format table cannot record an empty commit: keeping one means writing a " + "snapshot that adds no files, and there are no snapshots here"); + } + if (context->UseRESTCatalogCommit()) { + return Status::NotImplemented( + "a format table commits by renaming files into place, not by sending a snapshot " + "to a rest catalog"); + } + if (context->GetTableId()) { + return Status::NotImplemented( + "a format table sends no commit table request, so there is nowhere to put a table id"); + } + if (context->GetCatalog() != nullptr) { + return Status::NotImplemented( + "a format table keeps no snapshot for a catalog to take, so it commits by renaming " + "files into place rather than through the catalog it was loaded from"); + } + if (context->AppendCommitCheckConflict()) { + return Status::NotImplemented( + "a format table has no manifests to check a concurrent commit against"); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr format_commit, + FormatTableFileStoreCommit::Create(given_table)); + return std::unique_ptr(std::move(format_commit)); +} + } // namespace paimon diff --git a/src/paimon/core/table/format/format_table_loader.cpp b/src/paimon/core/table/format/format_table_loader.cpp deleted file mode 100644 index 797bc3351..000000000 --- a/src/paimon/core/table/format/format_table_loader.cpp +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "paimon/core/table/format/format_table_loader.h" - -#include -#include - -#include "fmt/format.h" -#include "paimon/catalog/identifier.h" -#include "paimon/common/utils/checked_cast.h" -#include "paimon/common/utils/path_util.h" -#include "paimon/core/options/table_type.h" -#include "paimon/core/schema/schema_manager.h" -#include "paimon/core/schema/table_schema.h" -#include "paimon/core/utils/branch_manager.h" -#include "paimon/schema/schema.h" -#include "paimon/table/format/format_table.h" - -namespace paimon { - -Result> FormatTableLoader::TryLoad( - const std::shared_ptr& file_system, const std::string& table_path, - const std::string& branch, const std::map& options, - const std::optional& specific_table_schema, const SchemaManager* schema_manager, - std::shared_ptr* table_schema_out) { - assert(table_schema_out != nullptr); - table_schema_out->reset(); - - std::shared_ptr table_schema; - if (branch == BranchManager::DEFAULT_MAIN_BRANCH && specific_table_schema) { - // Handing the schema over only saves reading it. A context always names a table path and - // this library writes the metadata under it, so `schema` and `branch` stay metadata. - PAIMON_ASSIGN_OR_RAISE(table_schema, - TableSchema::CreateFromJson(specific_table_schema.value())); - } else { - // Through the caller's manager when it has one, so that the read warms the cache it goes - // on to use rather than a cache that dies with this call. - SchemaManager own_schema_manager(file_system, table_path, branch); - const SchemaManager& reader = - schema_manager != nullptr ? *schema_manager : own_schema_manager; - PAIMON_ASSIGN_OR_RAISE(std::optional> latest_schema, - reader.Latest()); - if (!latest_schema) { - return std::shared_ptr(); - } - table_schema = *latest_schema; - } - *table_schema_out = table_schema; - - // From the schema alone: see `FormatTable::Create()`. - PAIMON_ASSIGN_OR_RAISE(bool is_format_table, - TableTypeDefine::IsFormatTable(table_schema->Options())); - if (!is_format_table) { - return std::shared_ptr(); - } - // A format table has no branches, so it is refused rather than served from the table path - // under another branch's name. See `SchemaValidation::ValidateFormatTableSchema()`. - if (BranchManager::NormalizeBranch(branch) != BranchManager::DEFAULT_MAIN_BRANCH) { - return Status::Invalid( - fmt::format("a format table has no branches, so {} cannot be read on branch '{}'", - table_path, branch)); - } - // A context names a path, never a catalog identifier, so the table is known by the directory - // it sits in. - Identifier identifier(Identifier::kUnknownDatabase, PathUtil::GetName(table_path)); - return FormatTable::Create(file_system, table_path, identifier, - checked_pointer_cast(table_schema), - /*location_carries_paimon_metadata=*/true, options); -} - -} // namespace paimon diff --git a/src/paimon/core/table/format/format_table_loader.h b/src/paimon/core/table/format/format_table_loader.h deleted file mode 100644 index 290ee94a7..000000000 --- a/src/paimon/core/table/format/format_table_loader.h +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include -#include -#include -#include - -#include "paimon/result.h" - -namespace paimon { - -class FileSystem; -class FormatTable; -class SchemaManager; -class TableSchema; - -/// Recognises a format table behind a table path. -/// -/// The scan, read, write and commit entry points all take a path and have to answer the same -/// question before they can dispatch, so one place answers it and they cannot disagree. -class FormatTableLoader { - public: - FormatTableLoader() = delete; - ~FormatTableLoader() = delete; - - /// Loads `table_path` as a format table, or returns null when it is not one. - /// - /// Null is also the answer when there is no schema under the path at all: what to say about a - /// table that is not there is the managed path's to decide, and it says it in more detail. - /// - /// The table type is read from the schema, never from `options`: `type` is structural, and one - /// call's options must not decide what kind of table this is. - /// - /// @param branch Branch the schema is read from. Take it from the same place the managed path - /// in the same entry point takes it, or the two dispatch on different schemas. Anything - /// but the main branch is refused once the schema turns out to be a format table's, - /// which has no branches to read. - /// @param options Options given at the call, which win over the ones the schema stored. - /// @param specific_table_schema A schema the caller already holds, as json, which saves - /// reading one from under the path. Used only on the main branch, as on the managed - /// read path. - /// @param schema_manager The manager to read the schema through, or null to use one of this - /// call's own. A caller that has one must pass it: a manager caches what it read. - /// @param table_schema_out Set to the schema this read, or null when there is none, so that a - /// caller falling back to the managed path does not read it twice. - static Result> TryLoad( - const std::shared_ptr& file_system, const std::string& table_path, - const std::string& branch, const std::map& options, - const std::optional& specific_table_schema, - const SchemaManager* schema_manager, std::shared_ptr* table_schema_out); -}; - -} // namespace paimon diff --git a/src/paimon/core/table/format/format_table_test.cpp b/src/paimon/core/table/format/format_table_test.cpp index e8391f68c..2430ffbda 100644 --- a/src/paimon/core/table/format/format_table_test.cpp +++ b/src/paimon/core/table/format/format_table_test.cpp @@ -34,6 +34,7 @@ #include "arrow/c/bridge.h" #include "gtest/gtest.h" #include "paimon/cache/cache.h" +#include "paimon/catalog/table.h" #include "paimon/commit_context.h" #include "paimon/commit_message.h" #include "paimon/common/table/special_fields.h" @@ -58,6 +59,7 @@ #include "paimon/memory/memory_pool.h" #include "paimon/predicate/literal.h" #include "paimon/predicate/predicate_builder.h" +#include "paimon/predicate/vector_search.h" #include "paimon/read_context.h" #include "paimon/record_batch.h" #include "paimon/scan_context.h" @@ -2843,6 +2845,159 @@ TEST(FormatTableTest, TestTheGenericEntryPointsTakeALoadedFormatTable) { ASSERT_EQ(rows, 3); } +TEST(FormatTableTest, TestATableServesAFormatTableThroughItsOwnEntryPoints) { + // `NewScan()` and its kin take the table they were called on as the one to serve, the way Java + // Paimon's `Table` serves both kinds through one `newReadBuilder()`, so a caller holding a + // format table as the `Table` a catalog hands out reaches it through them as well. The + // contexts name a path only and carry no format table, so it is the table itself that + // answers. + std::unique_ptr dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + std::shared_ptr
table; + ASSERT_OK_AND_ASSIGN(table, CreateTable(dir->GetFileSystem(), dir->Str(), {"dt"})); + + WriteContextBuilder write_builder(dir->Str(), "test-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write, + table->NewWrite(std::move(write_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, + MakeBatch({1, 2}, {"alice", "bob"}, "20240101", {{"dt", "20240101"}})); + ASSERT_OK(write->Write(std::move(batch))); + ASSERT_OK_AND_ASSIGN(std::vector> messages, + write->PrepareCommit()); + + CommitContextBuilder commit_builder(dir->Str(), "test-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, + table->NewCommit(std::move(commit_context))); + ASSERT_OK(commit->Commit(messages)); + ASSERT_OK(write->Close()); + + ScanContextBuilder scan_builder(dir->Str()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_EQ(plan->Splits().size(), 1u); + + ReadContextBuilder read_builder(dir->Str()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, table->NewRead(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, read->CreateReader(plan->Splits())); + std::vector rows; + while (true) { + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch read_batch, reader->NextBatch()); + if (BatchReader::IsEofBatch(read_batch)) { + break; + } + ASSERT_OK_AND_ASSIGN(std::shared_ptr record_batch, + ImportBatch(read_batch)); + // `_VALUE_KIND` first, then the table's own columns, exactly as the generic entry points + // give them. + auto ids = checked_pointer_cast(record_batch->column(1)); + auto names = checked_pointer_cast(record_batch->column(2)); + auto dts = checked_pointer_cast(record_batch->column(3)); + for (int64_t i = 0; i < record_batch->num_rows(); i++) { + rows.push_back(std::to_string(ids->Value(i)) + "|" + names->GetString(i) + "|" + + dts->GetString(i)); + } + } + reader->Close(); + ASSERT_EQ(rows, (std::vector{"1|alice|20240101", "2|bob|20240101"})); +} + +TEST(FormatTableTest, TestTheTableEntryPointsRefuseWhatAFormatTableCannotHonour) { + // A refusal is a promise about what was not done, so the table's own entry points must make + // the same promises the generic ones do. Each context is handed to both, and the answer has + // to come out the same whichever way in the caller takes. + std::unique_ptr dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN(std::shared_ptr format_table, + CreateTable(dir->GetFileSystem(), dir->Str(), {"dt"})); + std::shared_ptr
table = format_table; + + // A context that never was is refused first of all, and by name, exactly as the generic + // entry points refuse one. + ASSERT_NOK_WITH_MSG(TableScan::Create(nullptr), "scan context is null pointer"); + ASSERT_NOK_WITH_MSG(table->NewScan(nullptr), "scan context is null pointer"); + ASSERT_NOK_WITH_MSG(TableRead::Create(nullptr), "read context is null pointer"); + ASSERT_NOK_WITH_MSG(table->NewRead(nullptr), "read context is null pointer"); + ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(nullptr), "write context is null pointer"); + ASSERT_NOK_WITH_MSG(table->NewWrite(nullptr), "write context is null pointer"); + ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(nullptr), "commit context is null pointer"); + ASSERT_NOK_WITH_MSG(table->NewCommit(nullptr), "commit context is null pointer"); + + // A streaming scan follows snapshots, which a format table keeps none of. + auto streaming_scan_context = [&dir]() -> Result> { + ScanContextBuilder builder(dir->Str()); + builder.WithStreamingMode(true); + return builder.Finish(); + }; + ASSERT_OK_AND_ASSIGN(std::unique_ptr streaming_context, streaming_scan_context()); + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(streaming_context)), + "nothing for a streaming scan to follow"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr streaming_context_again, + streaming_scan_context()); + ASSERT_NOK_WITH_MSG(table->NewScan(std::move(streaming_context_again)), + "nothing for a streaming scan to follow"); + + // Vector search needs an index a format table never builds. + auto vector_read_context = [&dir]() -> Result> { + ReadContextBuilder builder(dir->Str()); + builder.SetVectorSearch(std::make_shared( + /*field_name=*/"name", /*limit=*/1, std::vector{0.0f}, /*pre_filter=*/nullptr, + /*predicate=*/nullptr, /*distance_type=*/std::nullopt, + /*options=*/std::map{})); + return builder.Finish(); + }; + ASSERT_OK_AND_ASSIGN(std::unique_ptr vector_context, vector_read_context()); + ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(vector_context)), + "not supported for format tables"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr vector_context_again, vector_read_context()); + ASSERT_NOK_WITH_MSG(table->NewRead(std::move(vector_context_again)), + "not supported for format tables"); + + // A catalog takes the snapshot a native table commits, and a format table has none to give. + auto catalog = std::make_shared(); + catalog->SetTableSchema(format_table->LatestSchema()); + + auto catalog_write_context = [&dir, &catalog]() -> Result> { + WriteContextBuilder builder(dir->Str(), "test-user"); + return builder.WithCatalog(catalog, Identifier("db", "tbl")).Finish(); + }; + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog_write, catalog_write_context()); + ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(catalog_write)), + "use WriteContextBuilder(FormatTable)"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog_write_again, + catalog_write_context()); + ASSERT_NOK_WITH_MSG(table->NewWrite(std::move(catalog_write_again)), + "use WriteContextBuilder(FormatTable)"); + + auto catalog_commit_context = [&dir, &catalog]() -> Result> { + CommitContextBuilder builder(dir->Str(), "test-user"); + return builder.WithCatalog(catalog, Identifier("db", "fmt")).Finish(); + }; + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog_commit, catalog_commit_context()); + ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(catalog_commit)), + "keeps no snapshot for a catalog to take"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog_commit_again, + catalog_commit_context()); + ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(catalog_commit_again)), + "keeps no snapshot for a catalog to take"); + + // A write id prefixes a postpone-bucket writer's files; a format table has no buckets. + auto write_id_context = [&dir]() -> Result> { + WriteContextBuilder builder(dir->Str(), "test-user"); + builder.WithWriteId(3); + return builder.Finish(); + }; + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_id, write_id_context()); + ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_id)), + "a write id would name nothing"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_id_again, write_id_context()); + ASSERT_NOK_WITH_MSG(table->NewWrite(std::move(write_id_again)), + "a write id would name nothing"); +} + TEST(FormatTableTest, TestAContextBuiltFromAFormatTableRefusesASecondAnswer) { // The table already carries its schema and the file system it was loaded through, from a // source the context cannot see behind, so a second answer is refused rather than dropped. diff --git a/src/paimon/core/table/table.cpp b/src/paimon/core/table/table.cpp index dd7e18f20..46287881b 100644 --- a/src/paimon/core/table/table.cpp +++ b/src/paimon/core/table/table.cpp @@ -30,14 +30,17 @@ #include "arrow/c/bridge.h" #include "fmt/format.h" #include "paimon/catalog/catalog.h" +#include "paimon/catalog/identifier.h" #include "paimon/commit_context.h" #include "paimon/common/global_index/btree/btree_defs.h" #include "paimon/common/predicate/predicate_validator.h" #include "paimon/common/types/data_field.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/binary_row_partition_computer.h" +#include "paimon/common/utils/checked_cast.h" #include "paimon/common/utils/fields_comparator.h" #include "paimon/common/utils/options_utils.h" +#include "paimon/common/utils/path_util.h" #include "paimon/common/utils/string_utils.h" #include "paimon/core/catalog/catalog_snapshot_commit.h" #include "paimon/core/catalog/catalog_utils.h" @@ -66,6 +69,7 @@ #include "paimon/core/operation/key_value_file_store_scan.h" #include "paimon/core/operation/key_value_file_store_write.h" #include "paimon/core/options/merge_engine.h" +#include "paimon/core/options/table_type.h" #include "paimon/core/postpone/postpone_bucket_file_store_write.h" #include "paimon/core/realtime/realtime_context_impl.h" #include "paimon/core/realtime/realtime_schema_layout.h" @@ -73,11 +77,7 @@ #include "paimon/core/schema/schema_validation.h" #include "paimon/core/schema/table_schema.h" #include "paimon/core/table/bucket_mode.h" -#include "paimon/core/table/format/format_table_file_store_commit.h" -#include "paimon/core/table/format/format_table_file_store_write.h" -#include "paimon/core/table/format/format_table_loader.h" -#include "paimon/core/table/format/format_table_read.h" -#include "paimon/core/table/format/format_table_scan.h" +#include "paimon/core/table/file_store_table.h" #include "paimon/core/table/source/abstract_table_scan.h" #include "paimon/core/table/source/append_only_split_generator.h" #include "paimon/core/table/source/append_only_table_read.h" @@ -131,6 +131,45 @@ class MergeFunctionWrapper; namespace { +/// Recognises a format table behind `table_path`, or returns null when it is not one. +/// +/// The scan, read, write and commit entry points below all take a path and have to answer the +/// same question before they can dispatch, so one place answers it and they cannot disagree. +/// Null is also the answer when no schema was found under the path at all: what to say about a +/// table that is not there is the managed path's to decide, and it says it in more detail. +/// +/// The table type is read from the schema, never from `options`: `type` is structural, and one +/// call's options must not decide what kind of table this is. `branch` names the branch the +/// schema was read from: anything but the main branch is refused once the schema turns out to be +/// a format table's, which has no branches to read. +Result> TryLoadFormatTable( + const std::shared_ptr& file_system, const std::string& table_path, + const std::string& branch, const std::map& options, + const std::shared_ptr& table_schema) { + if (table_schema == nullptr) { + return std::shared_ptr(); + } + // From the schema alone: see `FormatTable::Create()`. + PAIMON_ASSIGN_OR_RAISE(bool is_format_table, + TableTypeDefine::IsFormatTable(table_schema->Options())); + if (!is_format_table) { + return std::shared_ptr(); + } + // A format table has no branches, so it is refused rather than served from the table path + // under another branch's name. See `SchemaValidation::ValidateFormatTableSchema()`. + if (BranchManager::NormalizeBranch(branch) != BranchManager::DEFAULT_MAIN_BRANCH) { + return Status::Invalid( + fmt::format("a format table has no branches, so {} cannot be read on branch '{}'", + table_path, branch)); + } + // A context names a path, never a catalog identifier, so the table is known by the directory + // it sits in. + Identifier identifier(Identifier::kUnknownDatabase, PathUtil::GetName(table_path)); + return FormatTable::Create(file_system, table_path, identifier, + checked_pointer_cast(table_schema), + /*location_carries_paimon_metadata=*/true, options); +} + Status RestoreRealtimeCommittedProgress(const std::shared_ptr& realtime_context, const std::shared_ptr& snapshot_manager, const CoreOptions& options) { @@ -149,39 +188,6 @@ Status RestoreRealtimeCommittedProgress(const std::shared_ptr& return Status::OK(); } -/// Maps a `WriteContext` onto `FormatTableFileStoreWrite`, refusing by name what a format table -/// cannot honour rather than silently dropping it. -Result> NewFormatTableWrite( - const std::shared_ptr& table, const WriteContext& ctx) { - if (ctx.GetCatalog() != nullptr) { - return Status::Invalid( - "WithCatalog() requires a native table; use WriteContextBuilder(FormatTable) for a " - "format table"); - } - if (ctx.IsStreamingMode()) { - return Status::NotImplemented( - "a format table has no snapshots, so there is nothing a streaming write could " - "commit against"); - } - if (ctx.GetRealtimeContext() != nullptr) { - return Status::NotImplemented("a format table has no real-time store to write into"); - } - if (!ctx.GetWriteSchema().empty()) { - return Status::NotImplemented( - "a format table write takes the table's own columns; a write schema naming a " - "subset of them is not supported yet"); - } - // A write id prefixes a postpone-bucket writer's files so that one compaction reader can - // put them back in order. A format table has no buckets, so it would identify nothing. - if (ctx.GetWriteId().has_value()) { - return Status::NotImplemented( - "a format table has no buckets, so a write id would name nothing"); - } - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr format_write, - FormatTableFileStoreWrite::Create(table, ctx.GetMemoryPool())); - return std::unique_ptr(std::move(format_write)); -} - Result HasHistoricalPrimaryKeyBTreeDefinition( const std::shared_ptr& schema_manager, int64_t current_schema_id) { if (current_schema_id <= TableSchema::FIRST_SCHEMA_ID) { @@ -215,20 +221,22 @@ Table::Table(const std::shared_ptr& file_system, const std::string& identifier_(identifier), uuid_(uuid) {} -Result> Table::NewScan(std::unique_ptr context) const { +Result> FileStoreTable::NewScan( + std::unique_ptr context) const { return TableScan::Create(std::move(context)); } -Result> Table::NewRead(std::unique_ptr context) const { +Result> FileStoreTable::NewRead( + std::unique_ptr context) const { return TableRead::Create(std::move(context)); } -Result> Table::NewWrite( +Result> FileStoreTable::NewWrite( std::unique_ptr context) const { return FileStoreWrite::Create(std::move(context)); } -Result> Table::NewCommit( +Result> FileStoreTable::NewCommit( std::unique_ptr context) const { return FileStoreCommit::Create(std::move(context)); } @@ -246,9 +254,7 @@ Result> FileStoreWrite::Create(std::unique_ptrGetFormatTable() != nullptr) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr given_table, - FormatTable::Copy(ctx->GetFormatTable(), ctx->GetOptions())); - return NewFormatTableWrite(given_table, *ctx); + return ctx->GetFormatTable()->NewWrite(std::move(ctx)); } PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog())); @@ -293,16 +299,28 @@ Result> FileStoreWrite::Create(std::unique_ptr(tmp_options.GetFileSystem(), ctx->GetRootPath(), branch); std::shared_ptr latest_schema; - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr format_table, - FormatTableLoader::TryLoad(tmp_options.GetFileSystem(), ctx->GetRootPath(), branch, - ctx->GetOptions(), catalog_table_schema, schema_manager.get(), - &latest_schema)); + if (branch == BranchManager::DEFAULT_MAIN_BRANCH && catalog_table_schema) { + // Handing the schema over only saves reading it. A context always names a table path and + // this library writes the metadata under it, so `schema` and `branch` stay metadata. + PAIMON_ASSIGN_OR_RAISE(latest_schema, + TableSchema::CreateFromJson(catalog_table_schema.value())); + } else { + // Through the manager built above, so the read warms the cache the managed path below + // goes on to use. + PAIMON_ASSIGN_OR_RAISE(std::optional> latest, + schema_manager->Latest()); + if (latest) { + latest_schema = *latest; + } + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, + TryLoadFormatTable(tmp_options.GetFileSystem(), ctx->GetRootPath(), + branch, ctx->GetOptions(), latest_schema)); if (format_table != nullptr) { - return NewFormatTableWrite(format_table, *ctx); + return format_table->NewWrite(std::move(ctx)); } - // The schema the dispatch above already read through `schema_manager`, rather than a second - // read of the same file. + // The schema the branch read above through `schema_manager`, rather than a second read of + // the same file. if (latest_schema == nullptr) { return Status::Invalid(fmt::format("not found latest schema in branch {}", branch)); } @@ -679,53 +697,6 @@ class TableScanImpl { Result> NewDataTableScan( const std::shared_ptr& context, const std::shared_ptr& loaded_schema); -/// Maps a `ScanContext` onto `FormatTableScan`, which takes one partition filter and a limit and -/// nothing else. -/// -/// A format table has no buckets, so a bucket filter has nothing to act on. A predicate is -/// refused too, which is narrower than Java: `FormatReadBuilder.newScan()` hands the partition -/// half of one to the scan to prune directories with, which this does not do yet. -Result> NewFormatTableScan(const std::shared_ptr& table, - const std::shared_ptr& context) { - // Anything a `ScanContext` carries that a format table cannot honour is refused rather than - // silently dropped. - if (context->IsStreamingMode()) { - return Status::NotImplemented( - "a format table has no snapshots, so there is nothing for a streaming scan to follow"); - } - if (context->GetRealtimeContext() != nullptr) { - return Status::NotImplemented( - "a format table has no real-time store to union with what is on disk"); - } - if (context->GetGlobalIndexResult() != nullptr) { - return Status::NotImplemented("a format table carries no global index"); - } - std::map partition_filter; - std::shared_ptr filters = context->GetScanFilters(); - if (filters != nullptr) { - if (filters->GetPredicate() != nullptr) { - return Status::NotImplemented( - "a format table scan does not take a predicate: its files carry no statistics to " - "skip by, and pruning by the partition columns a predicate names is not " - "implemented yet; give the partition values as a partition filter instead"); - } - if (filters->GetBucketFilter()) { - return Status::NotImplemented("a format table has no buckets to filter by"); - } - const std::vector>& partition_filters = - filters->GetPartitionFilters(); - if (partition_filters.size() > 1) { - return Status::NotImplemented("a format table scan takes at most one partition filter"); - } - if (partition_filters.size() == 1) { - partition_filter = partition_filters.front(); - } - } - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, - FormatTableScan::Create(table, partition_filter, context->GetLimit())); - return std::unique_ptr(std::move(scan)); -} - } // namespace Result> TableScan::Create(std::unique_ptr context) { @@ -739,43 +710,49 @@ Result> TableScan::Create(std::unique_ptr shared_context = std::move(context); // A table the caller already loaded says what it is, so nothing is read to find out. - if (shared_context->GetFormatTable() != nullptr) { - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr format_table, - FormatTable::Copy(shared_context->GetFormatTable(), shared_context->GetOptions())); - return NewFormatTableScan(format_table, shared_context); + if (context->GetFormatTable() != nullptr) { + return context->GetFormatTable()->NewScan(std::move(context)); } // load schema - PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, - CoreOptions::FromMap(shared_context->GetOptions(), - shared_context->GetSpecificFileSystem(), {})); + PAIMON_ASSIGN_OR_RAISE( + CoreOptions tmp_options, + CoreOptions::FromMap(context->GetOptions(), context->GetSpecificFileSystem(), {})); PAIMON_ASSIGN_OR_RAISE(std::optional system_table_path, - SystemTableLoader::TryParsePath(shared_context->GetPath())); + SystemTableLoader::TryParsePath(context->GetPath())); if (system_table_path) { PAIMON_ASSIGN_OR_RAISE( std::shared_ptr system_table, - SystemTableLoader::LoadFromPath(tmp_options.GetFileSystem(), shared_context->GetPath(), - shared_context->GetOptions())); - return system_table->NewScan(shared_context); + SystemTableLoader::LoadFromPath(tmp_options.GetFileSystem(), context->GetPath(), + context->GetOptions())); + return system_table->NewScan(std::move(context)); } // A format table is planned by listing directories, so it never reaches the manifest path // below. One `TableScan` interface serves both, as Java Paimon serves both through one // `ReadBuilder`. + const std::string branch = BranchManager::NormalizeBranch(tmp_options.GetBranch()); std::shared_ptr latest_schema; - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr format_table, - FormatTableLoader::TryLoad(tmp_options.GetFileSystem(), shared_context->GetPath(), - BranchManager::NormalizeBranch(tmp_options.GetBranch()), - shared_context->GetOptions(), - shared_context->GetSpecificTableSchema(), - /*schema_manager=*/nullptr, &latest_schema)); + if (branch == BranchManager::DEFAULT_MAIN_BRANCH && context->GetSpecificTableSchema()) { + // Handing the schema over only saves reading it. A context always names a table path and + // this library writes the metadata under it, so `schema` and `branch` stay metadata. + PAIMON_ASSIGN_OR_RAISE( + latest_schema, TableSchema::CreateFromJson(context->GetSpecificTableSchema().value())); + } else { + SchemaManager schema_manager(tmp_options.GetFileSystem(), context->GetPath(), branch); + PAIMON_ASSIGN_OR_RAISE(std::optional> latest, + schema_manager.Latest()); + if (latest) { + latest_schema = *latest; + } + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, + TryLoadFormatTable(tmp_options.GetFileSystem(), context->GetPath(), + branch, context->GetOptions(), latest_schema)); if (format_table != nullptr) { - return NewFormatTableScan(format_table, shared_context); + return format_table->NewScan(std::move(context)); } - // With the schema the dispatch already read, so the managed path does not read it again. - return NewDataTableScan(shared_context, latest_schema); + // With the schema already read above, so the managed path does not read it again. + return NewDataTableScan(std::move(context), latest_schema); } namespace { @@ -822,29 +799,16 @@ Status ValidateRealtimeScan(const TableSchema& table_schema, const CoreOptions& return Status::OK(); } +/// @param loaded_schema The schema `TableScan::Create()` already read while dispatching on the +/// table type, or null when neither the caller's json nor the branch's own directory had +/// one. Result> NewDataTableScan( const std::shared_ptr& context, const std::shared_ptr& loaded_schema) { - PAIMON_ASSIGN_OR_RAISE( - CoreOptions tmp_options, - CoreOptions::FromMap(context->GetOptions(), context->GetSpecificFileSystem(), {})); - std::string branch = BranchManager::NormalizeBranch(tmp_options.GetBranch()); - std::shared_ptr table_schema; - const auto& specific_table_schema = context->GetSpecificTableSchema(); - if (loaded_schema != nullptr) { - table_schema = loaded_schema; - } else if (branch == BranchManager::DEFAULT_MAIN_BRANCH && specific_table_schema) { - PAIMON_ASSIGN_OR_RAISE(table_schema, - TableSchema::CreateFromJson(specific_table_schema.value())); - } else { - SchemaManager schema_manager(tmp_options.GetFileSystem(), context->GetPath(), branch); - PAIMON_ASSIGN_OR_RAISE(std::optional> latest_table_schema, - schema_manager.Latest()); - if (latest_table_schema == std::nullopt) { - return Status::Invalid("not found latest schema"); - } - table_schema = latest_table_schema.value(); + if (loaded_schema == nullptr) { + return Status::Invalid("not found latest schema"); } + const std::shared_ptr& table_schema = loaded_schema; PAIMON_ASSIGN_OR_RAISE(bool read_optimized, OptionsUtils::GetValueFromMap(context->GetOptions(), kReadOptimizedScanOption, false)); @@ -1054,84 +1018,69 @@ Result> NewDataTableRead( std::move(fallback_table_read), arrow_pool); } -/// Maps a `ReadContext` onto `FormatTableRead`, which reads everything it needs out of the context -/// itself: the columns, the predicate, the pool and executor, and what a file is opened with. It -/// refuses by name what a format table cannot honour. -Result> NewFormatTableRead(const std::shared_ptr& table, - const std::shared_ptr& context) { - if (PredicateUtils::ContainsFullTextMatch(context->GetPredicate())) { - return Status::NotImplemented("FullTextMatch predicate execution is not implemented yet"); - } - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, - FormatTableRead::Create(table, context)); - return std::unique_ptr(std::move(read)); -} - } // namespace Result> TableRead::Create(std::unique_ptr ctx) { - std::shared_ptr context = std::move(ctx); - if (context == nullptr) { + if (ctx == nullptr) { return Status::Invalid("read context is null pointer"); } - if (context->GetMemoryPool() == nullptr) { + if (ctx->GetMemoryPool() == nullptr) { return Status::Invalid("memory pool is null pointer"); } - if (context->GetExecutor() == nullptr) { + if (ctx->GetExecutor() == nullptr) { return Status::Invalid("executor is null pointer"); } // A table the caller already loaded says what it is, so nothing is read to find out. - if (context->GetFormatTable() != nullptr) { - if (context->HasVectorOrFullTextSearch()) { - return Status::NotImplemented( - "VectorSearch or FullTextSearch is not supported for format tables"); - } - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, - FormatTable::Copy(context->GetFormatTable(), context->GetOptions())); - return NewFormatTableRead(format_table, context); + if (ctx->GetFormatTable() != nullptr) { + return ctx->GetFormatTable()->NewRead(std::move(ctx)); } - PAIMON_ASSIGN_OR_RAISE( - CoreOptions tmp_core_options, - CoreOptions::FromMap(context->GetOptions(), context->GetSpecificFileSystem(), - context->GetFileSystemSchemeToIdentifierMap())); + PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_core_options, + CoreOptions::FromMap(ctx->GetOptions(), ctx->GetSpecificFileSystem(), + ctx->GetFileSystemSchemeToIdentifierMap())); PAIMON_ASSIGN_OR_RAISE(std::optional system_table_path, - SystemTableLoader::TryParsePath(context->GetPath())); + SystemTableLoader::TryParsePath(ctx->GetPath())); if (system_table_path) { - if (PredicateUtils::ContainsFullTextMatch(context->GetPredicate())) { + if (PredicateUtils::ContainsFullTextMatch(ctx->GetPredicate())) { return Status::NotImplemented( "FullTextMatch predicate execution is not implemented yet"); } - if (context->HasVectorOrFullTextSearch()) { + if (ctx->HasVectorOrFullTextSearch()) { return Status::NotImplemented( "VectorSearch or FullTextSearch is not supported for system tables"); } - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr system_table, - SystemTableLoader::LoadFromPath(tmp_core_options.GetFileSystem(), context->GetPath(), - context->GetOptions())); - return system_table->NewRead(context); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, + SystemTableLoader::LoadFromPath(tmp_core_options.GetFileSystem(), + ctx->GetPath(), ctx->GetOptions())); + return system_table->NewRead(std::move(ctx)); } // A format table has no manifest and a file another engine wrote carries no field ids, so // which files there are and how a row is put back together is its own. How a file is opened // is not: both paths go through `DataFileReaderFactory`. + const std::string branch = ctx->GetBranch(); std::shared_ptr latest_schema; - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr format_table, - FormatTableLoader::TryLoad(tmp_core_options.GetFileSystem(), context->GetPath(), - context->GetBranch(), context->GetOptions(), - context->GetSpecificTableSchema(), - /*schema_manager=*/nullptr, &latest_schema)); - if (format_table != nullptr) { - if (context->HasVectorOrFullTextSearch()) { - return Status::NotImplemented( - "VectorSearch or FullTextSearch is not supported for format tables"); + if (branch == BranchManager::DEFAULT_MAIN_BRANCH && ctx->GetSpecificTableSchema()) { + // Handing the schema over only saves reading it. A context always names a table path and + // this library writes the metadata under it, so `schema` and `branch` stay metadata. + PAIMON_ASSIGN_OR_RAISE(latest_schema, + TableSchema::CreateFromJson(ctx->GetSpecificTableSchema().value())); + } else { + SchemaManager schema_manager(tmp_core_options.GetFileSystem(), ctx->GetPath(), branch); + PAIMON_ASSIGN_OR_RAISE(std::optional> latest, + schema_manager.Latest()); + if (latest) { + latest_schema = *latest; } - return NewFormatTableRead(format_table, context); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, + TryLoadFormatTable(tmp_core_options.GetFileSystem(), ctx->GetPath(), + branch, ctx->GetOptions(), latest_schema)); + if (format_table != nullptr) { + return format_table->NewRead(std::move(ctx)); } - // With the schema the dispatch already read, so the managed path does not read it again. - return NewDataTableRead(context, latest_schema); + // With the schema already read above, so the managed path does not read it again. + return NewDataTableRead(std::move(ctx), latest_schema); } namespace { @@ -1216,37 +1165,6 @@ Result> NewSnapshotCommit( std::make_shared(options.GetFileSystem(), snapshot_manager)); } -/// Creates a format-table commit, rejecting non-default options that require snapshots. -Result> NewFormatTableCommit( - const std::shared_ptr& table, const CommitContext& ctx) { - if (!ctx.IgnoreEmptyCommit()) { - return Status::NotImplemented( - "a format table cannot record an empty commit: keeping one means writing a " - "snapshot that adds no files, and there are no snapshots here"); - } - if (ctx.UseRESTCatalogCommit()) { - return Status::NotImplemented( - "a format table commits by renaming files into place, not by sending a snapshot " - "to a rest catalog"); - } - if (ctx.GetTableId()) { - return Status::NotImplemented( - "a format table sends no commit table request, so there is nowhere to put a table id"); - } - if (ctx.GetCatalog() != nullptr) { - return Status::NotImplemented( - "a format table keeps no snapshot for a catalog to take, so it commits by renaming " - "files into place rather than through the catalog it was loaded from"); - } - if (ctx.AppendCommitCheckConflict()) { - return Status::NotImplemented( - "a format table has no manifests to check a concurrent commit against"); - } - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr format_commit, - FormatTableFileStoreCommit::Create(table)); - return std::unique_ptr(std::move(format_commit)); -} - } // namespace Result> FileStoreCommit::Create( @@ -1263,9 +1181,7 @@ Result> FileStoreCommit::Create( // A table the caller already loaded says what it is, so nothing is read to find out. if (ctx->GetFormatTable() != nullptr) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr given_table, - FormatTable::Copy(ctx->GetFormatTable(), ctx->GetOptions())); - return NewFormatTableCommit(given_table, *ctx); + return ctx->GetFormatTable()->NewCommit(std::move(ctx)); } PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog())); @@ -1330,12 +1246,25 @@ Result> FileStoreCommit::Create( auto schema_manager = std::make_shared(tmp_options.GetFileSystem(), root_path, branch); std::shared_ptr latest_schema; + if (branch == BranchManager::DEFAULT_MAIN_BRANCH && catalog_table_schema) { + // Handing the schema over only saves reading it. A context always names a table path and + // this library writes the metadata under it, so `schema` and `branch` stay metadata. + PAIMON_ASSIGN_OR_RAISE(latest_schema, + TableSchema::CreateFromJson(catalog_table_schema.value())); + } else { + // Through the manager built above, so the read warms the cache the managed path below + // goes on to use. + PAIMON_ASSIGN_OR_RAISE(std::optional> latest, + schema_manager->Latest()); + if (latest) { + latest_schema = *latest; + } + } PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, - FormatTableLoader::TryLoad( - tmp_options.GetFileSystem(), root_path, branch, ctx->GetOptions(), - catalog_table_schema, schema_manager.get(), &latest_schema)); + TryLoadFormatTable(tmp_options.GetFileSystem(), root_path, branch, + ctx->GetOptions(), latest_schema)); if (format_table != nullptr) { - return NewFormatTableCommit(format_table, *ctx); + return format_table->NewCommit(std::move(ctx)); } if (latest_schema == nullptr) { return Status::Invalid(fmt::format("not found latest schema in branch {}", branch)); diff --git a/src/paimon/core/table/table_test.cpp b/src/paimon/core/table/table_test.cpp index 4792a8461..78d01e5a5 100644 --- a/src/paimon/core/table/table_test.cpp +++ b/src/paimon/core/table/table_test.cpp @@ -31,11 +31,13 @@ #include "paimon/core/catalog/file_system_catalog.h" #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/file_store_table.h" #include "paimon/file_store_commit.h" #include "paimon/file_store_write.h" #include "paimon/read_context.h" #include "paimon/scan_context.h" #include "paimon/schema/schema.h" +#include "paimon/table/format/format_table.h" #include "paimon/table/source/table_read.h" #include "paimon/table/source/table_scan.h" #include "paimon/testing/mock/mock_catalog.h" @@ -57,7 +59,7 @@ Result> CreateManagedTable(const std::shared_ptr created, schema_manager.CreateTable(schema, /*partition_keys=*/{}, /*primary_keys=*/{}, /*options=*/{})); - return std::make_shared
( + return std::make_shared( file_system, path, checked_pointer_cast(std::shared_ptr(std::move(created))), identifier); } @@ -164,7 +166,7 @@ TEST(TableTest, TestMainBranchAliasResolvesToTheMainBranch) { SCOPED_TRACE(main_identifier.ToString()); // A catalog that loaded the schema itself calls the constructor directly, which a // rest catalog is shaped like. The alias resolves to the main branch as well. - Table table(fs, dir->Str(), schema, main_identifier); + FileStoreTable table(fs, dir->Str(), schema, main_identifier); ASSERT_OK_AND_ASSIGN(std::string branch, table.CurrentBranch()); EXPECT_EQ("main", branch); } @@ -211,10 +213,9 @@ TEST(TableTest, TestCatalogOpenedTableServesThePerTableFileSystem) { EXPECT_EQ(opened->GetRootPath(), ""); } -TEST(TableTest, TestGetTableRejectsATableTypeThatIsNotAManagedTable) { - // A `Table` describes a managed paimon table. Handing back another type as one would promise - // snapshots and manifests it never had, so the failure belongs at the call that opens the - // table rather than at the first read. +TEST(TableTest, TestGetTableServesEachKindOfTableAsWhatItIs) { + // The table's `type` option says what it is, so the one call that opens a table hands each + // kind back as what it is, through the `Table` interface the two kinds share. auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); auto fs = dir->GetFileSystem(); @@ -230,7 +231,10 @@ TEST(TableTest, TestGetTableRejectsATableTypeThatIsNotAManagedTable) { format_schema_manager.CreateTable(schema, /*partition_keys=*/{"dt"}, /*primary_keys=*/{}, {{"type", "format-table"}, {"file.format", "parquet"}})); - ASSERT_NOK_WITH_MSG(catalog.GetTable(Identifier("db", "format")), "Cannot open format table"); + // A format table comes back as the `FormatTable` it is. + ASSERT_OK_AND_ASSIGN(std::shared_ptr
opened, + catalog.GetTable(Identifier("db", "format"))); + ASSERT_NE(nullptr, std::dynamic_pointer_cast(opened)); // A managed table with no `type` option at all is what a `Table` is for, and still opens. const std::string managed_dir = dir->Str() + "/db.db/managed"; diff --git a/src/paimon/rest/rest_catalog.cpp b/src/paimon/rest/rest_catalog.cpp index 273804d0d..ad005f58a 100644 --- a/src/paimon/rest/rest_catalog.cpp +++ b/src/paimon/rest/rest_catalog.cpp @@ -32,8 +32,10 @@ #include "paimon/common/utils/string_utils.h" #include "paimon/core/catalog/catalog_utils.h" #include "paimon/core/core_options.h" +#include "paimon/core/options/table_type.h" #include "paimon/core/schema/schema_validation.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/file_store_table.h" #include "paimon/core/table/system/global_system_tables.h" #include "paimon/core/table/system/system_table.h" #include "paimon/core/table/system/system_table_schema.h" @@ -431,36 +433,35 @@ Result> RestCatalog::LoadTableSchema(const Identifier& i return checked_pointer_cast(schema); } -Result> RestCatalog::LoadFormatTable( - const Identifier& identifier) const { - // A system table is already refused by `Catalog::GetFormatTable()`, which every caller goes - // through, so this only has to load. - PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); - branch = NormalizeBranch(std::move(branch)); - PAIMON_ASSIGN_OR_RAISE(Identifier load_identifier, ToLoadIdentifier(identifier)); - std::string location; - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr schema, - LoadDataTableSchema(load_identifier, branch, &location, /*table_id=*/nullptr)); - // A rest catalog holds the schema itself, so everything below the location is data. - return FormatTable::Create(fs_, location, identifier, checked_pointer_cast(schema), - /*location_carries_paimon_metadata=*/false, - /*dynamic_options=*/{}); -} - Result> RestCatalog::GetTable(const Identifier& identifier) const { // Read UUID, path and schema together to avoid mixing table generations after recreation. std::string uuid; std::string table_path; PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, LoadTableSchema(identifier, &uuid, &table_path)); + // The schema the same response carried says what the table is: a format table is handed + // back as the `FormatTable` it is, built from the location of the same response, rather + // than refused as the managed table it is not. + std::shared_ptr data_schema = std::dynamic_pointer_cast(schema); + if (data_schema != nullptr) { + PAIMON_ASSIGN_OR_RAISE(bool is_format_table, + TableTypeDefine::IsFormatTable(data_schema->Options())); + if (is_format_table) { + // A rest catalog holds the schema itself, so everything below the location is data. + // The catalog-wide file system, as a format table reads no metadata of its own. + return FormatTable::Create(fs_, table_path, identifier, data_schema, + /*location_carries_paimon_metadata=*/false, + /*dynamic_options=*/{}); + } + } PAIMON_RETURN_NOT_OK( CatalogUtils::CheckManagedTableType(identifier, schema, "Catalog::GetTable")); // The table holds the file system its own data is served by, which a server issuing data // tokens binds to this table. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table_file_system, GetTableFileSystem(identifier)); - return std::make_shared
(table_file_system, table_path, schema, identifier, uuid); + return std::make_shared(table_file_system, table_path, schema, identifier, + uuid); } std::string RestCatalog::GetRootPath() const { diff --git a/src/paimon/rest/rest_catalog.h b/src/paimon/rest/rest_catalog.h index 2f57e912d..076346df8 100644 --- a/src/paimon/rest/rest_catalog.h +++ b/src/paimon/rest/rest_catalog.h @@ -117,12 +117,6 @@ class RestCatalog : public Catalog, public VersionManagedCatalog { /// Options merged with the server side config. const std::map& GetOptions() const override; - protected: - /// Loads the location and the schema from one `GetTable` response, so the table cannot be - /// built from a location and a schema that two requests disagreed on. - Result> LoadFormatTable( - const Identifier& identifier) const override; - private: RestCatalog(std::shared_ptr api, const std::shared_ptr& fs, const std::string& warehouse, bool data_token_enabled, bool fs_explicitly_supplied, diff --git a/src/paimon/rest/rest_catalog_test.cpp b/src/paimon/rest/rest_catalog_test.cpp index 147a908ce..94ccc19a1 100644 --- a/src/paimon/rest/rest_catalog_test.cpp +++ b/src/paimon/rest/rest_catalog_test.cpp @@ -1213,8 +1213,10 @@ TEST_F(RestCatalogTest, CreateRefusesAFormatTableThisClientCouldNotOpen) { // And a schema this library can open is created and opens. ASSERT_OK(create("fine", good, {"dt"}, format_table)); - ASSERT_OK_AND_ASSIGN(std::shared_ptr table, - catalog->GetFormatTable(Identifier("db1", "fine"))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
loaded, + catalog->GetTable(Identifier("db1", "fine"))); + std::shared_ptr table = std::dynamic_pointer_cast(loaded); + ASSERT_NE(nullptr, table); ASSERT_EQ(FormatTable::Format::PARQUET, table->GetFormat()); } @@ -1236,7 +1238,7 @@ TEST_F(RestCatalogTest, TableResponseWithoutAPathIsRejected) { // would be checked against an empty one has no boundary at all, so the response is refused // rather than turned into a table. What the response said is not repeated back: a body may // carry credentials, so a failure to read one names the request and nothing else. - ASSERT_NOK_WITH_MSG(catalog->GetFormatTable(Identifier("db1", "no_path")), + ASSERT_NOK_WITH_MSG(catalog->GetTable(Identifier("db1", "no_path")), "failed to deserialize the response"); ASSERT_NOK_WITH_MSG(catalog->GetTableLocation(Identifier("db1", "no_path")), "failed to deserialize the response"); @@ -1259,8 +1261,10 @@ TEST_F(RestCatalogTest, FormatTableIsLoadedInOneRequest) { state_->get_table_requests = 0; } - ASSERT_OK_AND_ASSIGN(std::shared_ptr table, - catalog->GetFormatTable(Identifier("db1", "fmt"))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
loaded, + catalog->GetTable(Identifier("db1", "fmt"))); + std::shared_ptr table = std::dynamic_pointer_cast(loaded); + ASSERT_NE(nullptr, table); ASSERT_EQ("wh1/db1.db/fmt", table->Location()); ASSERT_EQ(FormatTable::Format::PARQUET, table->GetFormat()); ASSERT_EQ((std::vector{"dt"}), table->PartitionKeys()); @@ -1275,18 +1279,11 @@ TEST_F(RestCatalogTest, FormatTableIsLoadedInOneRequest) { // directory named "schema" there is a partition value, not metadata to skip. ASSERT_FALSE(table->LocationCarriesPaimonMetadata()); - ASSERT_NOK_WITH_MSG(catalog->GetTable(Identifier("db1", "fmt")), "Cannot open format table"); - - // A managed table stays out of this path, and so does a system table. + // A managed table comes back through the same call, as the managed table it is. ASSERT_OK(CreateSampleTable(catalog.get(), Identifier("db1", "t1"))); - ASSERT_NOK_WITH_MSG(catalog->GetFormatTable(Identifier("db1", "t1")), "is not a format table"); - ASSERT_NOK_WITH_MSG(catalog->GetFormatTable(Identifier("db1", "t1$snapshots")), - "for system table"); - // The database is checked before the table name is parsed, so a malformed name under "sys" - // is refused as a system table rather than surfacing a parse error. - ASSERT_NOK_WITH_MSG(catalog->GetFormatTable(Identifier("sys", "all_table_options")), - "for system table"); - ASSERT_NOK_WITH_MSG(catalog->GetFormatTable(Identifier("sys", "a$b$c$d")), "for system table"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
managed, + catalog->GetTable(Identifier("db1", "t1"))); + ASSERT_EQ(nullptr, std::dynamic_pointer_cast(managed)); } TEST_F(RestCatalogTest, FormatTableWithAnUnusableSchemaIsRejectedOnLoad) { @@ -1312,24 +1309,23 @@ TEST_F(RestCatalogTest, FormatTableWithAnUnusableSchemaIsRejectedOnLoad) { // A format table format with no reader here. seed("csv_table", R"({"type": "format-table", "file.format": "csv"})", R"(["dt"])", default_fields); - ASSERT_NOK_WITH_MSG(catalog->GetFormatTable(Identifier("db1", "csv_table")), + ASSERT_NOK_WITH_MSG(catalog->GetTable(Identifier("db1", "csv_table")), "not supported by paimon-cpp yet"); // A row count no file could ever reach. seed("bad_rows", R"({"type": "format-table", "file.format": "parquet", "target-file-row-num": "0"})", R"(["dt"])", default_fields); - ASSERT_NOK_WITH_MSG(catalog->GetFormatTable(Identifier("db1", "bad_rows")), - "should be at least 1"); + ASSERT_NOK_WITH_MSG(catalog->GetTable(Identifier("db1", "bad_rows")), "should be at least 1"); // The failure names the table, which is all a caller holding several has to go on. - ASSERT_NOK_WITH_MSG(catalog->GetFormatTable(Identifier("db1", "bad_rows")), "db1.bad_rows"); + ASSERT_NOK_WITH_MSG(catalog->GetTable(Identifier("db1", "bad_rows")), "db1.bad_rows"); // The structural rules run here too, not only the format-table ones: a partition key that is // not a field of the schema is a table nothing could ever read. seed("bad_partition_key", R"({"type": "format-table", "file.format": "parquet"})", R"(["nosuchfield"])", default_fields); - ASSERT_NOK_WITH_MSG(catalog->GetFormatTable(Identifier("db1", "bad_partition_key")), + ASSERT_NOK_WITH_MSG(catalog->GetTable(Identifier("db1", "bad_partition_key")), "should include all partition fields"); // Two schemas this library cannot open, refused when the table is opened rather than when a @@ -1339,12 +1335,12 @@ TEST_F(RestCatalogTest, FormatTableWithAnUnusableSchemaIsRejectedOnLoad) { R"([{"id": 0, "name": "f0", "type": "INT NOT NULL"},)" // Not a raw string: `TIMESTAMP(6)"` holds the sequence that would end one. " {\"id\": 1, \"name\": \"dt\", \"type\": \"TIMESTAMP(6)\"}]"); - ASSERT_NOK_WITH_MSG(catalog->GetFormatTable(Identifier("db1", "timestamp_partition")), + ASSERT_NOK_WITH_MSG(catalog->GetTable(Identifier("db1", "timestamp_partition")), "cannot be TIMESTAMP/DECIMAL/BLOB"); seed("only_partitions", R"({"type": "format-table", "file.format": "parquet"})", R"(["dt"])", R"([{"id": 0, "name": "dt", "type": "STRING"}])"); - ASSERT_NOK_WITH_MSG(catalog->GetFormatTable(Identifier("db1", "only_partitions")), + ASSERT_NOK_WITH_MSG(catalog->GetTable(Identifier("db1", "only_partitions")), "every one of its columns"); // An option this library does not honour keeps its own status code, so a caller can still @@ -1353,8 +1349,7 @@ TEST_F(RestCatalogTest, FormatTableWithAnUnusableSchemaIsRejectedOnLoad) { R"({"type": "format-table", "file.format": "parquet",)" R"( "metastore.partitioned-table": "true"})", R"(["dt"])", default_fields); - Status not_implemented = - catalog->GetFormatTable(Identifier("db1", "catalog_partitions")).status(); + Status not_implemented = catalog->GetTable(Identifier("db1", "catalog_partitions")).status(); ASSERT_TRUE(not_implemented.IsNotImplemented()) << not_implemented.ToString(); } diff --git a/src/paimon/testing/mock/mock_catalog.h b/src/paimon/testing/mock/mock_catalog.h index 1a2f0b7d7..88a00e67d 100644 --- a/src/paimon/testing/mock/mock_catalog.h +++ b/src/paimon/testing/mock/mock_catalog.h @@ -35,6 +35,7 @@ #include "paimon/core/catalog/version_managed_catalog.h" #include "paimon/core/partition/partition_statistics.h" #include "paimon/core/snapshot.h" +#include "paimon/core/table/file_store_table.h" #include "paimon/result.h" #include "paimon/schema/schema.h" #include "paimon/status.h" @@ -115,7 +116,7 @@ class MockVersionManagedCatalog : public Catalog, public VersionManagedCatalog { // Serves the per-table file system when one was set, matching `GetTableFileSystem` // without recording the request, so a test can still tell the two apart. No location: // this mock's `GetTableLocation` is the unsupported stub. - return std::make_shared
( + return std::make_shared( table_file_system_ != nullptr ? table_file_system_ : file_system_, /*root_path=*/"", table_schema_, identifier, table_uuid_); } From 6d0b70ec82766676fcdf9084eb6e9117cbe0e132 Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Tue, 20 Oct 2026 16:18:14 +0800 Subject: [PATCH 02/30] fix --- include/paimon/catalog/catalog.h | 6 +- include/paimon/catalog/table.h | 14 ++-- include/paimon/table/format/format_table.h | 8 ++ .../core/catalog/file_system_catalog.cpp | 67 ++++++++------- src/paimon/core/catalog/file_system_catalog.h | 5 ++ .../core/catalog/file_system_catalog_test.cpp | 83 ++++++++++++++++++- src/paimon/core/table/file_store_table.h | 8 ++ .../table/system/audit_log_system_table.cpp | 9 +- .../table/system/audit_log_system_table.h | 6 +- .../core/table/system/binlog_system_table.cpp | 3 +- .../core/table/system/binlog_system_table.h | 3 +- .../table/system/global_system_tables.cpp | 16 ++-- .../table/system/in_memory_system_table.cpp | 11 ++- .../table/system/in_memory_system_table.h | 9 +- .../table/system/metadata_system_tables.cpp | 19 +++-- .../table/system/metadata_system_tables.h | 3 +- .../system/read_optimized_system_table.cpp | 12 ++- .../system/read_optimized_system_table.h | 10 +-- src/paimon/core/table/system/system_table.cpp | 41 +++++++-- src/paimon/core/table/system/system_table.h | 46 +++++++--- .../core/table/system/system_table_test.cpp | 2 +- src/paimon/rest/rest_catalog.cpp | 65 ++++++++------- src/paimon/rest/rest_catalog.h | 7 ++ src/paimon/rest/rest_catalog_test.cpp | 9 +- test/inte/read_inte_test.cpp | 8 +- 25 files changed, 330 insertions(+), 140 deletions(-) diff --git a/include/paimon/catalog/catalog.h b/include/paimon/catalog/catalog.h index d501bc7e3..0bb578383 100644 --- a/include/paimon/catalog/catalog.h +++ b/include/paimon/catalog/catalog.h @@ -149,8 +149,10 @@ class PAIMON_EXPORT Catalog { /// /// The table's `type` option decides what is handed back: a `format-table` comes back as /// the `FormatTable` it is - a directory of data files, with no snapshot and no manifest - - /// and every other implemented type as the managed table it is. Both go through this one - /// `Table` interface: `NewScan()`, `NewRead()`, `NewWrite()` and `NewCommit()` serve either. + /// and every other implemented type as the managed table it is. A `T$system` identifier + /// names a read-only view instead, which comes back as the `SystemTable` it is. All go + /// through this one `Table` interface: `NewScan()` and `NewRead()` serve any of them, + /// `NewWrite()` and `NewCommit()` the writable ones. /// /// @param identifier Identifier of the table to get. /// @return A result containing the table, or an error status. diff --git a/include/paimon/catalog/table.h b/include/paimon/catalog/table.h index d18bd07b8..17b3b3d21 100644 --- a/include/paimon/catalog/table.h +++ b/include/paimon/catalog/table.h @@ -66,15 +66,13 @@ class PAIMON_EXPORT Table { return root_path_; } - /// A name to identify this table. - std::string Name() const { - return identifier_.GetObjectName(); - } + /// A name to identify this table, which each kind of table answers for itself: a managed or + /// format table is named by its identifier, a system table by its own kind. + virtual std::string Name() const = 0; - /// Full name of the table, default is database.tableName. - std::string FullName() const { - return identifier_.GetFullName(); - } + /// Full name of the table, which each kind of table answers for itself. A managed or format + /// table answers database.tableName, a system table the same name as `Name()`. + virtual std::string FullName() const = 0; /// Returns the branch the table was opened for, which its identifier carries; `main` /// when it names none or a spelling of it, which its managers are bound to. A diff --git a/include/paimon/table/format/format_table.h b/include/paimon/table/format/format_table.h index d2db6aa4a..d15a8f6ce 100644 --- a/include/paimon/table/format/format_table.h +++ b/include/paimon/table/format/format_table.h @@ -123,6 +123,14 @@ class PAIMON_EXPORT FormatTable : public Table, public std::enable_shared_from_t ~FormatTable(); + std::string Name() const override { + return identifier_.GetObjectName(); + } + + std::string FullName() const override { + return identifier_.GetFullName(); + } + /// Directory the data files live in. const std::string& Location() const { return GetRootPath(); diff --git a/src/paimon/core/catalog/file_system_catalog.cpp b/src/paimon/core/catalog/file_system_catalog.cpp index 14293021e..46131f729 100644 --- a/src/paimon/core/catalog/file_system_catalog.cpp +++ b/src/paimon/core/catalog/file_system_catalog.cpp @@ -294,40 +294,16 @@ Result> FileSystemCatalog::LoadTableSchema( context.catalog_options = catalog_options_; PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, GlobalSystemTableLoader::Load(identifier.GetObjectName(), context)); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr arrow_schema, - system_table->ArrowSchema()); - return std::make_shared(std::move(arrow_schema)); + return system_table->LatestSchema(); } // The system table suffix is dropped when the data table identifier is rebuilt below, so the // identifier is validated as a whole here. PAIMON_RETURN_NOT_OK(CatalogUtils::CheckValidTableName(identifier)); PAIMON_ASSIGN_OR_RAISE(bool is_system_table, identifier.IsSystemTable()); if (is_system_table) { - PAIMON_ASSIGN_OR_RAISE(std::optional system_table_name, - identifier.GetSystemTableName()); - if (!system_table_name || !SystemTableLoader::IsSupported(system_table_name.value())) { - return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); - } - PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); - PAIMON_ASSIGN_OR_RAISE( - Identifier data_identifier, - CatalogUtils::BranchIdentifier(identifier, branch.value_or(std::string()))); - PAIMON_ASSIGN_OR_RAISE(std::optional> latest_schema, - TableSchemaExists(data_identifier)); - if (!latest_schema) { - return Status::NotExist(fmt::format("{} not exist", data_identifier.ToString())); - } - std::map dynamic_options; - if (branch) { - dynamic_options[Options::BRANCH] = branch.value(); - } - PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(data_identifier)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, - SystemTableLoader::Load(system_table_name.value(), fs_, table_path, - latest_schema.value(), dynamic_options)); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr arrow_schema, - system_table->ArrowSchema()); - return std::make_shared(std::move(arrow_schema)); + LoadSystemTable(identifier)); + return system_table->LatestSchema(); } PAIMON_ASSIGN_OR_RAISE(std::optional> latest_schema, TableSchemaExists(identifier)); @@ -340,13 +316,9 @@ Result> FileSystemCatalog::LoadTableSchema( Result> FileSystemCatalog::GetTable(const Identifier& identifier) const { PAIMON_ASSIGN_OR_RAISE(bool is_system_table, identifier.IsSystemTable()); if (is_system_table) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, LoadTableSchema(identifier)); - // A system table reads the files of the data table it belongs to, so its root path - // is that table's. - PAIMON_ASSIGN_OR_RAISE(std::string data_table_name, identifier.GetDataTableName()); - Identifier data_identifier(identifier.GetDatabaseName(), data_table_name); - PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(data_identifier)); - return std::make_shared(fs_, table_path, schema, identifier); + // A system table reads the files of the data table it belongs to, through the file + // system of that table's catalog, and the schema it exposes is its own read-only one. + return LoadSystemTable(identifier); } // The schema is read on the table's own branch: `root/branch/branch-` below the // table path, which `main` resolves to itself. `GetBranchNameOrDefault` folds any @@ -379,6 +351,33 @@ Result> FileSystemCatalog::GetTable(const Identifier& ide fs_, table_path, checked_pointer_cast(*latest_schema), identifier); } +Result> FileSystemCatalog::LoadSystemTable( + const Identifier& identifier) const { + PAIMON_ASSIGN_OR_RAISE(std::optional system_table_name, + identifier.GetSystemTableName()); + if (!system_table_name || !SystemTableLoader::IsSupported(system_table_name.value())) { + return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); + } + PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); + PAIMON_ASSIGN_OR_RAISE( + Identifier data_identifier, + CatalogUtils::BranchIdentifier(identifier, branch.value_or(std::string()))); + PAIMON_ASSIGN_OR_RAISE(std::optional> latest_schema, + TableSchemaExists(data_identifier)); + if (!latest_schema) { + return Status::NotExist(fmt::format("{} not exist", data_identifier.ToString())); + } + std::map dynamic_options; + if (branch) { + dynamic_options[Options::BRANCH] = branch.value(); + } + // A system table reads the files of the data table it belongs to, so its root path is + // that table's. + PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(data_identifier)); + return SystemTableLoader::Load(system_table_name.value(), fs_, table_path, + latest_schema.value(), dynamic_options); +} + Status FileSystemCatalog::DropDatabase(const std::string& name, bool ignore_if_not_exists, bool cascade) { PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotSystemDatabase(name, "dropDatabase")); diff --git a/src/paimon/core/catalog/file_system_catalog.h b/src/paimon/core/catalog/file_system_catalog.h index fd4360755..203e1b92e 100644 --- a/src/paimon/core/catalog/file_system_catalog.h +++ b/src/paimon/core/catalog/file_system_catalog.h @@ -37,6 +37,7 @@ class TableSchema; class FileSystem; class Identifier; class Logger; +class SystemTable; class FileSystemCatalog : public Catalog { public: @@ -85,6 +86,10 @@ class FileSystemCatalog : public Catalog { Result>> TableSchemaExists( const Identifier& identifier) const; + /// Loads the system table a `T$system` identifier names, on the branch the identifier + /// carries. The base table it belongs to must exist and carry a schema. + Result> LoadSystemTable(const Identifier& identifier) const; + Status CreateDatabaseImpl(const std::string& db_name, const std::map& options); Result TableExistsInFileSystem(const std::string& table_path) const; diff --git a/src/paimon/core/catalog/file_system_catalog_test.cpp b/src/paimon/core/catalog/file_system_catalog_test.cpp index cfe361bae..f31c0391d 100644 --- a/src/paimon/core/catalog/file_system_catalog_test.cpp +++ b/src/paimon/core/catalog/file_system_catalog_test.cpp @@ -34,14 +34,21 @@ #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/table_schema.h" #include "paimon/core/table/system/global_system_tables.h" +#include "paimon/core/table/system/system_table.h" #include "paimon/core/table/system/system_table_schema.h" #include "paimon/defs.h" #include "paimon/file_store_commit.h" +#include "paimon/file_store_write.h" #include "paimon/fs/file_system.h" #include "paimon/fs/file_system_factory.h" +#include "paimon/read_context.h" +#include "paimon/scan_context.h" #include "paimon/snapshot/snapshot_info.h" #include "paimon/table/format/format_table.h" +#include "paimon/table/source/table_read.h" +#include "paimon/table/source/table_scan.h" #include "paimon/testing/utils/testharness.h" +#include "paimon/write_context.h" namespace paimon::test { @@ -223,10 +230,15 @@ TEST(FileSystemCatalogTest, TestOptionsSystemTableCatalog) { ASSERT_FALSE(loaded_schema->field(1)->nullable()); ASSERT_OK_AND_ASSIGN(auto system_table, catalog.GetTable(options_identifier)); - ASSERT_EQ(system_table->Name(), "tbl1$options"); - // A system table reads the files of the data table it belongs to, so the table opened - // for it carries that table's root path. + // A system table comes back as the `SystemTable` it is: it answers for itself with its own + // read-only schema and the root path of the data table it reads. + ASSERT_TRUE(std::dynamic_pointer_cast(system_table) != nullptr); + ASSERT_TRUE(std::dynamic_pointer_cast(system_table) == nullptr); + ASSERT_EQ("options", system_table->Name()); + ASSERT_EQ("options", system_table->FullName()); ASSERT_EQ(system_table->GetRootPath(), table_path); + ASSERT_EQ((std::vector{"key", "value"}), + system_table->LatestSchema()->FieldNames()); ASSERT_NOK_WITH_MSG(catalog.LoadTableSchema(Identifier("db1", "tbl1$unknown")), "not exist"); ASSERT_NOK_WITH_MSG(catalog.LoadTableSchema(Identifier("db1", "missing$options")), "not exist"); @@ -242,6 +254,71 @@ TEST(FileSystemCatalogTest, TestOptionsSystemTableCatalog) { "Cannot 'renameTable' for system table"); } +// A `T$system` identifier opens the read-only view it names, which serves scans and reads of +// the data table it belongs to through the same `Table` interface, and refuses the operations +// that would modify the data table. +TEST(FileSystemCatalogTest, TestGetTableServesASystemTable) { + std::map options; + options[Options::FILE_SYSTEM] = "local"; + options[Options::FILE_FORMAT] = "orc"; + ASSERT_OK_AND_ASSIGN(auto core_options, CoreOptions::FromMap(options)); + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str(), options); + ASSERT_OK(catalog.CreateDatabase("db1", options, /*ignore_if_exists=*/true)); + + auto typed_schema = arrow::schema({arrow::field("f0", arrow::int32(), /*nullable=*/false), + arrow::field("dt", arrow::utf8())}); + ::ArrowSchema schema; + ASSERT_TRUE(arrow::ExportSchema(*typed_schema, &schema).ok()); + ASSERT_OK(catalog.CreateTable(Identifier("db1", "tbl1"), &schema, /*partition_keys=*/{"dt"}, + /*primary_keys=*/{}, options, /*ignore_if_exists=*/false)); + ArrowSchemaRelease(&schema); + + // A managed table with no `type` option still opens, as the managed table it is. + ASSERT_OK_AND_ASSIGN(auto managed, catalog.GetTable(Identifier("db1", "tbl1"))); + ASSERT_TRUE(std::dynamic_pointer_cast(managed) == nullptr); + + ASSERT_OK_AND_ASSIGN(auto system_table, catalog.GetTable(Identifier("db1", "tbl1$options"))); + ASSERT_TRUE(std::dynamic_pointer_cast(system_table) != nullptr); + ASSERT_EQ("options", system_table->Name()); + + // The scan and the read work from a context of the caller's, like every other table. + ScanContextBuilder scan_builder(system_table->GetRootPath()); + scan_builder.SetOptions(options); + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan, system_table->NewScan(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto plan, scan->CreatePlan()); + + ReadContextBuilder read_builder(system_table->GetRootPath()); + read_builder.SetOptions(options); + ASSERT_OK_AND_ASSIGN(auto read_context, read_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto read, system_table->NewRead(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto reader, read->CreateReader(plan->Splits())); + ASSERT_OK_AND_ASSIGN(auto table_result, reader->NextBatch()); + + // The options the table was created with come back, one row per key. + auto& [c_array, c_schema] = table_result; + std::shared_ptr batch; + auto maybe_batch = arrow::ImportRecordBatch(c_array.get(), c_schema.get()); + ASSERT_TRUE(maybe_batch.ok()) << maybe_batch.status().ToString(); + batch = std::move(maybe_batch).ValueUnsafe(); + ASSERT_GE(batch->num_rows(), 2); + + // The operations that would modify the data table underneath are refused by name. + WriteContextBuilder write_builder(system_table->GetRootPath(), "commit_user"); + write_builder.SetOptions(options); + ASSERT_OK_AND_ASSIGN(auto write_context, write_builder.Finish()); + ASSERT_NOK_WITH_MSG(system_table->NewWrite(std::move(write_context)), + "options system table is read-only: writing is not supported"); + + CommitContextBuilder commit_builder(system_table->GetRootPath(), "commit_user"); + commit_builder.SetOptions(options); + ASSERT_OK_AND_ASSIGN(auto commit_context, commit_builder.Finish()); + ASSERT_NOK_WITH_MSG(system_table->NewCommit(std::move(commit_context)), + "options system table is read-only: committing for it is not supported"); +} + TEST(FileSystemCatalogTest, TestBranchTableLoadsBranchSchema) { std::map options; options[Options::FILE_SYSTEM] = "local"; diff --git a/src/paimon/core/table/file_store_table.h b/src/paimon/core/table/file_store_table.h index 409fefca9..f02b4e528 100644 --- a/src/paimon/core/table/file_store_table.h +++ b/src/paimon/core/table/file_store_table.h @@ -51,6 +51,14 @@ class FileStoreTable : public Table { const std::string& uuid = "") : Table(file_system, root_path, schema, identifier, uuid) {} + std::string Name() const override { + return identifier_.GetObjectName(); + } + + std::string FullName() const override { + return identifier_.GetFullName(); + } + Result> NewScan(std::unique_ptr context) const override; Result> NewRead(std::unique_ptr context) const override; diff --git a/src/paimon/core/table/system/audit_log_system_table.cpp b/src/paimon/core/table/system/audit_log_system_table.cpp index 8de9742e1..3d3f7b3e5 100644 --- a/src/paimon/core/table/system/audit_log_system_table.cpp +++ b/src/paimon/core/table/system/audit_log_system_table.cpp @@ -358,7 +358,8 @@ std::unique_ptr CreateChangelogBatchReader( AuditLogSystemTable::AuditLogSystemTable(std::shared_ptr fs, std::string table_path, std::shared_ptr table_schema, std::map options) - : fs_(std::move(fs)), + : SystemTable(fs, table_path), + fs_(std::move(fs)), table_path_(std::move(table_path)), table_schema_(std::move(table_schema)), options_(std::move(options)) {} @@ -383,7 +384,8 @@ Result> AuditLogSystemTable::ArrowSchema() const } Result> AuditLogSystemTable::NewScan( - const std::shared_ptr& context) const { + std::unique_ptr ctx) const { + const std::shared_ptr context = std::move(ctx); if (context->GetScanFilters() && context->GetScanFilters()->GetPredicate()) { return Status::NotImplemented("audit_log system table predicate pushdown is not supported"); } @@ -411,7 +413,8 @@ Result> AuditLogSystemTable::NewScan( } Result> AuditLogSystemTable::NewRead( - const std::shared_ptr& context) const { + std::unique_ptr ctx) const { + const std::shared_ptr context = std::move(ctx); return NewChangelogRead(context, std::make_shared()); } diff --git a/src/paimon/core/table/system/audit_log_system_table.h b/src/paimon/core/table/system/audit_log_system_table.h index 780f5fa72..7395c9797 100644 --- a/src/paimon/core/table/system/audit_log_system_table.h +++ b/src/paimon/core/table/system/audit_log_system_table.h @@ -60,10 +60,8 @@ class AuditLogSystemTable : public SystemTable { std::string Name() const override; Result> ArrowSchema() const override; - Result> NewScan( - const std::shared_ptr& context) const override; - Result> NewRead( - const std::shared_ptr& context) const override; + Result> NewScan(std::unique_ptr context) const override; + Result> NewRead(std::unique_ptr context) const override; protected: Result> NewChangelogRead( diff --git a/src/paimon/core/table/system/binlog_system_table.cpp b/src/paimon/core/table/system/binlog_system_table.cpp index 88541194d..8a821d45f 100644 --- a/src/paimon/core/table/system/binlog_system_table.cpp +++ b/src/paimon/core/table/system/binlog_system_table.cpp @@ -98,7 +98,8 @@ Result> BinlogSystemTable::ArrowSchema() const { } Result> BinlogSystemTable::NewRead( - const std::shared_ptr& context) const { + std::unique_ptr ctx) const { + const std::shared_ptr context = std::move(ctx); return NewChangelogRead(context, CreateBinlogBatchConverter()); } diff --git a/src/paimon/core/table/system/binlog_system_table.h b/src/paimon/core/table/system/binlog_system_table.h index 3b4f45bd0..3a939fcef 100644 --- a/src/paimon/core/table/system/binlog_system_table.h +++ b/src/paimon/core/table/system/binlog_system_table.h @@ -42,8 +42,7 @@ class BinlogSystemTable : public AuditLogSystemTable { std::string Name() const override; Result> ArrowSchema() const override; - Result> NewRead( - const std::shared_ptr& context) const override; + Result> NewRead(std::unique_ptr context) const override; }; } // namespace paimon diff --git a/src/paimon/core/table/system/global_system_tables.cpp b/src/paimon/core/table/system/global_system_tables.cpp index 647da76c3..a66362b39 100644 --- a/src/paimon/core/table/system/global_system_tables.cpp +++ b/src/paimon/core/table/system/global_system_tables.cpp @@ -300,7 +300,13 @@ Result> GlobalSystemTableLoader::Load( return Status::NotImplemented("global system table requires catalog context: ", table_name); } - return entry.factory(context); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, + entry.factory(context)); + // The table is handed out through the `Table` interface, which serves the schema + // the base holds, so its own read-only schema is adopted before it leaves the + // loader. + PAIMON_RETURN_NOT_OK(system_table->AdoptOwnSchema()); + return system_table; } } return Status::NotImplemented("unsupported global system table: ", table_name); @@ -324,7 +330,7 @@ Result> GlobalSystemTableLoader::GetSupportedTableNames // ============================================================================= CatalogOptionsSystemTable::CatalogOptionsSystemTable(GlobalSystemTableContext context) - : InMemorySystemTable("sys/catalog_options"), context_(std::move(context)) {} + : InMemorySystemTable(context.fs, "sys/catalog_options"), context_(std::move(context)) {} std::string CatalogOptionsSystemTable::Name() const { return kName; @@ -360,7 +366,7 @@ Result> CatalogOptionsSystemTable::BuildRows() const { // ============================================================================= AllTableOptionsSystemTable::AllTableOptionsSystemTable(GlobalSystemTableContext context) - : InMemorySystemTable("sys/all_table_options"), context_(std::move(context)) {} + : InMemorySystemTable(context.fs, "sys/all_table_options"), context_(std::move(context)) {} std::string AllTableOptionsSystemTable::Name() const { return kName; @@ -399,7 +405,7 @@ Result> AllTableOptionsSystemTable::BuildRows() const { // ============================================================================= TablesSystemTable::TablesSystemTable(GlobalSystemTableContext context) - : InMemorySystemTable("sys/tables"), context_(std::move(context)) {} + : InMemorySystemTable(context.fs, "sys/tables"), context_(std::move(context)) {} std::string TablesSystemTable::Name() const { return kName; @@ -471,7 +477,7 @@ Result> TablesSystemTable::BuildRows() const { // ============================================================================= PartitionsSystemTable::PartitionsSystemTable(GlobalSystemTableContext context) - : InMemorySystemTable("sys/partitions"), context_(std::move(context)) {} + : InMemorySystemTable(context.fs, "sys/partitions"), context_(std::move(context)) {} std::string PartitionsSystemTable::Name() const { return kName; diff --git a/src/paimon/core/table/system/in_memory_system_table.cpp b/src/paimon/core/table/system/in_memory_system_table.cpp index 8d6111d79..773f621b8 100644 --- a/src/paimon/core/table/system/in_memory_system_table.cpp +++ b/src/paimon/core/table/system/in_memory_system_table.cpp @@ -31,6 +31,7 @@ #include "paimon/core/table/system/system_table_scan.h" #include "paimon/memory/memory_pool.h" #include "paimon/read_context.h" +#include "paimon/scan_context.h" #include "paimon/status.h" #include "paimon/table/source/table_read.h" @@ -104,16 +105,18 @@ class InMemorySystemTableRead : public TableRead { } // namespace -InMemorySystemTable::InMemorySystemTable(std::string table_path) - : table_path_(std::move(table_path)) {} +InMemorySystemTable::InMemorySystemTable(const std::shared_ptr& fs, + std::string table_path) + : SystemTable(fs, table_path), table_path_(std::move(table_path)) {} Result> InMemorySystemTable::NewScan( - const std::shared_ptr& /*context*/) const { + std::unique_ptr /*context*/) const { return std::make_unique(table_path_); } Result> InMemorySystemTable::NewRead( - const std::shared_ptr& context) const { + std::unique_ptr ctx) const { + const std::shared_ptr context = std::move(ctx); return std::make_unique( checked_pointer_cast(shared_from_this()), context->GetMemoryPool()); diff --git a/src/paimon/core/table/system/in_memory_system_table.h b/src/paimon/core/table/system/in_memory_system_table.h index f9fcebdb3..90cd46d84 100644 --- a/src/paimon/core/table/system/in_memory_system_table.h +++ b/src/paimon/core/table/system/in_memory_system_table.h @@ -28,18 +28,19 @@ namespace paimon { +class FileSystem; + /// Base class for system tables whose result can be materialized as a single in-memory /// RecordBatch. /// /// It provides the common singleton split scan and one-shot batch reader. class InMemorySystemTable : public SystemTable { public: - explicit InMemorySystemTable(std::string table_path); + InMemorySystemTable(const std::shared_ptr& fs, std::string table_path); Result> NewScan( - const std::shared_ptr& context) const override; - Result> NewRead( - const std::shared_ptr& context) const override; + std::unique_ptr /*context*/) const override; + Result> NewRead(std::unique_ptr context) const override; virtual Result> BuildRows() const = 0; protected: diff --git a/src/paimon/core/table/system/metadata_system_tables.cpp b/src/paimon/core/table/system/metadata_system_tables.cpp index 5279e6871..e9e00ea42 100644 --- a/src/paimon/core/table/system/metadata_system_tables.cpp +++ b/src/paimon/core/table/system/metadata_system_tables.cpp @@ -504,9 +504,10 @@ Result> WriteColsValue( } // namespace -OptionsSystemTable::OptionsSystemTable(std::string table_path, +OptionsSystemTable::OptionsSystemTable(const std::shared_ptr& fs, + std::string table_path, std::shared_ptr table_schema) - : InMemorySystemTable(std::move(table_path)), table_schema_(std::move(table_schema)) {} + : InMemorySystemTable(fs, table_path), table_schema_(std::move(table_schema)) {} std::string OptionsSystemTable::Name() const { return kName; @@ -532,7 +533,7 @@ Result> OptionsSystemTable::BuildRows() const { SnapshotsSystemTable::SnapshotsSystemTable(std::shared_ptr fs, std::string table_path, std::string branch) - : InMemorySystemTable(table_path), + : InMemorySystemTable(fs, table_path), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch))) {} std::string SnapshotsSystemTable::Name() const { @@ -594,7 +595,7 @@ Result> SnapshotsSystemTable::BuildRows() const { SchemasSystemTable::SchemasSystemTable(std::shared_ptr fs, std::string table_path, std::string branch) - : InMemorySystemTable(table_path), + : InMemorySystemTable(fs, table_path), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch))) {} std::string SchemasSystemTable::Name() const { @@ -650,7 +651,7 @@ Result> SchemasSystemTable::BuildRows() const { TagsSystemTable::TagsSystemTable(std::shared_ptr fs, std::string table_path, std::string branch) - : InMemorySystemTable(table_path), + : InMemorySystemTable(fs, table_path), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch))) {} std::string TagsSystemTable::Name() const { @@ -700,7 +701,7 @@ Result> TagsSystemTable::BuildRows() const { BranchesSystemTable::BranchesSystemTable(std::shared_ptr fs, std::string table_path, std::string branch) - : InMemorySystemTable(table_path), + : InMemorySystemTable(fs, table_path), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch))) {} std::string BranchesSystemTable::Name() const { @@ -739,7 +740,7 @@ Result> BranchesSystemTable::BuildRows() const { ConsumersSystemTable::ConsumersSystemTable(std::shared_ptr fs, std::string table_path, std::string branch) - : InMemorySystemTable(table_path), + : InMemorySystemTable(fs, table_path), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch))) {} std::string ConsumersSystemTable::Name() const { @@ -774,7 +775,7 @@ ManifestsSystemTable::ManifestsSystemTable(std::shared_ptr fs, std:: std::string branch, std::shared_ptr table_schema, std::map options) - : InMemorySystemTable(table_path), + : InMemorySystemTable(fs, table_path), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch), std::move(table_schema), std::move(options))) {} @@ -843,7 +844,7 @@ Result> ManifestsSystemTable::BuildRows() const { FilesSystemTable::FilesSystemTable(std::shared_ptr fs, std::string table_path, std::string branch, std::shared_ptr table_schema, std::map options) - : InMemorySystemTable(table_path), + : InMemorySystemTable(fs, table_path), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch), std::move(table_schema), std::move(options))) {} diff --git a/src/paimon/core/table/system/metadata_system_tables.h b/src/paimon/core/table/system/metadata_system_tables.h index e3961e476..ae2e5155b 100644 --- a/src/paimon/core/table/system/metadata_system_tables.h +++ b/src/paimon/core/table/system/metadata_system_tables.h @@ -35,7 +35,8 @@ class OptionsSystemTable : public InMemorySystemTable { public: static constexpr const char* kName = "options"; - OptionsSystemTable(std::string table_path, std::shared_ptr table_schema); + OptionsSystemTable(const std::shared_ptr& fs, std::string table_path, + std::shared_ptr table_schema); std::string Name() const override; Result> ArrowSchema() const override; diff --git a/src/paimon/core/table/system/read_optimized_system_table.cpp b/src/paimon/core/table/system/read_optimized_system_table.cpp index 8d5696447..172fc964c 100644 --- a/src/paimon/core/table/system/read_optimized_system_table.cpp +++ b/src/paimon/core/table/system/read_optimized_system_table.cpp @@ -35,10 +35,12 @@ namespace paimon { -ReadOptimizedSystemTable::ReadOptimizedSystemTable(std::string table_path, +ReadOptimizedSystemTable::ReadOptimizedSystemTable(const std::shared_ptr& fs, + std::string table_path, std::shared_ptr table_schema, std::map options) - : table_path_(std::move(table_path)), + : SystemTable(fs, table_path), + table_path_(std::move(table_path)), table_schema_(std::move(table_schema)), options_(std::move(options)) {} @@ -57,7 +59,8 @@ std::map ReadOptimizedSystemTable::ReadOptimizedOption } Result> ReadOptimizedSystemTable::NewScan( - const std::shared_ptr& context) const { + std::unique_ptr ctx) const { + const std::shared_ptr context = std::move(ctx); if (context->GetRealtimeContext() && !table_schema_->PrimaryKeys().empty()) { return Status::NotImplemented( "PK real-time union read does not support read-optimized scans"); @@ -133,7 +136,8 @@ Result> ReadOptimizedSystemTable::CreateDataReadCon } Result> ReadOptimizedSystemTable::NewRead( - const std::shared_ptr& context) const { + std::unique_ptr ctx) const { + const std::shared_ptr context = std::move(ctx); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr base_context, CreateDataReadContext(context)); return TableRead::Create(std::move(base_context)); diff --git a/src/paimon/core/table/system/read_optimized_system_table.h b/src/paimon/core/table/system/read_optimized_system_table.h index 3123fe03b..7f92e04e2 100644 --- a/src/paimon/core/table/system/read_optimized_system_table.h +++ b/src/paimon/core/table/system/read_optimized_system_table.h @@ -26,6 +26,7 @@ #include "paimon/core/table/system/system_table.h" namespace paimon { +class FileSystem; class TableSchema; /// System table for `T$ro`, exposing read-optimized data. @@ -33,15 +34,14 @@ class ReadOptimizedSystemTable : public SystemTable { public: static constexpr const char* kName = "ro"; - ReadOptimizedSystemTable(std::string table_path, std::shared_ptr table_schema, + ReadOptimizedSystemTable(const std::shared_ptr& fs, std::string table_path, + std::shared_ptr table_schema, std::map options); std::string Name() const override; Result> ArrowSchema() const override; - Result> NewScan( - const std::shared_ptr& context) const override; - Result> NewRead( - const std::shared_ptr& context) const override; + Result> NewScan(std::unique_ptr context) const override; + Result> NewRead(std::unique_ptr context) const override; private: /// Build the context for the data table underneath this read-optimized view. diff --git a/src/paimon/core/table/system/system_table.cpp b/src/paimon/core/table/system/system_table.cpp index 5a1a1de00..dd00b7bcd 100644 --- a/src/paimon/core/table/system/system_table.cpp +++ b/src/paimon/core/table/system/system_table.cpp @@ -28,6 +28,7 @@ #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" +#include "paimon/commit_context.h" #include "paimon/common/utils/path_util.h" #include "paimon/common/utils/string_utils.h" #include "paimon/core/schema/schema_manager.h" @@ -37,8 +38,12 @@ #include "paimon/core/table/system/global_system_tables.h" #include "paimon/core/table/system/metadata_system_tables.h" #include "paimon/core/table/system/read_optimized_system_table.h" +#include "paimon/core/table/system/system_table_schema.h" #include "paimon/core/utils/branch_manager.h" +#include "paimon/file_store_commit.h" +#include "paimon/file_store_write.h" #include "paimon/status.h" +#include "paimon/write_context.h" namespace paimon { namespace { @@ -70,11 +75,11 @@ std::string LoadBranch(const std::map& options) { const std::vector& SystemTableRegistry() { static const std::vector registry = { {OptionsSystemTable::kName, - [](const std::shared_ptr& /*fs*/, const std::string& table_path, + [](const std::shared_ptr& fs, const std::string& table_path, const std::shared_ptr& table_schema, const std::map& /*dynamic_options*/) -> Result> { - return std::make_shared(table_path, table_schema); + return std::make_shared(fs, table_path, table_schema); }}, {AuditLogSystemTable::kName, [](const std::shared_ptr& fs, const std::string& table_path, @@ -93,12 +98,12 @@ const std::vector& SystemTableRegistry() { fs, table_path, table_schema, MergeOptions(table_schema, dynamic_options)); }}, {ReadOptimizedSystemTable::kName, - [](const std::shared_ptr& /*fs*/, const std::string& table_path, + [](const std::shared_ptr& fs, const std::string& table_path, const std::shared_ptr& table_schema, const std::map& dynamic_options) -> Result> { return std::make_shared( - table_path, table_schema, MergeOptions(table_schema, dynamic_options)); + fs, table_path, table_schema, MergeOptions(table_schema, dynamic_options)); }}, {SnapshotsSystemTable::kName, [](const std::shared_ptr& fs, const std::string& table_path, @@ -176,6 +181,27 @@ std::optional FindSystemTableFactory(const std::string& syst } // namespace +SystemTable::SystemTable(const std::shared_ptr& fs, const std::string& table_path) + : Table(fs, table_path, /*schema=*/nullptr, + Identifier(Identifier::kUnknownDatabase, "system_table"), /*uuid=*/"") {} + +Status SystemTable::AdoptOwnSchema() { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr arrow_schema, ArrowSchema()); + schema_ = std::make_shared(std::move(arrow_schema)); + return Status::OK(); +} + +Result> SystemTable::NewWrite( + std::unique_ptr /*context*/) const { + return Status::NotImplemented(Name(), " system table is read-only: writing is not supported"); +} + +Result> SystemTable::NewCommit( + std::unique_ptr /*context*/) const { + return Status::NotImplemented(Name(), + " system table is read-only: committing for it is not supported"); +} + bool SystemTableLoader::IsSupported(const std::string& system_table_name) { return FindSystemTableFactory(system_table_name).has_value(); } @@ -186,7 +212,12 @@ Result> SystemTableLoader::Load( const std::map& dynamic_options) { std::optional factory = FindSystemTableFactory(system_table_name); if (factory) { - return factory.value()(fs, table_path, table_schema, dynamic_options); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, + factory.value()(fs, table_path, table_schema, dynamic_options)); + // The table is handed out through the `Table` interface, which serves the schema the + // base holds, so its own read-only schema is adopted before it leaves the loader. + PAIMON_RETURN_NOT_OK(system_table->AdoptOwnSchema()); + return system_table; } return Status::NotImplemented("unsupported system table: ", system_table_name); } diff --git a/src/paimon/core/table/system/system_table.h b/src/paimon/core/table/system/system_table.h index 39e1da31b..2e4dff55d 100644 --- a/src/paimon/core/table/system/system_table.h +++ b/src/paimon/core/table/system/system_table.h @@ -25,6 +25,8 @@ #include #include "arrow/api.h" +#include "paimon/catalog/identifier.h" +#include "paimon/catalog/table.h" #include "paimon/result.h" #include "paimon/status.h" #include "paimon/type_fwd.h" @@ -33,9 +35,8 @@ namespace paimon { class FileSystem; class ReadContext; class ScanContext; -class TableScan; -class TableRead; class TableSchema; +class SystemTableSchema; /// Parsed information for a table-scoped system table path. struct SystemTablePath { @@ -49,19 +50,42 @@ struct SystemTablePath { bool is_global = false; }; -/// Base interface for table-scoped system tables such as `T$options` and `T$snapshots`. +/// Base class for table-scoped system tables such as `T$options` and `T$snapshots`. /// -/// Implementations expose a read-only schema and create their own scan/read objects. -class SystemTable : public std::enable_shared_from_this { +/// A system table is the read-only view a `T$system` identifier names: it exposes a schema of +/// its own and serves scans and reads from the data table it belongs to, through the same +/// `Table` interface as every other kind of table. `Name()` - which each kind of system table +/// answers for itself - is its full name as well: the identifier it is opened by belongs to the +/// data table underneath and would name that one. +class SystemTable : public Table, public std::enable_shared_from_this { public: - virtual ~SystemTable() = default; + std::string FullName() const override { + return Name(); + } - virtual std::string Name() const = 0; + /// The read-only schema this system table exposes, which is not the data table's schema. virtual Result> ArrowSchema() const = 0; - virtual Result> NewScan( - const std::shared_ptr& context) const = 0; - virtual Result> NewRead( - const std::shared_ptr& context) const = 0; + + /// A system table is read-only: writing it or committing for it is refused rather than + /// routed to the data table underneath. + Result> NewWrite( + std::unique_ptr context) const override; + + Result> NewCommit( + std::unique_ptr context) const override; + + protected: + /// @param fs File system the data table underneath is read through. + /// @param table_path Root path of the data table this system table belongs to. + SystemTable(const std::shared_ptr& fs, const std::string& table_path); + + /// Loaders call this once the object exists, because `ArrowSchema()` is virtual and cannot + /// run from a constructor: it fills the schema the base hands out with this table's own + /// read-only `SystemTableSchema`. + Status AdoptOwnSchema(); + + friend class SystemTableLoader; + friend class GlobalSystemTableLoader; }; /// Loads system table implementations from parsed table identifiers or table paths. diff --git a/src/paimon/core/table/system/system_table_test.cpp b/src/paimon/core/table/system/system_table_test.cpp index 7591e53bf..49cf6838f 100644 --- a/src/paimon/core/table/system/system_table_test.cpp +++ b/src/paimon/core/table/system/system_table_test.cpp @@ -249,7 +249,7 @@ TEST(SystemTableTest, TestNewReadPropagatesWarmupLevel) { AuditLogSystemTable audit_log(/*fs=*/nullptr, "/tmp/table", table_schema, options); BinlogSystemTable binlog(/*fs=*/nullptr, "/tmp/table", table_schema, options); - ReadOptimizedSystemTable read_optimized("/tmp/table", table_schema, options); + ReadOptimizedSystemTable read_optimized(/*fs=*/nullptr, "/tmp/table", table_schema, options); for (WarmupLevel level : {WarmupLevel::NONE, WarmupLevel::RAW, WarmupLevel::DECODED}) { ReadContextBuilder builder("/tmp/table"); diff --git a/src/paimon/rest/rest_catalog.cpp b/src/paimon/rest/rest_catalog.cpp index ad005f58a..1005b760d 100644 --- a/src/paimon/rest/rest_catalog.cpp +++ b/src/paimon/rest/rest_catalog.cpp @@ -391,49 +391,58 @@ Result> RestCatalog::LoadTableSchema(const Identifier& i context.catalog_options = api_->GetMergedOptions(); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, GlobalSystemTableLoader::Load(identifier.GetObjectName(), context)); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr arrow_schema, - system_table->ArrowSchema()); - return std::make_shared(std::move(arrow_schema)); + return system_table->LatestSchema(); } PAIMON_ASSIGN_OR_RAISE(bool is_system_table, identifier.IsSystemTable()); PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); branch = NormalizeBranch(std::move(branch)); PAIMON_ASSIGN_OR_RAISE(Identifier load_identifier, ToLoadIdentifier(identifier)); if (is_system_table) { - PAIMON_ASSIGN_OR_RAISE(std::optional system_table_name, - identifier.GetSystemTableName()); - if (!system_table_name || !SystemTableLoader::IsSupported(system_table_name.value())) { - return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); - } - std::string data_table_path; - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr latest_schema, - LoadDataTableSchema(load_identifier, branch, &data_table_path, - /*table_id=*/nullptr)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, + LoadSystemTable(identifier, load_identifier, branch)); if (table_path != nullptr) { - *table_path = data_table_path; - } - std::map dynamic_options; - if (branch) { - dynamic_options[Options::BRANCH] = branch.value(); + *table_path = system_table->GetRootPath(); } - // The file system is only used to build the system table, whose schema does not - // read any file, so the catalog wide one is enough here. The credentials of the - // table are applied when its rows are read, through the file system of the read or - // scan context. - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr system_table, - SystemTableLoader::Load(system_table_name.value(), fs_, data_table_path, latest_schema, - dynamic_options)); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr arrow_schema, - system_table->ArrowSchema()); - return std::make_shared(std::move(arrow_schema)); + return system_table->LatestSchema(); } PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, LoadDataTableSchema(load_identifier, branch, table_path, table_id)); return checked_pointer_cast(schema); } +Result> RestCatalog::LoadSystemTable( + const Identifier& identifier, const Identifier& load_identifier, + const std::optional& branch) const { + PAIMON_ASSIGN_OR_RAISE(std::optional system_table_name, + identifier.GetSystemTableName()); + if (!system_table_name || !SystemTableLoader::IsSupported(system_table_name.value())) { + return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); + } + std::string data_table_path; + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr latest_schema, + LoadDataTableSchema(load_identifier, branch, &data_table_path, + /*table_id=*/nullptr)); + std::map dynamic_options; + if (branch) { + dynamic_options[Options::BRANCH] = branch.value(); + } + // The file system is only used to build the system table, whose schema does not + // read any file, so the catalog wide one is enough here. The credentials of the + // table are applied when its rows are read, through the file system of the read or + // scan context. + return SystemTableLoader::Load(system_table_name.value(), fs_, data_table_path, latest_schema, + dynamic_options); +} + Result> RestCatalog::GetTable(const Identifier& identifier) const { + // A `T$system` identifier opens the read-only view it names, which reads the data table it + // belongs to and answers for itself through the same `Table` interface. + PAIMON_ASSIGN_OR_RAISE(bool is_system_table, identifier.IsSystemTable()); + if (is_system_table) { + PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); + PAIMON_ASSIGN_OR_RAISE(Identifier load_identifier, ToLoadIdentifier(identifier)); + return LoadSystemTable(identifier, load_identifier, NormalizeBranch(std::move(branch))); + } // Read UUID, path and schema together to avoid mixing table generations after recreation. std::string uuid; std::string table_path; diff --git a/src/paimon/rest/rest_catalog.h b/src/paimon/rest/rest_catalog.h index 076346df8..5c62ed472 100644 --- a/src/paimon/rest/rest_catalog.h +++ b/src/paimon/rest/rest_catalog.h @@ -36,6 +36,7 @@ struct ArrowSchema; namespace paimon { class FileSystem; +class SystemTable; class TableSchema; /// A catalog backed by a REST catalog server. Metadata operations are delegated to @@ -134,6 +135,12 @@ class RestCatalog : public Catalog, public VersionManagedCatalog { const Identifier& data_identifier, const std::optional& branch, std::string* table_path, std::string* table_id) const; + /// Loads the system table a `T$system` identifier names, on the branch the identifier + /// carries, from the same response that carries the data table's schema and path. + Result> LoadSystemTable( + const Identifier& identifier, const Identifier& load_identifier, + const std::optional& branch) const; + /// Converts a `GetTableResponse` into a `TableSchema`, applying the table path, audit /// and branch options. static Result> ToTableSchema( diff --git a/src/paimon/rest/rest_catalog_test.cpp b/src/paimon/rest/rest_catalog_test.cpp index 94ccc19a1..43e579abb 100644 --- a/src/paimon/rest/rest_catalog_test.cpp +++ b/src/paimon/rest/rest_catalog_test.cpp @@ -49,6 +49,7 @@ #include "paimon/core/core_options.h" #include "paimon/core/partition/partition_statistics.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/system/system_table.h" #include "paimon/core/utils/snapshot_manager.h" #include "paimon/defs.h" #include "paimon/file_store_commit.h" @@ -1061,9 +1062,13 @@ TEST_F(RestCatalogTest, SystemTableSchema) { requests_before = state_->get_table_requests; } ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog->GetTable(system_identifier)); - ASSERT_EQ("t1$options", table->Name()); + // A system table comes back as the read-only `SystemTable` it is, answering for itself + // with the kind it serves rather than the identifier it was opened by. + std::shared_ptr system_table = std::dynamic_pointer_cast(table); + ASSERT_NE(nullptr, system_table); + ASSERT_EQ("options", system_table->Name()); ASSERT_EQ((std::vector{"key", "value"}), table->LatestSchema()->FieldNames()); - ASSERT_EQ("db1.t1$options", table->Uuid()); + ASSERT_EQ("options", table->Uuid()); { std::lock_guard lock(state_->mutex); ASSERT_EQ(requests_before + 1, state_->get_table_requests); diff --git a/test/inte/read_inte_test.cpp b/test/inte/read_inte_test.cpp index 284cd362f..2c24607a2 100644 --- a/test/inte/read_inte_test.cpp +++ b/test/inte/read_inte_test.cpp @@ -4190,18 +4190,18 @@ Result> ReadGlobalSystemTable( ScanContextBuilder scan_context_builder(sys_path); scan_context_builder.SetOptions(options); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr scan_context, + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_scan, - system_table->NewScan(scan_context)); + system_table->NewScan(std::move(scan_context))); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, table_scan->CreatePlan()); ReadContextBuilder read_context_builder(sys_path); read_context_builder.SetOptions(options); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr read_context, + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, - system_table->NewRead(read_context)); + system_table->NewRead(std::move(read_context))); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr result, From 319e9aa97893437d0857809bd27d6530e3b366a6 Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Tue, 20 Oct 2026 17:49:07 +0800 Subject: [PATCH 03/30] tmp --- include/paimon/catalog/table.h | 10 +- include/paimon/file_store_commit.h | 3 +- include/paimon/file_store_write.h | 3 +- include/paimon/table/source/table_read.h | 3 +- include/paimon/table/source/table_scan.h | 3 +- src/paimon/core/table/file_store_table.h | 10 +- src/paimon/core/table/table.cpp | 594 +++++++++++++---------- src/paimon/core/table/table_test.cpp | 23 + 8 files changed, 391 insertions(+), 258 deletions(-) diff --git a/include/paimon/catalog/table.h b/include/paimon/catalog/table.h index 17b3b3d21..69abbe6d5 100644 --- a/include/paimon/catalog/table.h +++ b/include/paimon/catalog/table.h @@ -99,29 +99,29 @@ class PAIMON_EXPORT Table { } /// Creates a scan of this table from a finished scan context, the table-first entry point - /// that `TableScan::Create()` also builds through. The scan is derived from the context. + /// that `TableScan::Create()` loads the table the context names and hands the context to. /// @param context The finished scan context. /// @return The scan, or the error building it reports. virtual Result> NewScan( std::unique_ptr context) const = 0; /// Creates a read of this table from a finished read context, the table-first entry point - /// that `TableRead::Create()` also builds through. The read is derived from the context. + /// that `TableRead::Create()` loads the table the context names and hands the context to. /// @param context The finished read context. /// @return The read, or the error building it reports. virtual Result> NewRead( std::unique_ptr context) const = 0; /// Creates a write of this table from a finished write context, the table-first entry point - /// that `FileStoreWrite::Create()` also builds through. The write is derived from the context. + /// that `FileStoreWrite::Create()` loads the table the context names and hands the context to. /// @param context The finished write context. /// @return The write, or the error building it reports. virtual Result> NewWrite( std::unique_ptr context) const = 0; /// Creates a commit of this table from a finished commit context, the table-first entry point - /// that `FileStoreCommit::Create()` also builds through. The commit is derived from the - /// context. + /// that `FileStoreCommit::Create()` loads the table the context names and hands the context + /// to. /// @param context The finished commit context. /// @return The commit, or the error building it reports. virtual Result> NewCommit( diff --git a/include/paimon/file_store_commit.h b/include/paimon/file_store_commit.h index fcb094e05..581cadf2d 100644 --- a/include/paimon/file_store_commit.h +++ b/include/paimon/file_store_commit.h @@ -50,7 +50,8 @@ class CommitMessage; /// manages versions. `UseRESTCatalogCommit()` only prepares a request for the caller to send. class PAIMON_EXPORT FileStoreCommit { public: - /// Create an instance of `FileStoreCommit`. + /// Create an instance of `FileStoreCommit`: the table the context names is loaded first, then + /// the context is handed to that table's `NewCommit`. /// /// @param context A unique pointer to the `CommitContext` used for commit operations. /// diff --git a/include/paimon/file_store_write.h b/include/paimon/file_store_write.h index 8859e4f3f..5f74d6536 100644 --- a/include/paimon/file_store_write.h +++ b/include/paimon/file_store_write.h @@ -42,7 +42,8 @@ class WriteContext; /// Interface for write operations in a file store. class PAIMON_EXPORT FileStoreWrite { public: - /// Create an instance of `FileStoreWrite`. + /// Create an instance of `FileStoreWrite`: the table the context names is loaded first, then + /// the context is handed to that table's `NewWrite`. /// /// @param context A unique pointer to the `WriteContext` used for write operations. /// diff --git a/include/paimon/table/source/table_read.h b/include/paimon/table/source/table_read.h index c77b323da..2bfd84fff 100644 --- a/include/paimon/table/source/table_read.h +++ b/include/paimon/table/source/table_read.h @@ -42,7 +42,8 @@ class PAIMON_EXPORT TableRead { public: virtual ~TableRead() = default; - /// Create an instance of `TableRead`. + /// Create an instance of `TableRead`: the table the context names is loaded first, then the + /// context is handed to that table's `NewRead`. /// /// @param context A unique pointer to the `ReadContext` used for read operations. /// @return A Result containing a unique pointer to the `TableRead` instance. diff --git a/include/paimon/table/source/table_scan.h b/include/paimon/table/source/table_scan.h index 8bae74c41..39f0453c8 100644 --- a/include/paimon/table/source/table_scan.h +++ b/include/paimon/table/source/table_scan.h @@ -36,7 +36,8 @@ class ScanContext; /// A scanner interface for reading table's meta and create a plan. class PAIMON_EXPORT TableScan { public: - /// Create an instance of `TableScan`. + /// Create an instance of `TableScan`: the table the context names is loaded first, then the + /// context is handed to that table's `NewScan`. /// /// @param context A unique pointer to the `ScanContext` used for scan operations. /// @return A Result containing a unique pointer to the `TableScan` instance. diff --git a/src/paimon/core/table/file_store_table.h b/src/paimon/core/table/file_store_table.h index f02b4e528..428513e16 100644 --- a/src/paimon/core/table/file_store_table.h +++ b/src/paimon/core/table/file_store_table.h @@ -35,10 +35,12 @@ class Schema; /// its root path describes. A format table - a directory of plain data files - is a `FormatTable`, /// the other kind of `Table`. /// -/// The operators create through the context-only entry points every table serves, so a context -/// built without a table reaches the same operators: `TableScan::Create()`, `TableRead::Create()`, -/// `FileStoreWrite::Create()` and `FileStoreCommit::Create()`, which read the table's schema to -/// dispatch on its type. +/// Its `NewScan`/`NewRead`/`NewWrite`/`NewCommit` build the operators here, from the context plus +/// the schema the table was constructed with, so they serve the table as it was loaded. The +/// context-only entry points - `TableScan::Create()`, `TableRead::Create()`, +/// `FileStoreWrite::Create()` and `FileStoreCommit::Create()` - load the table the context names +/// and hand the context to that table's `NewXxx`, which is how a context built without a table +/// reaches the same operators. class FileStoreTable : public Table { public: /// The constructor a catalog builds a managed table through, so a `shared_ptr` can hold one; diff --git a/src/paimon/core/table/table.cpp b/src/paimon/core/table/table.cpp index 46287881b..bff18db29 100644 --- a/src/paimon/core/table/table.cpp +++ b/src/paimon/core/table/table.cpp @@ -210,94 +210,166 @@ Result HasHistoricalPrimaryKeyBTreeDefinition( return false; } -} // namespace - -Table::Table(const std::shared_ptr& file_system, const std::string& root_path, - const std::shared_ptr& schema, const Identifier& identifier, - const std::string& uuid) - : file_system_(file_system), - root_path_(root_path), - schema_(schema), - identifier_(identifier), - uuid_(uuid) {} - -Result> FileStoreTable::NewScan( - std::unique_ptr context) const { - return TableScan::Create(std::move(context)); -} - -Result> FileStoreTable::NewRead( - std::unique_ptr context) const { - return TableRead::Create(std::move(context)); -} - -Result> FileStoreTable::NewWrite( - std::unique_ptr context) const { - return FileStoreWrite::Create(std::move(context)); -} - -Result> FileStoreTable::NewCommit( - std::unique_ptr context) const { - return FileStoreCommit::Create(std::move(context)); +/// The tail every `TableOf` ends with: the schema one of them read says what the path names. A +/// format table is handed back as the `FormatTable` it is, everything else as the +/// `FileStoreTable` a managed table is, carrying the schema for its `NewXxx` to build from. A +/// null schema is handed over as it is: what to say about a table that is not there is the +/// managed path's to decide, and it says it in more detail. +Result> TableOfLoadedSchema( + const std::shared_ptr& file_system, const std::string& path, + const std::string& branch, const std::map& options, + const std::shared_ptr& latest_schema, const std::optional& identifier, + const std::string& uuid = "") { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, + TryLoadFormatTable(file_system, path, branch, options, latest_schema)); + if (format_table != nullptr) { + return std::shared_ptr
(std::move(format_table)); + } + // A context names a path, never a catalog identifier, so the table is known by the directory + // it sits in - the same convention `TryLoadFormatTable` follows above. + Identifier table_identifier = + identifier.value_or(Identifier(Identifier::kUnknownDatabase, PathUtil::GetName(path))); + return std::make_shared(file_system, path, latest_schema, table_identifier, + uuid); } -Result> FileStoreWrite::Create(std::unique_ptr ctx) { - if (ctx == nullptr) { - return Status::Invalid("write context is null pointer"); +/// The table a scan context names, read from what the context carries: a format table the +/// builder was handed, a system table behind a `T$system` path, or the table - format or +/// managed - the schema under the context's path says is there. +Result> TableOf(const ScanContext& ctx) { + // A table the caller already loaded says what it is, so nothing is read to find out. + if (ctx.GetFormatTable() != nullptr) { + return std::shared_ptr
(ctx.GetFormatTable()); } - if (ctx->GetMemoryPool() == nullptr) { - return Status::Invalid("memory pool is null pointer"); + PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, + CoreOptions::FromMap(ctx.GetOptions(), ctx.GetSpecificFileSystem(), {})); + PAIMON_ASSIGN_OR_RAISE(std::optional system_table_path, + SystemTableLoader::TryParsePath(ctx.GetPath())); + if (system_table_path) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, + SystemTableLoader::LoadFromPath(tmp_options.GetFileSystem(), + ctx.GetPath(), ctx.GetOptions())); + return std::shared_ptr
(std::move(system_table)); } - if (ctx->GetExecutor() == nullptr) { - return Status::Invalid("executor is null pointer"); + // A format table is planned by listing directories, so it never reaches the manifest path + // below. One `TableScan` interface serves both, as Java Paimon serves both through one + // `ReadBuilder`. + const std::string branch = BranchManager::NormalizeBranch(tmp_options.GetBranch()); + std::shared_ptr latest_schema; + if (branch == BranchManager::DEFAULT_MAIN_BRANCH && ctx.GetSpecificTableSchema()) { + // Handing the schema over only saves reading it. A context always names a table path and + // this library writes the metadata under it, so `schema` and `branch` stay metadata. + PAIMON_ASSIGN_OR_RAISE(latest_schema, + TableSchema::CreateFromJson(ctx.GetSpecificTableSchema().value())); + } else { + SchemaManager schema_manager(tmp_options.GetFileSystem(), ctx.GetPath(), branch); + PAIMON_ASSIGN_OR_RAISE(std::optional> latest, + schema_manager.Latest()); + if (latest) { + latest_schema = *latest; + } } + return TableOfLoadedSchema(tmp_options.GetFileSystem(), ctx.GetPath(), branch, ctx.GetOptions(), + latest_schema, ctx.GetIdentifier()); +} +/// The table a read context names, read from what the context carries: a format table the +/// builder was handed, a system table behind a `T$system` path, or the table - format or +/// managed - the schema under the context's path says is there. +Result> TableOf(const ReadContext& ctx) { // A table the caller already loaded says what it is, so nothing is read to find out. - if (ctx->GetFormatTable() != nullptr) { - return ctx->GetFormatTable()->NewWrite(std::move(ctx)); + if (ctx.GetFormatTable() != nullptr) { + return std::shared_ptr
(ctx.GetFormatTable()); + } + PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_core_options, + CoreOptions::FromMap(ctx.GetOptions(), ctx.GetSpecificFileSystem(), + ctx.GetFileSystemSchemeToIdentifierMap())); + PAIMON_ASSIGN_OR_RAISE(std::optional system_table_path, + SystemTableLoader::TryParsePath(ctx.GetPath())); + if (system_table_path) { + if (PredicateUtils::ContainsFullTextMatch(context->GetPredicate())) { + return Status::NotImplemented( + "FullTextMatch predicate execution is not implemented yet"); + } + if (ctx.HasVectorOrFullTextSearch()) { + return Status::NotImplemented( + "VectorSearch or FullTextSearch is not supported for system tables"); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, + SystemTableLoader::LoadFromPath(tmp_core_options.GetFileSystem(), + ctx.GetPath(), ctx.GetOptions())); + return std::shared_ptr
(std::move(system_table)); } - PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog())); - std::shared_ptr specific_fs = ctx->GetSpecificFileSystem(); - if (specific_fs == nullptr && ctx->GetFileSystemSchemeToIdentifierMap().empty() && - ctx->GetCatalog() != nullptr) { - if (!ctx->GetIdentifier()) { + // A format table has no manifest and a file another engine wrote carries no field ids, so + // which files there are and how a row is put back together is its own. How a file is opened + // is not: both paths go through `DataFileReaderFactory`. + const std::string branch = ctx.GetBranch(); + std::shared_ptr latest_schema; + if (branch == BranchManager::DEFAULT_MAIN_BRANCH && ctx.GetSpecificTableSchema()) { + // Handing the schema over only saves reading it. A context always names a table path and + // this library writes the metadata under it, so `schema` and `branch` stay metadata. + PAIMON_ASSIGN_OR_RAISE(latest_schema, + TableSchema::CreateFromJson(ctx.GetSpecificTableSchema().value())); + } else { + SchemaManager schema_manager(tmp_core_options.GetFileSystem(), ctx.GetPath(), branch); + PAIMON_ASSIGN_OR_RAISE(std::optional> latest, + schema_manager.Latest()); + if (latest) { + latest_schema = *latest; + } + } + return TableOfLoadedSchema(tmp_core_options.GetFileSystem(), ctx.GetPath(), branch, + ctx.GetOptions(), latest_schema, ctx.GetIdentifier()); +} + +/// The table a write context names, read from what the context carries: a format table the +/// builder was handed, or the table - format or managed - the schema under the context's root +/// path says is there. +Result> TableOf(const WriteContext& ctx) { + // A table the caller already loaded says what it is, so nothing is read to find out. + if (ctx.GetFormatTable() != nullptr) { + return std::shared_ptr
(ctx.GetFormatTable()); + } + PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx.GetCatalog())); + std::shared_ptr specific_fs = ctx.GetSpecificFileSystem(); + if (specific_fs == nullptr && ctx.GetFileSystemSchemeToIdentifierMap().empty() && + ctx.GetCatalog() != nullptr) { + if (!ctx.GetIdentifier()) { return Status::Invalid("a catalog write requires a table identifier"); } // A catalog issuing per-table temporary credentials only hands them out through // GetTableFileSystem, so the write uses the credentials of this table. PAIMON_ASSIGN_OR_RAISE(specific_fs, - ctx->GetCatalog()->GetTableFileSystem(ctx->GetIdentifier().value())); + ctx.GetCatalog()->GetTableFileSystem(ctx.GetIdentifier().value())); } PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, - CoreOptions::FromMap(ctx->GetOptions(), specific_fs, - ctx->GetFileSystemSchemeToIdentifierMap())); - std::string branch = ctx->GetBranch(); + CoreOptions::FromMap(ctx.GetOptions(), specific_fs, + ctx.GetFileSystemSchemeToIdentifierMap())); + std::string branch = ctx.GetBranch(); std::optional catalog_identifier; - if (ctx->GetIdentifier()) { - PAIMON_ASSIGN_OR_RAISE( - Identifier branch_identifier, - CatalogUtils::BranchIdentifier(ctx->GetIdentifier().value(), branch)); + if (ctx.GetIdentifier()) { + PAIMON_ASSIGN_OR_RAISE(Identifier branch_identifier, + CatalogUtils::BranchIdentifier(ctx.GetIdentifier().value(), branch)); catalog_identifier.emplace(std::move(branch_identifier)); } // Only the main branch may have a current schema the catalog alone holds. A branch publishes // its own under `branch/branch-`, which is where the load below and a read of the branch // both read it, so a catalog answering for no branch schema does not fail a write to one. std::optional catalog_table_schema; - if (ctx->GetCatalog() != nullptr) { + if (ctx.GetCatalog() != nullptr) { if (!catalog_identifier) { return Status::Invalid("a catalog write requires a table identifier"); } if (BranchManager::IsMainBranch(branch)) { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, - ctx->GetCatalog()->LoadTableSchema(catalog_identifier.value())); + ctx.GetCatalog()->LoadTableSchema(catalog_identifier.value())); PAIMON_ASSIGN_OR_RAISE(catalog_table_schema, schema->GetJsonSchema()); } } // A format table writes plain data files into a directory, so it never reaches the manifest // path below. - auto schema_manager = - std::make_shared(tmp_options.GetFileSystem(), ctx->GetRootPath(), branch); + SchemaManager schema_manager(tmp_options.GetFileSystem(), ctx.GetRootPath(), branch); std::shared_ptr latest_schema; if (branch == BranchManager::DEFAULT_MAIN_BRANCH && catalog_table_schema) { // Handing the schema over only saves reading it. A context always names a table path and @@ -305,22 +377,136 @@ Result> FileStoreWrite::Create(std::unique_ptr> latest, - schema_manager->Latest()); + schema_manager.Latest()); if (latest) { latest_schema = *latest; } } - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, - TryLoadFormatTable(tmp_options.GetFileSystem(), ctx->GetRootPath(), - branch, ctx->GetOptions(), latest_schema)); - if (format_table != nullptr) { - return format_table->NewWrite(std::move(ctx)); + return TableOfLoadedSchema(tmp_options.GetFileSystem(), ctx.GetRootPath(), branch, + ctx.GetOptions(), latest_schema, ctx.GetIdentifier()); +} + +/// The table a commit context names, read from what the context carries: a format table the +/// builder was handed, or the table - format or managed - the schema under the context's root +/// path says is there. +Result> TableOf(const CommitContext& ctx) { + // A table the caller already loaded says what it is, so nothing is read to find out. + if (ctx.GetFormatTable() != nullptr) { + return std::shared_ptr
(ctx.GetFormatTable()); + } + PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx.GetCatalog())); + // Use catalog credentials unless the caller supplied a file system. + std::shared_ptr specific_fs = ctx.GetSpecificFileSystem(); + if (specific_fs == nullptr && ctx.GetCatalog() != nullptr) { + if (!ctx.GetIdentifier()) { + return Status::Invalid("a catalog commit requires a table identifier"); + } + // Use the credentials of this table, which a catalog issuing per-table temporary + // ones only hands out through GetTableFileSystem. + PAIMON_ASSIGN_OR_RAISE(specific_fs, + ctx.GetCatalog()->GetTableFileSystem(ctx.GetIdentifier().value())); + } + PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, + CoreOptions::FromMap(ctx.GetOptions(), specific_fs)); + const std::string& root_path = ctx.GetRootPath(); + // The branch the context was built for, already validated by `CommitContextBuilder`. + PAIMON_ASSIGN_OR_RAISE(std::string branch, BranchManager::ResolveBranch( + ctx.GetIdentifier(), ctx.GetOptions(), + /*explicit_branch=*/std::nullopt, "commit")); + std::optional catalog_identifier; + if (ctx.GetIdentifier()) { + PAIMON_ASSIGN_OR_RAISE(Identifier branch_identifier, + CatalogUtils::BranchIdentifier(ctx.GetIdentifier().value(), branch)); + catalog_identifier.emplace(std::move(branch_identifier)); + } + // Catalog-managed schemas may be absent from the table directory, but only on the main branch: + // a branch publishes its own under `branch/branch-`, which is where the load below and a + // read of the branch both read it, so a catalog answering for no branch schema does not fail + // a commit to one. + std::optional catalog_table_schema; + if (ctx.GetCatalog() != nullptr) { + if (!catalog_identifier) { + return Status::Invalid("a catalog commit requires a table identifier"); + } + if (BranchManager::IsMainBranch(branch)) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr loaded_schema, + ctx.GetCatalog()->LoadTableSchema(catalog_identifier.value())); + PAIMON_ASSIGN_OR_RAISE(std::string schema_json, loaded_schema->GetJsonSchema()); + catalog_table_schema = std::move(schema_json); + } + } + // A format table commits by renaming files into place, so it never reaches the snapshot path + // below. The managed path here reads the commit's branch, so this reads the same one: the two + // must not dispatch on different schemas. + SchemaManager schema_manager(tmp_options.GetFileSystem(), root_path, branch); + std::shared_ptr latest_schema; + if (branch == BranchManager::DEFAULT_MAIN_BRANCH && catalog_table_schema) { + // Handing the schema over only saves reading it. A context always names a table path and + // this library writes the metadata under it, so `schema` and `branch` stay metadata. + PAIMON_ASSIGN_OR_RAISE(latest_schema, + TableSchema::CreateFromJson(catalog_table_schema.value())); + } else { + PAIMON_ASSIGN_OR_RAISE(std::optional> latest, + schema_manager.Latest()); + if (latest) { + latest_schema = *latest; + } } - // The schema the branch read above through `schema_manager`, rather than a second read of - // the same file. + return TableOfLoadedSchema(tmp_options.GetFileSystem(), root_path, branch, ctx.GetOptions(), + latest_schema, ctx.GetIdentifier(), ctx.GetTableId().value_or("")); +} + +} // namespace + +Table::Table(const std::shared_ptr& file_system, const std::string& root_path, + const std::shared_ptr& schema, const Identifier& identifier, + const std::string& uuid) + : file_system_(file_system), + root_path_(root_path), + schema_(schema), + identifier_(identifier), + uuid_(uuid) {} + +Result> FileStoreTable::NewWrite( + std::unique_ptr ctx) const { + if (ctx == nullptr) { + return Status::Invalid("write context is null pointer"); + } + if (ctx->GetMemoryPool() == nullptr) { + return Status::Invalid("memory pool is null pointer"); + } + if (ctx->GetExecutor() == nullptr) { + return Status::Invalid("executor is null pointer"); + } + PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog())); + std::shared_ptr specific_fs = ctx->GetSpecificFileSystem(); + if (specific_fs == nullptr && ctx->GetFileSystemSchemeToIdentifierMap().empty() && + ctx->GetCatalog() != nullptr) { + if (!ctx->GetIdentifier()) { + return Status::Invalid("a catalog write requires a table identifier"); + } + // A catalog issuing per-table temporary credentials only hands them out through + // GetTableFileSystem, so the write uses the credentials of this table. + PAIMON_ASSIGN_OR_RAISE(specific_fs, + ctx->GetCatalog()->GetTableFileSystem(ctx->GetIdentifier().value())); + } + PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, + CoreOptions::FromMap(ctx->GetOptions(), specific_fs, + ctx->GetFileSystemSchemeToIdentifierMap())); + std::string branch = ctx->GetBranch(); + std::optional catalog_identifier; + if (ctx->GetIdentifier()) { + PAIMON_ASSIGN_OR_RAISE( + Identifier branch_identifier, + CatalogUtils::BranchIdentifier(ctx->GetIdentifier().value(), branch)); + catalog_identifier.emplace(std::move(branch_identifier)); + } + auto schema_manager = + std::make_shared(tmp_options.GetFileSystem(), ctx->GetRootPath(), branch); + // The schema the table was constructed with, so a write serves the table as it was loaded + // rather than reading the schema again. + std::shared_ptr latest_schema = std::static_pointer_cast(schema_); if (latest_schema == nullptr) { return Status::Invalid(fmt::format("not found latest schema in branch {}", branch)); } @@ -587,6 +773,21 @@ Result> FileStoreWrite::Create(std::unique_ptr> FileStoreWrite::Create(std::unique_ptr ctx) { + if (ctx == nullptr) { + return Status::Invalid("write context is null pointer"); + } + if (ctx->GetMemoryPool() == nullptr) { + return Status::Invalid("memory pool is null pointer"); + } + if (ctx->GetExecutor() == nullptr) { + return Status::Invalid("executor is null pointer"); + } + // First the table the context names, then the context handed to that table's `NewWrite`. + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, TableOf(*ctx)); + return table->NewWrite(std::move(ctx)); +} + namespace { class TableScanImpl { @@ -692,71 +893,6 @@ class TableScanImpl { } }; -/// @param loaded_schema The table's schema, when the caller already read it, or null when it has -/// to be read here. `TableScan::Create()` reads one to dispatch on its table type. -Result> NewDataTableScan( - const std::shared_ptr& context, const std::shared_ptr& loaded_schema); - -} // namespace - -Result> TableScan::Create(std::unique_ptr context) { - if (context == nullptr) { - return Status::Invalid("scan context is null pointer"); - } - if (context->GetMemoryPool() == nullptr) { - return Status::Invalid("memory pool is null pointer"); - } - if (context->GetExecutor() == nullptr) { - return Status::Invalid("executor is null pointer"); - } - - // A table the caller already loaded says what it is, so nothing is read to find out. - if (context->GetFormatTable() != nullptr) { - return context->GetFormatTable()->NewScan(std::move(context)); - } - // load schema - PAIMON_ASSIGN_OR_RAISE( - CoreOptions tmp_options, - CoreOptions::FromMap(context->GetOptions(), context->GetSpecificFileSystem(), {})); - PAIMON_ASSIGN_OR_RAISE(std::optional system_table_path, - SystemTableLoader::TryParsePath(context->GetPath())); - if (system_table_path) { - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr system_table, - SystemTableLoader::LoadFromPath(tmp_options.GetFileSystem(), context->GetPath(), - context->GetOptions())); - return system_table->NewScan(std::move(context)); - } - // A format table is planned by listing directories, so it never reaches the manifest path - // below. One `TableScan` interface serves both, as Java Paimon serves both through one - // `ReadBuilder`. - const std::string branch = BranchManager::NormalizeBranch(tmp_options.GetBranch()); - std::shared_ptr latest_schema; - if (branch == BranchManager::DEFAULT_MAIN_BRANCH && context->GetSpecificTableSchema()) { - // Handing the schema over only saves reading it. A context always names a table path and - // this library writes the metadata under it, so `schema` and `branch` stay metadata. - PAIMON_ASSIGN_OR_RAISE( - latest_schema, TableSchema::CreateFromJson(context->GetSpecificTableSchema().value())); - } else { - SchemaManager schema_manager(tmp_options.GetFileSystem(), context->GetPath(), branch); - PAIMON_ASSIGN_OR_RAISE(std::optional> latest, - schema_manager.Latest()); - if (latest) { - latest_schema = *latest; - } - } - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, - TryLoadFormatTable(tmp_options.GetFileSystem(), context->GetPath(), - branch, context->GetOptions(), latest_schema)); - if (format_table != nullptr) { - return format_table->NewScan(std::move(context)); - } - // With the schema already read above, so the managed path does not read it again. - return NewDataTableScan(std::move(context), latest_schema); -} - -namespace { - Status ValidateRealtimeScan(const TableSchema& table_schema, const CoreOptions& core_options, const ScanContext& context, bool read_optimized) { if (!context.GetRealtimeContext()) { @@ -799,12 +935,22 @@ Status ValidateRealtimeScan(const TableSchema& table_schema, const CoreOptions& return Status::OK(); } -/// @param loaded_schema The schema `TableScan::Create()` already read while dispatching on the -/// table type, or null when neither the caller's json nor the branch's own directory had -/// one. -Result> NewDataTableScan( - const std::shared_ptr& context, - const std::shared_ptr& loaded_schema) { +} // namespace + +Result> FileStoreTable::NewScan( + std::unique_ptr context) const { + if (context == nullptr) { + return Status::Invalid("scan context is null pointer"); + } + if (context->GetMemoryPool() == nullptr) { + return Status::Invalid("memory pool is null pointer"); + } + if (context->GetExecutor() == nullptr) { + return Status::Invalid("executor is null pointer"); + } + // The schema the table was constructed with, so a scan serves the table as it was loaded + // rather than reading the schema again. + std::shared_ptr loaded_schema = std::static_pointer_cast(schema_); if (loaded_schema == nullptr) { return Status::Invalid("not found latest schema"); } @@ -907,12 +1053,25 @@ Result> NewDataTableScan( return batch_scan; } -} // namespace +Result> TableScan::Create(std::unique_ptr context) { + if (context == nullptr) { + return Status::Invalid("scan context is null pointer"); + } + if (context->GetMemoryPool() == nullptr) { + return Status::Invalid("memory pool is null pointer"); + } + if (context->GetExecutor() == nullptr) { + return Status::Invalid("executor is null pointer"); + } + // First the table the context names, then the context handed to that table's `NewScan`. + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, TableOf(*context)); + return table->NewScan(std::move(context)); +} namespace { /// @param loaded_schema The schema of `branch`, when the caller already read it, or null when it -/// has to be read here: `TableRead::Create()` reads one to dispatch on the table type, and +/// has to be read here: `FileStoreTable::NewRead()` hands the table's own schema over, and /// reading it again would cost a second listing and read. Result> CreateInternalReadContext( const std::shared_ptr& context, const std::string& branch, @@ -988,15 +1147,30 @@ Result> CreateTableRead( return KeyValueTableRead::Create(path_factory, internal_context, memory_pool, executor); } -Result> NewDataTableRead( - const std::shared_ptr& context, - const std::shared_ptr& loaded_schema) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr internal_context, - CreateInternalReadContext(context, context->GetBranch(), loaded_schema)); +} // namespace +Result> FileStoreTable::NewRead( + std::unique_ptr context) const { + std::shared_ptr read_context = std::move(context); + if (read_context == nullptr) { + return Status::Invalid("read context is null pointer"); + } + if (read_context->GetMemoryPool() == nullptr) { + return Status::Invalid("memory pool is null pointer"); + } + if (read_context->GetExecutor() == nullptr) { + return Status::Invalid("executor is null pointer"); + } + // The schema the table was constructed with, so a read serves the table as it was loaded + // rather than reading the schema again. + std::shared_ptr loaded_schema = std::static_pointer_cast(schema_); PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr table_read, - CreateTableRead(internal_context, context->GetMemoryPool(), context->GetExecutor())); + std::shared_ptr internal_context, + CreateInternalReadContext(read_context, read_context->GetBranch(), loaded_schema)); + + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, + CreateTableRead(internal_context, read_context->GetMemoryPool(), + read_context->GetExecutor())); std::optional scan_fallback_branch = internal_context->GetCoreOptions().GetScanFallbackBranch(); @@ -1007,19 +1181,17 @@ Result> NewDataTableRead( PAIMON_ASSIGN_OR_RAISE( std::shared_ptr fallback_context, - CreateInternalReadContext(context, /*branch=*/scan_fallback_branch.value(), + CreateInternalReadContext(read_context, /*branch=*/scan_fallback_branch.value(), /*loaded_schema=*/nullptr)); - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr fallback_table_read, - CreateTableRead(fallback_context, context->GetMemoryPool(), context->GetExecutor())); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr fallback_table_read, + CreateTableRead(fallback_context, read_context->GetMemoryPool(), + read_context->GetExecutor())); const std::shared_ptr& arrow_pool = internal_context->GetArrowMemoryPool(); return std::make_unique(std::move(table_read), std::move(fallback_table_read), arrow_pool); } -} // namespace - Result> TableRead::Create(std::unique_ptr ctx) { if (ctx == nullptr) { return Status::Invalid("read context is null pointer"); @@ -1030,57 +1202,9 @@ Result> TableRead::Create(std::unique_ptrGetExecutor() == nullptr) { return Status::Invalid("executor is null pointer"); } - // A table the caller already loaded says what it is, so nothing is read to find out. - if (ctx->GetFormatTable() != nullptr) { - return ctx->GetFormatTable()->NewRead(std::move(ctx)); - } - PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_core_options, - CoreOptions::FromMap(ctx->GetOptions(), ctx->GetSpecificFileSystem(), - ctx->GetFileSystemSchemeToIdentifierMap())); - PAIMON_ASSIGN_OR_RAISE(std::optional system_table_path, - SystemTableLoader::TryParsePath(ctx->GetPath())); - if (system_table_path) { - if (PredicateUtils::ContainsFullTextMatch(ctx->GetPredicate())) { - return Status::NotImplemented( - "FullTextMatch predicate execution is not implemented yet"); - } - if (ctx->HasVectorOrFullTextSearch()) { - return Status::NotImplemented( - "VectorSearch or FullTextSearch is not supported for system tables"); - } - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, - SystemTableLoader::LoadFromPath(tmp_core_options.GetFileSystem(), - ctx->GetPath(), ctx->GetOptions())); - return system_table->NewRead(std::move(ctx)); - } - - // A format table has no manifest and a file another engine wrote carries no field ids, so - // which files there are and how a row is put back together is its own. How a file is opened - // is not: both paths go through `DataFileReaderFactory`. - const std::string branch = ctx->GetBranch(); - std::shared_ptr latest_schema; - if (branch == BranchManager::DEFAULT_MAIN_BRANCH && ctx->GetSpecificTableSchema()) { - // Handing the schema over only saves reading it. A context always names a table path and - // this library writes the metadata under it, so `schema` and `branch` stay metadata. - PAIMON_ASSIGN_OR_RAISE(latest_schema, - TableSchema::CreateFromJson(ctx->GetSpecificTableSchema().value())); - } else { - SchemaManager schema_manager(tmp_core_options.GetFileSystem(), ctx->GetPath(), branch); - PAIMON_ASSIGN_OR_RAISE(std::optional> latest, - schema_manager.Latest()); - if (latest) { - latest_schema = *latest; - } - } - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, - TryLoadFormatTable(tmp_core_options.GetFileSystem(), ctx->GetPath(), - branch, ctx->GetOptions(), latest_schema)); - if (format_table != nullptr) { - return format_table->NewRead(std::move(ctx)); - } - - // With the schema already read above, so the managed path does not read it again. - return NewDataTableRead(std::move(ctx), latest_schema); + // First the table the context names, then the context handed to that table's `NewRead`. + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, TableOf(*ctx)); + return table->NewRead(std::move(ctx)); } namespace { @@ -1167,8 +1291,8 @@ Result> NewSnapshotCommit( } // namespace -Result> FileStoreCommit::Create( - std::unique_ptr ctx) { +Result> FileStoreTable::NewCommit( + std::unique_ptr ctx) const { if (ctx == nullptr) { return Status::Invalid("commit context is null pointer"); } @@ -1178,12 +1302,6 @@ Result> FileStoreCommit::Create( if (ctx->GetExecutor() == nullptr) { return Status::Invalid("executor is null pointer"); } - - // A table the caller already loaded says what it is, so nothing is read to find out. - if (ctx->GetFormatTable() != nullptr) { - return ctx->GetFormatTable()->NewCommit(std::move(ctx)); - } - PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog())); // Use catalog credentials unless the caller supplied a file system. std::shared_ptr specific_fs = ctx->GetSpecificFileSystem(); @@ -1210,22 +1328,13 @@ Result> FileStoreCommit::Create( CatalogUtils::BranchIdentifier(ctx->GetIdentifier().value(), branch)); catalog_identifier.emplace(std::move(branch_identifier)); } - // Catalog-managed schemas may be absent from the table directory, but only on the main branch: - // a branch publishes its own under `branch/branch-`, which is where the load below and a - // read of the branch both read it, so a catalog answering for no branch schema does not fail - // a commit to one. - std::optional catalog_table_schema; + // The id a snapshot records the catalog's current schema by, read through the loader below + // rather than trusted from a schema this commit will not re-read. FileStoreCommitImpl::SchemaIdLoader schema_id_loader; if (ctx->GetCatalog() != nullptr) { if (!catalog_identifier) { return Status::Invalid("a catalog commit requires a table identifier"); } - if (BranchManager::IsMainBranch(branch)) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr loaded_schema, - ctx->GetCatalog()->LoadTableSchema(catalog_identifier.value())); - PAIMON_ASSIGN_OR_RAISE(std::string schema_json, loaded_schema->GetJsonSchema()); - catalog_table_schema = std::move(schema_json); - } std::shared_ptr catalog = ctx->GetCatalog(); Identifier identifier = catalog_identifier.value(); schema_id_loader = [catalog, identifier]() -> Result { @@ -1240,32 +1349,11 @@ Result> FileStoreCommit::Create( return data_schema->Id(); }; } - // A format table commits by renaming files into place, so it never reaches the snapshot path - // below. The managed path here reads the commit's branch, so this reads the same one: the two - // must not dispatch on different schemas. auto schema_manager = std::make_shared(tmp_options.GetFileSystem(), root_path, branch); - std::shared_ptr latest_schema; - if (branch == BranchManager::DEFAULT_MAIN_BRANCH && catalog_table_schema) { - // Handing the schema over only saves reading it. A context always names a table path and - // this library writes the metadata under it, so `schema` and `branch` stay metadata. - PAIMON_ASSIGN_OR_RAISE(latest_schema, - TableSchema::CreateFromJson(catalog_table_schema.value())); - } else { - // Through the manager built above, so the read warms the cache the managed path below - // goes on to use. - PAIMON_ASSIGN_OR_RAISE(std::optional> latest, - schema_manager->Latest()); - if (latest) { - latest_schema = *latest; - } - } - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, - TryLoadFormatTable(tmp_options.GetFileSystem(), root_path, branch, - ctx->GetOptions(), latest_schema)); - if (format_table != nullptr) { - return format_table->NewCommit(std::move(ctx)); - } + // The schema the table was constructed with, so a commit serves the table as it was loaded + // rather than reading the schema again. + std::shared_ptr latest_schema = std::static_pointer_cast(schema_); if (latest_schema == nullptr) { return Status::Invalid(fmt::format("not found latest schema in branch {}", branch)); } @@ -1367,4 +1455,20 @@ Result> FileStoreCommit::Create( expire_snapshots, schema_manager, std::move(scan_supplier)); } +Result> FileStoreCommit::Create( + std::unique_ptr ctx) { + if (ctx == nullptr) { + return Status::Invalid("commit context is null pointer"); + } + if (ctx->GetMemoryPool() == nullptr) { + return Status::Invalid("memory pool is null pointer"); + } + if (ctx->GetExecutor() == nullptr) { + return Status::Invalid("executor is null pointer"); + } + // First the table the context names, then the context handed to that table's `NewCommit`. + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, TableOf(*ctx)); + return table->NewCommit(std::move(ctx)); +} + } // namespace paimon diff --git a/src/paimon/core/table/table_test.cpp b/src/paimon/core/table/table_test.cpp index 78d01e5a5..1ad8d9cd3 100644 --- a/src/paimon/core/table/table_test.cpp +++ b/src/paimon/core/table/table_test.cpp @@ -291,6 +291,29 @@ TEST(TableTest, TestNewScanCreatesATableScan) { table->NewScan(std::move(ctx))); } +TEST(TableTest, TestNewScanServesTheSchemaTheTableWasLoadedWith) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto fs = dir->GetFileSystem(); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + CreateManagedTable(fs, dir->Str(), identifier)); + + // With the schema file gone the table still serves a scan: it carries the schema it was + // loaded with, which is what a scan is built from. + SchemaManager schema_manager(fs, dir->Str()); + ASSERT_OK(fs->Delete(schema_manager.ToSchemaPath(0), /*recursive=*/false)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr held_ctx, + ScanContextBuilder(dir->Str()).Finish()); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr scan, + table->NewScan(std::move(held_ctx))); + + // A context without a table still reads the schema from the directory, which no longer has + // one. + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, ScanContextBuilder(dir->Str()).Finish()); + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(ctx)), "not found latest schema"); +} + TEST(TableTest, TestNewReadCreatesATableRead) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); From 05a46c490763c07e7e1b0a727b1c61e2fb16dff7 Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Thu, 22 Oct 2026 15:05:34 +0800 Subject: [PATCH 04/30] tmp --- include/paimon/catalog/table.h | 22 +- include/paimon/table/format/format_table.h | 35 +- src/paimon/CMakeLists.txt | 1 + .../core/catalog/file_system_catalog.cpp | 8 +- ...eted_primary_key_index_maintainer_test.cpp | 26 +- .../append_only_file_store_write_test.cpp | 22 +- .../core/operation/internal_read_context.cpp | 28 +- .../core/operation/internal_read_context.h | 14 +- src/paimon/core/schema/table_schema.cpp | 42 +- src/paimon/core/schema/table_schema.h | 13 + src/paimon/core/table/file_store.cpp | 100 +++ src/paimon/core/table/file_store.h | 125 +++ src/paimon/core/table/file_store_table.h | 59 +- src/paimon/core/table/format/format_table.cpp | 33 +- .../core/table/format/format_table_test.cpp | 26 +- .../core/table/source/table_read_test.cpp | 54 +- src/paimon/core/table/system/system_table.cpp | 5 + src/paimon/core/table/system/system_table.h | 6 + .../core/table/system/system_table_test.cpp | 14 + src/paimon/core/table/table.cpp | 717 +++++++++--------- src/paimon/core/table/table_test.cpp | 434 ++++++++++- src/paimon/rest/rest_catalog.cpp | 8 +- src/paimon/rest/rest_catalog_test.cpp | 6 +- test/inte/realtime_write_inte_test.cpp | 57 +- test/inte/scan_and_read_inte_test.cpp | 64 +- test/inte/scan_inte_test.cpp | 27 +- .../write_and_read_inte_with_index_test.cpp | 3 +- 27 files changed, 1449 insertions(+), 500 deletions(-) create mode 100644 src/paimon/core/table/file_store.cpp create mode 100644 src/paimon/core/table/file_store.h diff --git a/include/paimon/catalog/table.h b/include/paimon/catalog/table.h index 69abbe6d5..a31d3100d 100644 --- a/include/paimon/catalog/table.h +++ b/include/paimon/catalog/table.h @@ -55,13 +55,15 @@ class PAIMON_EXPORT Table { /// Returns the file system the table was opened with, which serves the table's /// files. A catalog that hands out per-table file systems opens a table with the - /// one for that table; it is null when the opener had none. + /// one for that table; it is null when the opener had none. The `NewXxx` + /// operations read the table through it. std::shared_ptr GetFileSystem() const { return file_system_; } /// Returns the root path of the table in its file system. It is empty when the /// opener had no location for the table, which a mock in a test is one case of. + /// The `NewXxx` operations serve the table from it. const std::string& GetRootPath() const { return root_path_; } @@ -74,13 +76,6 @@ class PAIMON_EXPORT Table { /// table answers database.tableName, a system table the same name as `Name()`. virtual std::string FullName() const = 0; - /// Returns the branch the table was opened for, which its identifier carries; `main` - /// when it names none or a spelling of it, which its managers are bound to. A - /// malformed object name that hides the branch is reported. - Result CurrentBranch() const { - return identifier_.GetBranchNameOrDefault(); - } - /// Returns the metastore UUID, falling back to `FullName()`. /// @note Use `CatalogUuid()` for commit requests to avoid sending a table name as an ID. std::string Uuid() const { @@ -98,6 +93,17 @@ class PAIMON_EXPORT Table { return schema_; } + /// Copies this table with `dynamic_options` on top of the options it carries, which is + /// the precedence every context builder promises for a table it was handed rather than loaded + /// itself: an option given at the call wins over the one the schema stored. + /// + /// `type` is not overridable: it is structural and is read from the schema alone. + /// + /// @param dynamic_options Options given at the call. + /// @return A result containing the copied table, or the error copying it reports. + virtual Result> Copy( + const std::map& dynamic_options) const = 0; + /// Creates a scan of this table from a finished scan context, the table-first entry point /// that `TableScan::Create()` loads the table the context names and hands the context to. /// @param context The finished scan context. diff --git a/include/paimon/table/format/format_table.h b/include/paimon/table/format/format_table.h index d15a8f6ce..88444d360 100644 --- a/include/paimon/table/format/format_table.h +++ b/include/paimon/table/format/format_table.h @@ -55,8 +55,7 @@ class FileSystem; /// loaded through the `FormatTable` constructor every context builder has, which is how a table /// whose schema lives in a metastore is reached. /// -/// Every instance lives in a `shared_ptr`, which is what its operators hand to -/// `FormatTable::Copy()`. +/// Every instance lives in a `shared_ptr`: a copy that adds nothing hands the same one back. /// /// See `docs/source/user_guide/format_table.rst` for what is not supported yet. class PAIMON_EXPORT FormatTable : public Table, public std::enable_shared_from_this { @@ -108,20 +107,7 @@ class PAIMON_EXPORT FormatTable : public Table, public std::enable_shared_from_t bool location_carries_paimon_metadata, const std::map& dynamic_options); - /// Copies `table` with `dynamic_options` on top of the options it already carries, which is - /// the precedence every context builder promises for a table it was handed rather than loaded - /// itself. `table` comes back as it is when there is nothing to add. - /// - /// `type` is not overridable: it is structural and is read from the schema alone. - /// - /// @param table The table to copy. - /// @param dynamic_options Options given at the call. - /// @return A result containing the copied table, or an error status. - static Result> Copy( - const std::shared_ptr& table, - const std::map& dynamic_options); - - ~FormatTable(); + ~FormatTable() override; std::string Name() const override { return identifier_.GetObjectName(); @@ -131,6 +117,11 @@ class PAIMON_EXPORT FormatTable : public Table, public std::enable_shared_from_t return identifier_.GetFullName(); } + /// Copies the table with `dynamic_options` on top of the options it carries, the precedence + /// every context builder promises. The same table comes back when there is nothing to add. + Result> Copy( + const std::map& dynamic_options) const override; + /// Directory the data files live in. const std::string& Location() const { return GetRootPath(); @@ -217,15 +208,21 @@ class PAIMON_EXPORT FormatTable : public Table, public std::enable_shared_from_t const std::string& file_compression, const std::string& partition_default_name, bool partition_only_value_in_path, bool location_carries_paimon_metadata); - /// This table as a `shared_ptr`, which the operators above hand to `FormatTable::Copy()`. + /// This table as a `shared_ptr`, which a copy that adds nothing hands back. /// /// `shared_from_this()` on a const method hands back a `shared_ptr`, while - /// `Copy()` takes a mutable one; the cast is safe because `Copy()` never writes to the table - /// it is given. + /// the copy below needs a mutable one; the cast is safe because the table is never written to. std::shared_ptr SharedSelf() const { return std::const_pointer_cast(shared_from_this()); } + /// Copies this table with `dynamic_options` on top of the options it carries, as a format + /// table rather than the `Table` `Copy()` hands back, which is what its operators need. The + /// same table comes back when there is nothing to add; `type` is not overridable, and + /// `Create()` keeps the schema's own. + Result> CopySelf( + const std::map& dynamic_options) const; + std::map options_; Format format_; std::string file_compression_; diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index 18519223a..b6ab5f92b 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -445,6 +445,7 @@ set(PAIMON_CORE_SRCS core/stats/simple_stats_evolution.cpp core/table/table.cpp core/table/bucket_mode.cpp + core/table/file_store.cpp core/table/format/format_file_listing.cpp core/table/format/format_file_naming.cpp core/table/format/format_path_validation.cpp diff --git a/src/paimon/core/catalog/file_system_catalog.cpp b/src/paimon/core/catalog/file_system_catalog.cpp index 46131f729..36721a148 100644 --- a/src/paimon/core/catalog/file_system_catalog.cpp +++ b/src/paimon/core/catalog/file_system_catalog.cpp @@ -347,8 +347,12 @@ Result> FileSystemCatalog::GetTable(const Identifier& ide // table rather than at the first read. PAIMON_RETURN_NOT_OK(CatalogUtils::CheckManagedTableType(identifier, *latest_schema, "FileSystemCatalog::GetTable")); - return std::make_shared( - fs_, table_path, checked_pointer_cast(*latest_schema), identifier); + // The branch the schema above was read on, which the table carries so its managers read + // the same branch: `GetBranchNameOrDefault` folds any spelling of `main`. + PAIMON_ASSIGN_OR_RAISE(std::string branch, identifier.GetBranchNameOrDefault()); + return std::make_shared(fs_, table_path, + checked_pointer_cast(*latest_schema), + identifier, /*uuid=*/"", branch); } Result> FileSystemCatalog::LoadSystemTable( diff --git a/src/paimon/core/index/pk/bucketed_primary_key_index_maintainer_test.cpp b/src/paimon/core/index/pk/bucketed_primary_key_index_maintainer_test.cpp index 81803ed90..769bbf326 100644 --- a/src/paimon/core/index/pk/bucketed_primary_key_index_maintainer_test.cpp +++ b/src/paimon/core/index/pk/bucketed_primary_key_index_maintainer_test.cpp @@ -33,6 +33,7 @@ #include "gtest/gtest.h" #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" +#include "paimon/catalog/table.h" #include "paimon/commit_context.h" #include "paimon/common/utils/path_util.h" #include "paimon/core/global_index/indexed_split_impl.h" @@ -473,15 +474,21 @@ TEST_P(BucketedPrimaryKeyIndexMaintainerTest, BuildsRestoresAndReplacesCompacted ASSERT_EQ(0, data_split_count); ReadContextBuilder read_builder(table_path_); - // The source build above keeps lazy decoding enabled. Disable it only for the final data-row - // assertion, whose predicate reader is outside the maintenance path covered by this test. read_builder.SetReadFieldNames({"id", "value"}) .SetPredicate(value_predicate) - .AddOption("orc.read.enable-lazy-decoding", "false") .EnablePredicateFilter(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); + // The source build above keeps lazy decoding enabled. Disable it only for the final data-row + // assertion, whose predicate reader is outside the maintenance path covered by this test. + // The table is copied with it off because a read through the loaded table refuses an option + // that contradicts the one it carries. + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, Catalog::Create(directory_->Str(), {})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
loaded_table, + catalog->GetTable(Identifier("db", "table"))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
copied_table, + loaded_table->Copy({{"orc.read.enable-lazy-decoding", "false"}})); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + copied_table->NewRead(std::move(read_context))); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, @@ -707,11 +714,18 @@ TEST_P(BucketedPrimaryKeyIndexMaintainerTest, ContinuesCompactionWhenIndexBuildF ReadContextBuilder read_builder(table_path_); read_builder.SetReadFieldNames({"id", "value"}) .SetPredicate(value_predicate) - .AddOption("orc.read.enable-lazy-decoding", "false") .EnablePredicateFilter(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); + // Like the test above, the assertion reads with lazy decoding disabled, so it reads through + // a copy of the table carrying it off: a read through the loaded table refuses an option that + // contradicts the one it carries. + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, Catalog::Create(directory_->Str(), {})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
loaded_table, + catalog->GetTable(Identifier("db", "table"))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
copied_table, + loaded_table->Copy({{"orc.read.enable-lazy-decoding", "false"}})); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + copied_table->NewRead(std::move(read_context))); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, diff --git a/src/paimon/core/operation/append_only_file_store_write_test.cpp b/src/paimon/core/operation/append_only_file_store_write_test.cpp index bfeaefced..3ce48ee2c 100644 --- a/src/paimon/core/operation/append_only_file_store_write_test.cpp +++ b/src/paimon/core/operation/append_only_file_store_write_test.cpp @@ -36,6 +36,7 @@ #include "gtest/gtest.h" #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" +#include "paimon/catalog/table.h" #include "paimon/commit_context.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/data/binary_row_writer.h" @@ -513,10 +514,25 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestSharedShreddingNewWriterIgnoresExisting /*commit_identifier=*/0); Commit(table_path, avro_options, avro_commit_msgs); - auto second_commit_msgs = WriteAndPrepare(table_path, logical_schema, shredding_options, R"([ + // The second batch is written through a copy of the table carrying the shredding options: + // an option on the context would contradict the table the catalog loaded, which a write + // through that table refuses. + ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(dir->Str(), avro_options)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
avro_table, + catalog->GetTable(Identifier("foo", "bar"))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
shredded_table, + avro_table->Copy(shredding_options)); + WriteContextBuilder shredded_builder(table_path, commit_user_); + ASSERT_OK_AND_ASSIGN(std::unique_ptr shredded_context, shredded_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto shredded_write, + shredded_table->NewWrite(std::move(shredded_context))); + ASSERT_OK(shredded_write->Write(MakeBatch(logical_schema, R"([ [2, [["c", 3], ["d", 4], ["e", 5]]] - ])", - /*commit_identifier=*/1); + ])"))); + ASSERT_OK_AND_ASSIGN(auto second_commit_msgs, + shredded_write->PrepareCommit(/*wait_compaction=*/false, + /*commit_identifier=*/1)); + ASSERT_OK(shredded_write->Close()); auto second_file_schema = ReadDataFileSchema(table_path, OnlyNewFile(second_commit_msgs), shredding_options); auto second_meta = ShreddingMeta(second_file_schema, /*field_index=*/1); diff --git a/src/paimon/core/operation/internal_read_context.cpp b/src/paimon/core/operation/internal_read_context.cpp index f36a15d71..6c0c43341 100644 --- a/src/paimon/core/operation/internal_read_context.cpp +++ b/src/paimon/core/operation/internal_read_context.cpp @@ -275,10 +275,15 @@ std::optional InternalReadContext::TryResolveSpecialFieldByName( Result> InternalReadContext::Create( const std::shared_ptr& context, const std::shared_ptr& table_schema, - const std::map& options) { - PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, - CoreOptions::FromMap(options, context->GetSpecificFileSystem(), - context->GetFileSystemSchemeToIdentifierMap())); + const std::map& options, + const std::shared_ptr& file_system, const std::string* path) { + // The file system a table reads through, or null for the read context's own, which callers + // without a table hand over. + PAIMON_ASSIGN_OR_RAISE( + CoreOptions core_options, + CoreOptions::FromMap( + options, file_system != nullptr ? file_system : context->GetSpecificFileSystem(), + context->GetFileSystemSchemeToIdentifierMap())); core_options.WithCache(context->GetCache()); if (context->HasVectorOrFullTextSearch()) { if (!core_options.FileIndexReadEnabled()) { @@ -390,10 +395,11 @@ Result> InternalReadContext::Create( return Status::Invalid("memory pool is null"); } std::shared_ptr arrow_pool = GetArrowPool(context->GetMemoryPool()); - auto internal_context = std::unique_ptr( - new InternalReadContext(context, table_schema, read_schema, core_options, arrow_pool)); + auto internal_context = std::unique_ptr(new InternalReadContext( + context, table_schema, read_schema, core_options, arrow_pool, + path != nullptr ? std::optional(*path) : std::nullopt)); PAIMON_RETURN_NOT_OK(internal_context->BindIndexScores()); - return internal_context; + return internal_context } Status InternalReadContext::BindIndexScores() { @@ -471,12 +477,14 @@ InternalReadContext::InternalReadContext(const std::shared_ptr& rea const std::shared_ptr& table_schema, const std::shared_ptr& read_schema, const CoreOptions& options, - const std::shared_ptr& arrow_pool) + const std::shared_ptr& arrow_pool, + std::optional path) : read_context_(read_context), table_schema_(table_schema), read_schema_(read_schema), arrow_pool_(arrow_pool), - options_(options) {} + options_(options), + path_(std::move(path)) {} Result> InternalReadContext::CreateWithSchema( const std::shared_ptr& original, @@ -490,7 +498,7 @@ Result> InternalReadContext::CreateWithSche // The new read_schema is the minimal column set for COUNT(*). return std::shared_ptr( new InternalReadContext(original->read_context_, original->table_schema_, new_read_schema, - original->options_, original->arrow_pool_)); + original->options_, original->arrow_pool_, original->path_)); } } // namespace paimon diff --git a/src/paimon/core/operation/internal_read_context.h b/src/paimon/core/operation/internal_read_context.h index 9de415d5f..b1619c5bc 100644 --- a/src/paimon/core/operation/internal_read_context.h +++ b/src/paimon/core/operation/internal_read_context.h @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -37,6 +38,7 @@ class Schema; namespace paimon { struct CatalogContext; class Executor; +class FileSystem; class MemoryPool; class Predicate; class RealtimeContext; @@ -59,7 +61,9 @@ class InternalReadContext { static Result> Create( const std::shared_ptr& read_context, const std::shared_ptr& table_schema, - const std::map& options); + const std::map& options, + const std::shared_ptr& file_system = nullptr, + const std::string* path = nullptr); /// Maps score output positions in the read schema to their logical search sources. const std::map& GetIndexScoreBindings() const { @@ -79,7 +83,7 @@ class InternalReadContext { return table_schema_; } const std::string& GetPath() const { - return read_context_->GetPath(); + return path_ ? *path_ : read_context_->GetPath(); } std::shared_ptr CreateDependencyCatalogContext() const; @@ -164,7 +168,8 @@ class InternalReadContext { const std::shared_ptr& table_schema, const std::shared_ptr& read_schema, const CoreOptions& options, - const std::shared_ptr& arrow_pool); + const std::shared_ptr& arrow_pool, + std::optional path = std::nullopt); static std::optional TryResolveSpecialFieldById(int32_t field_id, const CoreOptions& core_options, @@ -184,6 +189,9 @@ class InternalReadContext { std::shared_ptr arrow_pool_; CoreOptions options_; std::map index_score_bindings_; + /// The root path a table reads through, or null for the read context's own, which callers + /// without a table hand over. + std::optional path_; }; } // namespace paimon diff --git a/src/paimon/core/schema/table_schema.cpp b/src/paimon/core/schema/table_schema.cpp index 1d69c033a..e6151d4c7 100644 --- a/src/paimon/core/schema/table_schema.cpp +++ b/src/paimon/core/schema/table_schema.cpp @@ -194,6 +194,35 @@ TableSchema::TableSchema(int32_t version, int64_t id, const std::vector> TableSchema::MergeOptions( + const std::map& dynamic_options) const { + std::map options = options_; + for (const auto& [key, value] : dynamic_options) { + options[key] = value; + } + auto merged = std::unique_ptr( + new TableSchema(version_, id_, fields_, highest_field_id_, partition_keys_, primary_keys_, + options, comment_, time_millis_)); + // The derived members follow the merged options: a copy the options restyle must serve the + // bucket keys and the bucket count it carries, not the ones the base schema carried. + PAIMON_RETURN_NOT_OK(merged->DeriveBucketInfo()); + return merged; +} + +Status TableSchema::DeriveBucketInfo() { + PAIMON_ASSIGN_OR_RAISE(std::vector keys, TrimmedPrimaryKeys()); + + // Try to validate bucket keys + PAIMON_ASSIGN_OR_RAISE(std::vector bucket_keys, OriginalBucketKeys()); + if (bucket_keys.empty()) { + bucket_keys = keys; + } + bucket_keys_ = bucket_keys; + PAIMON_ASSIGN_OR_RAISE(num_bucket_, + OptionsUtils::GetValueFromMap(Options(), Options::BUCKET, -1)); + return Status::OK(); +} + bool TableSchema::operator==(const TableSchema& other) const { return version_ == other.version_ && fields_ == other.fields_ && partition_keys_ == other.partition_keys_ && primary_keys_ == other.primary_keys_ && @@ -332,18 +361,7 @@ Result> TableSchema::InitSchema( auto table_schema = std::unique_ptr( new TableSchema(TableSchema::CURRENT_VERSION, schema_id, fields, highest_field_id, partition_keys, primary_keys, options, comment, time_millis)); - PAIMON_ASSIGN_OR_RAISE(std::vector keys, table_schema->TrimmedPrimaryKeys()); - - // Try to validate bucket keys - PAIMON_ASSIGN_OR_RAISE(std::vector bucket_keys, - table_schema->OriginalBucketKeys()); - if (bucket_keys.empty()) { - bucket_keys = keys; - } - table_schema->bucket_keys_ = bucket_keys; - PAIMON_ASSIGN_OR_RAISE( - table_schema->num_bucket_, - OptionsUtils::GetValueFromMap(table_schema->Options(), Options::BUCKET, -1)); + PAIMON_RETURN_NOT_OK(table_schema->DeriveBucketInfo()); return table_schema; } diff --git a/src/paimon/core/schema/table_schema.h b/src/paimon/core/schema/table_schema.h index eab109cf1..aff4ee920 100644 --- a/src/paimon/core/schema/table_schema.h +++ b/src/paimon/core/schema/table_schema.h @@ -101,6 +101,15 @@ class TableSchema : public DataSchema, public Jsonizable { const std::map& Options() const override { return options_; } + + /// A copy of this schema with `dynamic_options` on top of the options it carries, which is + /// the precedence every copy of a table promises. Everything else - id, fields, keys, + /// comment - is copied as it is. + /// + /// @param dynamic_options Options given at the call, which win over the schema's own. + /// @return A result containing the merged schema, or an error status. + Result> MergeOptions( + const std::map& dynamic_options) const; const std::vector& Fields() const { return fields_; } @@ -142,6 +151,10 @@ class TableSchema : public DataSchema, public Jsonizable { Result> OriginalBucketKeys() const; + /// Derives `bucket_keys_` and `num_bucket_` from the options the schema carries: every + /// constructor path must call this, as both members serve scans and copies. + Status DeriveBucketInfo(); + static Result> AssignFieldIdsRecursively( const std::shared_ptr& field, bool set_field_id, int32_t* field_id); diff --git a/src/paimon/core/table/file_store.cpp b/src/paimon/core/table/file_store.cpp new file mode 100644 index 000000000..e70e85c1d --- /dev/null +++ b/src/paimon/core/table/file_store.cpp @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/table/file_store.h" + +#include +#include +#include +#include + +#include "paimon/catalog/catalog.h" +#include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/core/catalog/version_managed_catalog.h" +#include "paimon/core/core_options.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/core/utils/branch_manager.h" +#include "paimon/core/utils/file_store_path_factory.h" +#include "paimon/core/utils/snapshot_manager.h" +#include "paimon/result.h" +#include "paimon/status.h" + +namespace paimon { + +FileStore::FileStore(const std::shared_ptr& file_system, const std::string& root_path, + const std::shared_ptr& schema, const std::string& branch, + const std::shared_ptr& schema_manager, + const std::shared_ptr& snapshot_manager) + : file_system_(file_system), + root_path_(root_path), + schema_(schema), + branch_(BranchManager::NormalizeBranch(branch)), + schema_manager_(schema_manager), + snapshot_manager_(snapshot_manager) { + // A manager handed over points at `root_path` on `branch` already; a missing one is built + // here, so the operators always read through the managers the store carries. + if (schema_manager_ == nullptr) { + schema_manager_ = std::make_shared(file_system_, root_path_, branch_); + } + if (snapshot_manager_ == nullptr) { + snapshot_manager_ = std::make_shared(file_system_, root_path_, branch_); + } +} + +Result> FileStore::NewPathFactory( + const std::shared_ptr& arrow_schema, const CoreOptions& options, + const std::string& file_format_identifier, + const std::shared_ptr& memory_pool) const { + return MakePathFactory(root_path_, arrow_schema, schema_->PartitionKeys(), options, + file_format_identifier, memory_pool); +} + +Result> FileStore::NewSnapshotManager( + const CoreOptions& options, const std::shared_ptr& catalog, + const std::optional& catalog_identifier) const { + auto snapshot_manager = + std::make_shared(options.GetFileSystem(), root_path_, branch_); + if (VersionManagedCatalog* versioned = AsVersionManaged(catalog)) { + if (!catalog_identifier) { + return Status::Invalid("a version-managed catalog requires a table identifier"); + } + Identifier identifier = catalog_identifier.value(); + snapshot_manager->SetSnapshotLoader( + [catalog, versioned, identifier]() -> Result> { + return versioned->LoadSnapshot(identifier); + }); + } + return snapshot_manager; +} + +Result> FileStore::MakePathFactory( + const std::string& root_path, const std::shared_ptr& arrow_schema, + const std::vector& partition_keys, const CoreOptions& options, + const std::string& file_format_identifier, const std::shared_ptr& memory_pool) { + PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, options.CreateExternalPaths()); + PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, + options.CreateGlobalIndexExternalPath()); + return FileStorePathFactory::Create( + root_path, arrow_schema, partition_keys, options.GetPartitionDefaultName(), + file_format_identifier, options.DataFilePrefix(), options.LegacyPartitionNameEnabled(), + external_paths, global_index_external_path, options.IndexFileInDataFileDir(), memory_pool); +} + +} // namespace paimon diff --git a/src/paimon/core/table/file_store.h b/src/paimon/core/table/file_store.h new file mode 100644 index 000000000..ec3bc155c --- /dev/null +++ b/src/paimon/core/table/file_store.h @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "paimon/catalog/identifier.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/utils/branch_manager.h" +#include "paimon/core/utils/file_store_path_factory.h" +#include "paimon/core/utils/snapshot_manager.h" +#include "paimon/result.h" +#include "paimon/type_fwd.h" + +namespace arrow { +class Schema; +} // namespace arrow + +namespace paimon { + +class Catalog; +class CoreOptions; +class TableSchema; + +/// The storage a managed table is served through: the file system, root path, schema and branch +/// it was loaded with, and the schema and snapshot managers that read its root path on that +/// branch - the state Java Paimon's `AbstractFileStore` carries. `FileStoreTable` hands the +/// state it was loaded with to one of these and asks it back for the pieces every operator +/// opens with. +/// +/// The factory methods build a fresh instance on every call, never a cached one: a path factory +/// names each file it builds, and a snapshot manager a write or a commit arms with a catalog +/// loader is not one a scan should be served through afterwards. +class FileStore { + public: + /// @param file_system File system the table's data is accessed through. + /// @param root_path Root path of the table in the file system. + /// @param schema The schema the table was loaded with, whose partition keys name the + /// partitions the paths below are built for. + /// @param branch The branch the store serves, normalized here. + /// @param schema_manager A schema manager to carry instead of building one - the one a + /// context dispatch read the schema through, whose cache the operators then serve. + /// @param snapshot_manager A snapshot manager to carry instead of building one. + /// A null manager is built over `root_path` on `branch`, so the operators always read + /// through the managers the store carries. + FileStore(const std::shared_ptr& file_system, const std::string& root_path, + const std::shared_ptr& schema, const std::string& branch, + const std::shared_ptr& schema_manager = nullptr, + const std::shared_ptr& snapshot_manager = nullptr); + + /// The branch this store serves, as resolved when it was built. + const std::string& branch() const { + return branch_; + } + + /// The schema manager the operators read through. It carries the schema cache of `root_path` + /// on `branch()`. + const std::shared_ptr& schema_manager() const { + return schema_manager_; + } + + /// The snapshot manager a scan reads through. A write or a commit builds its own: see + /// `NewSnapshotManager`. + const std::shared_ptr& snapshot_manager() const { + return snapshot_manager_; + } + + /// The path factory an operator builds its metadata paths through: over this store's root + /// path and the partition keys of its schema. The file format identifier is the caller's to + /// name: a write names the format its level-0 files are written in, which the options may + /// set per level, while a scan, a read and a commit name the table's format. + Result> NewPathFactory( + const std::shared_ptr& arrow_schema, const CoreOptions& options, + const std::string& file_format_identifier, + const std::shared_ptr& memory_pool) const; + + /// The snapshot manager a write or a commit reads snapshots through, over this store's root + /// path and branch. The file system is the caller's to name, from the options its operator + /// serves: a table opened without one has its options build the resolving file system. A + /// catalog that version-manages the table hands its snapshots out itself, so its loader is + /// attached, the catalog captured to keep the versioned interface - and the loader with it + /// - alive. Such a catalog names the table by an identifier, which the context must have + /// supplied. + Result> NewSnapshotManager( + const CoreOptions& options, const std::shared_ptr& catalog, + const std::optional& catalog_identifier) const; + + /// The assembly the factory methods above share: a path factory over a root path and + /// partition keys named by the caller rather than the store's own, which the read path + /// needs - the schema it serves may be a fallback branch's, not the table's. + static Result> MakePathFactory( + const std::string& root_path, const std::shared_ptr& arrow_schema, + const std::vector& partition_keys, const CoreOptions& options, + const std::string& file_format_identifier, const std::shared_ptr& memory_pool); + + private: + std::shared_ptr file_system_; + std::string root_path_; + std::shared_ptr schema_; + std::string branch_; + std::shared_ptr schema_manager_; + std::shared_ptr snapshot_manager_; +}; + +} // namespace paimon diff --git a/src/paimon/core/table/file_store_table.h b/src/paimon/core/table/file_store_table.h index 428513e16..51c7c0b95 100644 --- a/src/paimon/core/table/file_store_table.h +++ b/src/paimon/core/table/file_store_table.h @@ -19,11 +19,16 @@ #pragma once +#include #include #include #include "paimon/catalog/identifier.h" #include "paimon/catalog/table.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/table/file_store.h" +#include "paimon/core/utils/branch_manager.h" +#include "paimon/core/utils/snapshot_manager.h" #include "paimon/result.h" #include "paimon/type_fwd.h" @@ -36,7 +41,8 @@ class Schema; /// the other kind of `Table`. /// /// Its `NewScan`/`NewRead`/`NewWrite`/`NewCommit` build the operators here, from the context plus -/// the schema the table was constructed with, so they serve the table as it was loaded. The +/// the schema, file system, root path, branch and managers the table was constructed with, so +/// they serve the table as it was loaded. The /// context-only entry points - `TableScan::Create()`, `TableRead::Create()`, /// `FileStoreWrite::Create()` and `FileStoreCommit::Create()` - load the table the context names /// and hand the context to that table's `NewXxx`, which is how a context built without a table @@ -48,10 +54,39 @@ class FileStoreTable : public Table { /// @param file_system File system the table's data is accessed through. /// @param root_path Root path of the table in the file system. /// @param uuid Metastore UUID, or empty if unavailable. + /// @param branch The branch the table serves, which the managers below read and write on. + /// @param schema_manager A schema manager to carry instead of building one - the one a + /// context dispatch read the schema through, whose cache the operators then serve. + /// @param snapshot_manager A snapshot manager to carry instead of building one, the way + /// `Copy` hands its table's over so both share one. + /// A null manager is built over `root_path` on `branch`, so the operators always read + /// through the managers the table carries. FileStoreTable(const std::shared_ptr& file_system, const std::string& root_path, const std::shared_ptr& schema, const Identifier& identifier, - const std::string& uuid = "") - : Table(file_system, root_path, schema, identifier, uuid) {} + const std::string& uuid = "", + const std::string& branch = BranchManager::DEFAULT_MAIN_BRANCH, + const std::shared_ptr& schema_manager = nullptr, + const std::shared_ptr& snapshot_manager = nullptr); + + /// The branch this table serves, as resolved when the table was built: the schema the table + /// was loaded with came from it, and its managers point at it. + const std::string& CurrentBranch() const { + return store_->branch(); + } + + /// The schema manager the operators read through. It carries the schema cache of `root_path` + /// on `CurrentBranch()`, which every operator serves rather than each reading the schema + /// files again. + const std::shared_ptr& GetSchemaManager() const { + return store_->schema_manager(); + } + + /// The snapshot manager a scan reads through. A write or a commit builds its own: it arms + /// the manager with a catalog snapshot loader, which a shared one would keep serving to + /// scans afterwards. + const std::shared_ptr& GetSnapshotManager() const { + return store_->snapshot_manager(); + } std::string Name() const override { return identifier_.GetObjectName(); @@ -61,6 +96,19 @@ class FileStoreTable : public Table { return identifier_.GetFullName(); } + /// Copies the table with `dynamic_options` on top of the ones its schema stored, the + /// precedence `Copy` promises. The file system, root path and identifier stay as they are: + /// they name this table, which an option does not rename. + Result> Copy( + const std::map& dynamic_options) const override; + + /// Returns the table of `branch`, as Java Paimon's `switchToBranch` does. The branch switched + /// to is served as it is: its schema is read from its own directory, so the dynamic options + /// of this table are not carried over. Switching to the branch already served hands back a + /// copy of this table, schema and managers shared, and a branch with no schema does not + /// exist. + Result> SwitchToBranch(const std::string& branch) const; + Result> NewScan(std::unique_ptr context) const override; Result> NewRead(std::unique_ptr context) const override; @@ -70,6 +118,11 @@ class FileStoreTable : public Table { Result> NewCommit( std::unique_ptr context) const override; + + private: + /// The storage this table serves: the file system, root path, schema, branch and managers it + /// was loaded with, and the factories that build the pieces every operator opens with. + std::shared_ptr store_; }; } // namespace paimon diff --git a/src/paimon/core/table/format/format_table.cpp b/src/paimon/core/table/format/format_table.cpp index 53d53b967..6f2ac16a4 100644 --- a/src/paimon/core/table/format/format_table.cpp +++ b/src/paimon/core/table/format/format_table.cpp @@ -192,24 +192,26 @@ Result> FormatTable::Create( core_options.FormatTablePartitionOnlyValueInPath(), location_carries_paimon_metadata)); } -Result> FormatTable::Copy( - const std::shared_ptr& table, - const std::map& dynamic_options) { - if (table == nullptr) { - return Status::Invalid("format table requires a table to copy"); - } +Result> FormatTable::CopySelf( + const std::map& dynamic_options) const { if (dynamic_options.empty()) { - return table; + return SharedSelf(); } // The options this table already carries go back in as dynamic ones, so an option an earlier // call set is not lost to the schema's own value underneath it. - std::map options = table->options_; + std::map options = options_; for (const auto& [key, value] : dynamic_options) { options[key] = value; } - return Create(table->file_system_, table->root_path_, table->identifier_, - std::static_pointer_cast(table->schema_), - table->location_carries_paimon_metadata_, options); + return Create(file_system_, root_path_, identifier_, + std::static_pointer_cast(schema_), location_carries_paimon_metadata_, + options); +} + +Result> FormatTable::Copy( + const std::map& dynamic_options) const { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr copied, CopySelf(dynamic_options)); + return std::shared_ptr
(std::move(copied)); } const std::vector& FormatTable::PartitionKeys() const { @@ -233,7 +235,7 @@ Result> FormatTable::NewScan( return Status::Invalid("executor is null pointer"); } PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, - FormatTable::Copy(SharedSelf(), shared_context->GetOptions())); + CopySelf(shared_context->GetOptions())); // Anything a `ScanContext` carries that a format table cannot honour is refused rather than // silently dropped. if (shared_context->IsStreamingMode()) { @@ -294,7 +296,7 @@ Result> FormatTable::NewRead( return Status::NotImplemented("FullTextMatch predicate execution is not implemented yet"); } PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, - FormatTable::Copy(SharedSelf(), read_context->GetOptions())); + CopySelf(read_context->GetOptions())); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, FormatTableRead::Create(table, read_context)); return std::unique_ptr(std::move(read)); @@ -311,8 +313,7 @@ Result> FormatTable::NewWrite( if (context->GetExecutor() == nullptr) { return Status::Invalid("executor is null pointer"); } - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, - FormatTable::Copy(SharedSelf(), context->GetOptions())); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, CopySelf(context->GetOptions())); if (context->GetCatalog() != nullptr) { return Status::Invalid( "WithCatalog() requires a native table; use WriteContextBuilder(FormatTable) for a " @@ -354,7 +355,7 @@ Result> FormatTable::NewCommit( return Status::Invalid("executor is null pointer"); } PAIMON_ASSIGN_OR_RAISE(std::shared_ptr given_table, - FormatTable::Copy(SharedSelf(), context->GetOptions())); + CopySelf(context->GetOptions())); if (!context->IgnoreEmptyCommit()) { return Status::NotImplemented( "a format table cannot record an empty commit: keeping one means writing a " diff --git a/src/paimon/core/table/format/format_table_test.cpp b/src/paimon/core/table/format/format_table_test.cpp index 2430ffbda..2dac6c102 100644 --- a/src/paimon/core/table/format/format_table_test.cpp +++ b/src/paimon/core/table/format/format_table_test.cpp @@ -3071,7 +3071,7 @@ TEST(FormatTableTest, TestABranchNamedThroughOptionsIsRefusedAtEveryEntryPoint) ASSERT_OK(WriteAndCommit(table, std::move(batch))); const std::map branch = {{Options::BRANCH, "dev"}}; - ASSERT_NOK_WITH_MSG(FormatTable::Copy(table, branch), "has no branches"); + ASSERT_NOK_WITH_MSG(table->Copy(branch), "has no branches"); ASSERT_NOK_WITH_MSG( FormatTable::Create(dir->GetFileSystem(), dir->Str(), Identifier("db", "tbl"), branch), "has no branches"); @@ -3108,10 +3108,32 @@ TEST(FormatTableTest, TestABranchNamedThroughOptionsIsRefusedAtEveryEntryPoint) // The branch that names no branch at all is the main one, and is left alone: refusing it // would refuse the default every context carries. for (const char* main_branch : {"main", ""}) { - ASSERT_OK(FormatTable::Copy(table, {{Options::BRANCH, main_branch}})) << main_branch; + ASSERT_OK(table->Copy({{Options::BRANCH, main_branch}})) << main_branch; } } +TEST(FormatTableTest, TestATableCopiesItselfWithTheOptionsGivenAtTheCall) { + std::unique_ptr dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN(std::shared_ptr table, + CreateTable(dir->GetFileSystem(), dir->Str(), {"dt"})); + + // The copy overlays what the call gives on the options the schema stored, the precedence + // `Copy()` promises: a probe option lands, `type` - structural - does not. + ASSERT_OK_AND_ASSIGN(std::shared_ptr
copied, + table->Copy({{Options::TYPE, "table"}, {"probe", "on-top"}})); + std::shared_ptr copied_table = std::static_pointer_cast(copied); + EXPECT_EQ(copied_table->Options().at("probe"), "on-top"); + EXPECT_EQ(copied_table->Options().at(Options::TYPE), "format-table"); + EXPECT_EQ(copied_table->FullName(), table->FullName()); + EXPECT_EQ(copied_table->Location(), table->Location()); + EXPECT_EQ(copied_table->GetFileSystem(), table->GetFileSystem()); + + // The refusal `Copy(table, ...)` reports is the one the table itself reports: the copy goes + // through the same entry point, not a shortcut around its checks. + ASSERT_NOK_WITH_MSG(table->Copy({{Options::BRANCH, "dev"}}), "has no branches"); +} + TEST(FormatTableTest, TestAFormatTableSchemaOnABranchIsRefusedRatherThanServedFromTheTableRoot) { // A context that names a path takes its branch from `WithBranch()`, and the schema of that // branch is what the entry point dispatches on. A format table's data is the files under the diff --git a/src/paimon/core/table/source/table_read_test.cpp b/src/paimon/core/table/source/table_read_test.cpp index ead31b17a..3d580d64d 100644 --- a/src/paimon/core/table/source/table_read_test.cpp +++ b/src/paimon/core/table/source/table_read_test.cpp @@ -250,25 +250,41 @@ TEST(TableReadTest, TestCreateAppendOnlyTableRead) { TEST(TableReadTest, TestMergeOptions) { std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; - ReadContextBuilder context_builder(path); - context_builder.SetReadFieldNames({"f0", "f1", "f2", "f3"}); - context_builder.AddOption("read.batch-size", "2"); - context_builder.AddOption("orc.read.enable-lazy-decoding", "true"); - context_builder.AddOption("bucket", "10"); - context_builder.AddOption("bucket-key", "f3"); - ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); - auto append_only_table_read = dynamic_cast(table_read.get()); - ASSERT_TRUE(append_only_table_read); - auto* typed_split_read = - dynamic_cast(append_only_table_read->split_reads_[0].get()); - ASSERT_TRUE(typed_split_read); - const auto& core_options = typed_split_read->options_; - std::map expected_options = { - {"read.batch-size", "2"}, {"orc.read.enable-lazy-decoding", "true"}, - {"bucket", "10"}, {"bucket-key", "f3"}, - {"manifest.format", "orc"}, {"file.format", "orc"}}; - ASSERT_EQ(expected_options, core_options.ToMap()); + { + // The options the table was loaded with stand as they are, and a runtime setting the + // context adds on top - the table was loaded with neither of these keys - is served as + // given. + ReadContextBuilder context_builder(path); + context_builder.SetReadFieldNames({"f0", "f1", "f2", "f3"}); + context_builder.AddOption("read.batch-size", "2"); + context_builder.AddOption("orc.read.enable-lazy-decoding", "true"); + ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + auto append_only_table_read = dynamic_cast(table_read.get()); + ASSERT_TRUE(append_only_table_read); + auto* typed_split_read = + dynamic_cast(append_only_table_read->split_reads_[0].get()); + ASSERT_TRUE(typed_split_read); + const auto& core_options = typed_split_read->options_; + std::map expected_options = { + {"read.batch-size", "2"}, + {"orc.read.enable-lazy-decoding", "true"}, + {"bucket", "2"}, + {"bucket-key", "f2"}, + {"manifest.format", "orc"}, + {"file.format", "orc"}}; + ASSERT_EQ(expected_options, core_options.ToMap()); + } + { + // An option that contradicts the table is refused: the read serves the options the table + // was loaded with, and a caller that means to change them copies the table. + ReadContextBuilder context_builder(path); + context_builder.SetReadFieldNames({"f0", "f1", "f2", "f3"}); + context_builder.AddOption("bucket", "10"); + ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); + ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), + "option 'bucket' is '10' in the context but '2' on the table"); + } } TEST(TableReadTest, TestDependencyReadWithoutCatalogUsesReadOptions) { diff --git a/src/paimon/core/table/system/system_table.cpp b/src/paimon/core/table/system/system_table.cpp index dd00b7bcd..cf4864071 100644 --- a/src/paimon/core/table/system/system_table.cpp +++ b/src/paimon/core/table/system/system_table.cpp @@ -191,6 +191,11 @@ Status SystemTable::AdoptOwnSchema() { return Status::OK(); } +Result> SystemTable::Copy( + const std::map& /*dynamic_options*/) const { + return Status::NotImplemented(Name(), " system table is read-only: copying is not supported"); +} + Result> SystemTable::NewWrite( std::unique_ptr /*context*/) const { return Status::NotImplemented(Name(), " system table is read-only: writing is not supported"); diff --git a/src/paimon/core/table/system/system_table.h b/src/paimon/core/table/system/system_table.h index 2e4dff55d..96eead153 100644 --- a/src/paimon/core/table/system/system_table.h +++ b/src/paimon/core/table/system/system_table.h @@ -66,6 +66,12 @@ class SystemTable : public Table, public std::enable_shared_from_this> ArrowSchema() const = 0; + /// A system table has no options of its own to override: its schema is the read-only view it + /// exposes, which is not reconfigured by a copy. The data table underneath is copied through + /// its own `Table`. + Result> Copy( + const std::map& dynamic_options) const override; + /// A system table is read-only: writing it or committing for it is refused rather than /// routed to the data table underneath. Result> NewWrite( diff --git a/src/paimon/core/table/system/system_table_test.cpp b/src/paimon/core/table/system/system_table_test.cpp index 49cf6838f..0a9f85a2e 100644 --- a/src/paimon/core/table/system/system_table_test.cpp +++ b/src/paimon/core/table/system/system_table_test.cpp @@ -227,6 +227,20 @@ TEST(SystemTableTest, TestReadOptimizedSystemTableRegistration) { ASSERT_EQ(arrow_schema->field(1)->type()->id(), arrow::Type::INT32); } +TEST(SystemTableTest, TestCopyIsRefused) { + // A system table is a read-only view over the data table underneath it, and the schema it + // exposes is not one a copy reconfigures. The data table is copied through its own `Table`. + std::map options = {{Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "orc"}}; + ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, + CreateTableSchemaForTest(options)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr system_table, + SystemTableLoader::Load(ReadOptimizedSystemTable::kName, /*fs=*/nullptr, + "/tmp/table", table_schema, + /*dynamic_options=*/{})); + ASSERT_NOK_WITH_MSG(system_table->Copy({{"probe", "on-top"}}), "copying is not supported"); +} + TEST(SystemTableTest, TestReadOptimizedSystemTablePathParsing) { ASSERT_OK_AND_ASSIGN(std::optional parsed, SystemTableLoader::TryParsePath("/tmp/db.db/t$branch_audit$ro")); diff --git a/src/paimon/core/table/table.cpp b/src/paimon/core/table/table.cpp index bff18db29..33ff27d78 100644 --- a/src/paimon/core/table/table.cpp +++ b/src/paimon/core/table/table.cpp @@ -77,6 +77,7 @@ #include "paimon/core/schema/schema_validation.h" #include "paimon/core/schema/table_schema.h" #include "paimon/core/table/bucket_mode.h" +#include "paimon/core/table/file_store.h" #include "paimon/core/table/file_store_table.h" #include "paimon/core/table/source/abstract_table_scan.h" #include "paimon/core/table/source/append_only_split_generator.h" @@ -218,6 +219,7 @@ Result HasHistoricalPrimaryKeyBTreeDefinition( Result> TableOfLoadedSchema( const std::shared_ptr& file_system, const std::string& path, const std::string& branch, const std::map& options, + const std::shared_ptr& schema_manager, const std::shared_ptr& latest_schema, const std::optional& identifier, const std::string& uuid = "") { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, @@ -230,7 +232,72 @@ Result> TableOfLoadedSchema( Identifier table_identifier = identifier.value_or(Identifier(Identifier::kUnknownDatabase, PathUtil::GetName(path))); return std::make_shared(file_system, path, latest_schema, table_identifier, - uuid); + uuid, branch, schema_manager); +} + +/// The managed table a context names behind a path: the schema is read through a manager of the +/// context's branch - handed over instead when the caller already read it - and what the schema +/// says is there decides the table `TableOfLoadedSchema` hands back. +/// +/// @param schema_json The schema of the table, when the caller already read it, or null when it +/// has to be read here. It is honoured on the main branch only: a branch publishes its own +/// under `branch/branch-`, which is where the read below goes. +Result> ManagedTableOf(const std::shared_ptr& file_system, + const std::string& path, const std::string& branch, + const std::map& options, + const std::optional& schema_json, + const std::optional& identifier, + const std::string& uuid = "") { + auto schema_manager = std::make_shared(file_system, path, branch); + std::shared_ptr latest_schema; + if (branch == BranchManager::DEFAULT_MAIN_BRANCH && schema_json) { + // Handing the schema over only saves reading it. A context always names a table path and + // this library writes the metadata under it, so `schema` and `branch` stay metadata. + PAIMON_ASSIGN_OR_RAISE(latest_schema, TableSchema::CreateFromJson(schema_json.value())); + } else { + PAIMON_ASSIGN_OR_RAISE(std::optional> latest, + schema_manager->Latest()); + if (latest) { + latest_schema = *latest; + } + } + return TableOfLoadedSchema(file_system, path, branch, options, schema_manager, latest_schema, + identifier, uuid); +} + +/// The identifier a catalog knows the table by, on `branch`: a non-main branch is folded into +/// the identifier, so the catalog is asked for the table on that branch rather than the main one. +Result> MakeBranchCatalogIdentifier( + const std::optional& identifier, const std::string& branch) { + if (!identifier) { + return std::optional(); + } + PAIMON_ASSIGN_OR_RAISE(Identifier branch_identifier, + CatalogUtils::BranchIdentifier(identifier.value(), branch)); + return std::optional(std::move(branch_identifier)); +} + +/// The schema a catalog may hold for the table, read from it on the main branch only and null +/// otherwise. A branch publishes its own schema under `branch/branch-`, which is where +/// `ManagedTableOf` reads instead, so a catalog answering for no branch schema does not fail the +/// `kind` operation to one. `kind` also names the operator the missing-identifier message speaks +/// of. +Result> CatalogMainBranchSchemaJson( + const std::shared_ptr& catalog, const std::optional& catalog_identifier, + const std::string& branch, const char* kind) { + if (catalog == nullptr) { + return std::optional(); + } + if (!catalog_identifier) { + return Status::Invalid(fmt::format("a catalog {} requires a table identifier", kind)); + } + if (BranchManager::IsMainBranch(branch)) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, + catalog->LoadTableSchema(catalog_identifier.value())); + PAIMON_ASSIGN_OR_RAISE(std::string schema_json, schema->GetJsonSchema()); + return std::optional(std::move(schema_json)); + } + return std::optional(); } /// The table a scan context names, read from what the context carries: a format table the @@ -255,22 +322,8 @@ Result> TableOf(const ScanContext& ctx) { // below. One `TableScan` interface serves both, as Java Paimon serves both through one // `ReadBuilder`. const std::string branch = BranchManager::NormalizeBranch(tmp_options.GetBranch()); - std::shared_ptr latest_schema; - if (branch == BranchManager::DEFAULT_MAIN_BRANCH && ctx.GetSpecificTableSchema()) { - // Handing the schema over only saves reading it. A context always names a table path and - // this library writes the metadata under it, so `schema` and `branch` stay metadata. - PAIMON_ASSIGN_OR_RAISE(latest_schema, - TableSchema::CreateFromJson(ctx.GetSpecificTableSchema().value())); - } else { - SchemaManager schema_manager(tmp_options.GetFileSystem(), ctx.GetPath(), branch); - PAIMON_ASSIGN_OR_RAISE(std::optional> latest, - schema_manager.Latest()); - if (latest) { - latest_schema = *latest; - } - } - return TableOfLoadedSchema(tmp_options.GetFileSystem(), ctx.GetPath(), branch, ctx.GetOptions(), - latest_schema, ctx.GetIdentifier()); + return ManagedTableOf(tmp_options.GetFileSystem(), ctx.GetPath(), branch, ctx.GetOptions(), + ctx.GetSpecificTableSchema(), ctx.GetIdentifier()); } /// The table a read context names, read from what the context carries: a format table the @@ -305,22 +358,8 @@ Result> TableOf(const ReadContext& ctx) { // which files there are and how a row is put back together is its own. How a file is opened // is not: both paths go through `DataFileReaderFactory`. const std::string branch = ctx.GetBranch(); - std::shared_ptr latest_schema; - if (branch == BranchManager::DEFAULT_MAIN_BRANCH && ctx.GetSpecificTableSchema()) { - // Handing the schema over only saves reading it. A context always names a table path and - // this library writes the metadata under it, so `schema` and `branch` stay metadata. - PAIMON_ASSIGN_OR_RAISE(latest_schema, - TableSchema::CreateFromJson(ctx.GetSpecificTableSchema().value())); - } else { - SchemaManager schema_manager(tmp_core_options.GetFileSystem(), ctx.GetPath(), branch); - PAIMON_ASSIGN_OR_RAISE(std::optional> latest, - schema_manager.Latest()); - if (latest) { - latest_schema = *latest; - } - } - return TableOfLoadedSchema(tmp_core_options.GetFileSystem(), ctx.GetPath(), branch, - ctx.GetOptions(), latest_schema, ctx.GetIdentifier()); + return ManagedTableOf(tmp_core_options.GetFileSystem(), ctx.GetPath(), branch, ctx.GetOptions(), + ctx.GetSpecificTableSchema(), ctx.GetIdentifier()); } /// The table a write context names, read from what the context carries: a format table the @@ -347,44 +386,18 @@ Result> TableOf(const WriteContext& ctx) { CoreOptions::FromMap(ctx.GetOptions(), specific_fs, ctx.GetFileSystemSchemeToIdentifierMap())); std::string branch = ctx.GetBranch(); - std::optional catalog_identifier; - if (ctx.GetIdentifier()) { - PAIMON_ASSIGN_OR_RAISE(Identifier branch_identifier, - CatalogUtils::BranchIdentifier(ctx.GetIdentifier().value(), branch)); - catalog_identifier.emplace(std::move(branch_identifier)); - } + PAIMON_ASSIGN_OR_RAISE(std::optional catalog_identifier, + MakeBranchCatalogIdentifier(ctx.GetIdentifier(), branch)); // Only the main branch may have a current schema the catalog alone holds. A branch publishes // its own under `branch/branch-`, which is where the load below and a read of the branch // both read it, so a catalog answering for no branch schema does not fail a write to one. - std::optional catalog_table_schema; - if (ctx.GetCatalog() != nullptr) { - if (!catalog_identifier) { - return Status::Invalid("a catalog write requires a table identifier"); - } - if (BranchManager::IsMainBranch(branch)) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, - ctx.GetCatalog()->LoadTableSchema(catalog_identifier.value())); - PAIMON_ASSIGN_OR_RAISE(catalog_table_schema, schema->GetJsonSchema()); - } - } + PAIMON_ASSIGN_OR_RAISE( + std::optional catalog_table_schema, + CatalogMainBranchSchemaJson(ctx.GetCatalog(), catalog_identifier, branch, "write")); // A format table writes plain data files into a directory, so it never reaches the manifest // path below. - SchemaManager schema_manager(tmp_options.GetFileSystem(), ctx.GetRootPath(), branch); - std::shared_ptr latest_schema; - if (branch == BranchManager::DEFAULT_MAIN_BRANCH && catalog_table_schema) { - // Handing the schema over only saves reading it. A context always names a table path and - // this library writes the metadata under it, so `schema` and `branch` stay metadata. - PAIMON_ASSIGN_OR_RAISE(latest_schema, - TableSchema::CreateFromJson(catalog_table_schema.value())); - } else { - PAIMON_ASSIGN_OR_RAISE(std::optional> latest, - schema_manager.Latest()); - if (latest) { - latest_schema = *latest; - } - } - return TableOfLoadedSchema(tmp_options.GetFileSystem(), ctx.GetRootPath(), branch, - ctx.GetOptions(), latest_schema, ctx.GetIdentifier()); + return ManagedTableOf(tmp_options.GetFileSystem(), ctx.GetRootPath(), branch, ctx.GetOptions(), + catalog_table_schema, ctx.GetIdentifier()); } /// The table a commit context names, read from what the context carries: a format table the @@ -414,47 +427,83 @@ Result> TableOf(const CommitContext& ctx) { PAIMON_ASSIGN_OR_RAISE(std::string branch, BranchManager::ResolveBranch( ctx.GetIdentifier(), ctx.GetOptions(), /*explicit_branch=*/std::nullopt, "commit")); - std::optional catalog_identifier; - if (ctx.GetIdentifier()) { - PAIMON_ASSIGN_OR_RAISE(Identifier branch_identifier, - CatalogUtils::BranchIdentifier(ctx.GetIdentifier().value(), branch)); - catalog_identifier.emplace(std::move(branch_identifier)); - } + PAIMON_ASSIGN_OR_RAISE(std::optional catalog_identifier, + MakeBranchCatalogIdentifier(ctx.GetIdentifier(), branch)); // Catalog-managed schemas may be absent from the table directory, but only on the main branch: // a branch publishes its own under `branch/branch-`, which is where the load below and a // read of the branch both read it, so a catalog answering for no branch schema does not fail // a commit to one. - std::optional catalog_table_schema; - if (ctx.GetCatalog() != nullptr) { - if (!catalog_identifier) { - return Status::Invalid("a catalog commit requires a table identifier"); - } - if (BranchManager::IsMainBranch(branch)) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr loaded_schema, - ctx.GetCatalog()->LoadTableSchema(catalog_identifier.value())); - PAIMON_ASSIGN_OR_RAISE(std::string schema_json, loaded_schema->GetJsonSchema()); - catalog_table_schema = std::move(schema_json); - } - } + PAIMON_ASSIGN_OR_RAISE( + std::optional catalog_table_schema, + CatalogMainBranchSchemaJson(ctx.GetCatalog(), catalog_identifier, branch, "commit")); // A format table commits by renaming files into place, so it never reaches the snapshot path // below. The managed path here reads the commit's branch, so this reads the same one: the two // must not dispatch on different schemas. - SchemaManager schema_manager(tmp_options.GetFileSystem(), root_path, branch); - std::shared_ptr latest_schema; - if (branch == BranchManager::DEFAULT_MAIN_BRANCH && catalog_table_schema) { - // Handing the schema over only saves reading it. A context always names a table path and - // this library writes the metadata under it, so `schema` and `branch` stay metadata. - PAIMON_ASSIGN_OR_RAISE(latest_schema, - TableSchema::CreateFromJson(catalog_table_schema.value())); - } else { - PAIMON_ASSIGN_OR_RAISE(std::optional> latest, - schema_manager.Latest()); - if (latest) { - latest_schema = *latest; + return ManagedTableOf(tmp_options.GetFileSystem(), root_path, branch, ctx.GetOptions(), + catalog_table_schema, ctx.GetIdentifier(), ctx.GetTableId().value_or("")); +} + +/// The options a context carries against the ones the table was loaded with. A table serves the +/// options it was loaded with, so an option the context contradicts is refused rather than +/// honoured - a caller that means to change them copies the table. A key the table was not loaded +/// with is no contradiction: it names a runtime setting - a scan mode, a batch size - which is +/// none of the table's business, and is served as given. +Status CheckOptionsAgreeWithTable(const TableSchema& table_schema, + const std::map& context_options) { + const std::map& table_options = table_schema.Options(); + for (const auto& [key, value] : context_options) { + auto table_option = table_options.find(key); + if (table_option != table_options.end() && table_option->second != value) { + return Status::Invalid( + fmt::format("option '{}' is '{}' in the context but '{}' on the table", key, value, + table_option->second)); } } - return TableOfLoadedSchema(tmp_options.GetFileSystem(), root_path, branch, ctx.GetOptions(), - latest_schema, ctx.GetIdentifier(), ctx.GetTableId().value_or("")); + return Status::OK(); +} + +/// The checks every operator entry point opens with: a context, a memory pool and an executor +/// must all be there for the operator to be built at all. `kind` names the operator the first +/// message speaks of, which is all that sets the entry points' messages apart. +template +Status CheckOperatorContext(const ContextPointer& context, const char* kind) { + if (context == nullptr) { + return Status::Invalid(fmt::format("{} context is null pointer", kind)); + } + if (context->GetMemoryPool() == nullptr) { + return Status::Invalid("memory pool is null pointer"); + } + if (context->GetExecutor() == nullptr) { + return Status::Invalid("executor is null pointer"); + } + return Status::OK(); +} + +/// The schema the table was constructed with, which a write or a commit serves as the table was +/// loaded rather than reading the schema again. +Result> LatestTableSchemaOfBranch( + const std::shared_ptr& schema, const std::string& branch) { + std::shared_ptr latest_schema = std::static_pointer_cast(schema); + if (latest_schema == nullptr) { + return Status::Invalid(fmt::format("not found latest schema in branch {}", branch)); + } + return latest_schema; +} + +/// The options a write or a commit serves: the ones the table was loaded with - an option the +/// context contradicts is refused by `CheckOptionsAgreeWithTable` above - the ones the context +/// adds on top, and the table's branch, which the metadata paths below are built for: what reads +/// through these options, whose cache keys are scoped by it, has to read the same one. +Result> MergedOperatorOptions( + const TableSchema& table_schema, const std::map& context_options, + const std::string& branch) { + PAIMON_RETURN_NOT_OK(CheckOptionsAgreeWithTable(table_schema, context_options)); + auto opts = table_schema.Options(); + for (const auto& [key, value] : context_options) { + opts[key] = value; + } + opts[Options::BRANCH] = branch; + return opts; } } // namespace @@ -468,60 +517,97 @@ Table::Table(const std::shared_ptr& file_system, const std::string& identifier_(identifier), uuid_(uuid) {} +FileStoreTable::FileStoreTable(const std::shared_ptr& file_system, + const std::string& root_path, const std::shared_ptr& schema, + const Identifier& identifier, const std::string& uuid, + const std::string& branch, + const std::shared_ptr& schema_manager, + const std::shared_ptr& snapshot_manager) + : Table(file_system, root_path, schema, identifier, uuid), + store_(std::make_shared(file_system, root_path, + std::static_pointer_cast(schema), branch, + schema_manager, snapshot_manager)) {} + +Result> FileStoreTable::Copy( + const std::map& dynamic_options) const { + std::shared_ptr loaded_schema = std::static_pointer_cast(schema_); + if (loaded_schema == nullptr) { + return Status::Invalid(fmt::format("not found latest schema of {}", FullName())); + } + // `type` is structural and read from the schema alone, the promise `Copy` makes: an option + // given at the call must not restyle the table. + std::map overridable = dynamic_options; + overridable.erase(Options::TYPE); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merged_schema, + loaded_schema->MergeOptions(overridable)); + // The managers are handed over, not rebuilt: the copy serves the same root path and branch, + // so its operators read the schema cache this table already warmed. + return std::shared_ptr
(std::make_shared( + file_system_, root_path_, std::shared_ptr(std::move(merged_schema)), + identifier_, CatalogUuid().value_or(""), CurrentBranch(), GetSchemaManager(), + GetSnapshotManager())); +} + +Result> FileStoreTable::SwitchToBranch(const std::string& branch) const { + const std::string target_branch = BranchManager::NormalizeBranch(branch); + if (target_branch == CurrentBranch()) { + // The table already serves this branch: a copy of it - schema and managers shared - is + // the same table to the caller. + return std::shared_ptr
(std::make_shared( + file_system_, root_path_, schema_, identifier_, CatalogUuid().value_or(""), + CurrentBranch(), GetSchemaManager(), GetSnapshotManager())); + } + // The schema of the branch switched to, read from its own directory, which a branch without + // one does not have. + auto schema_manager = std::make_shared(file_system_, root_path_, target_branch); + PAIMON_ASSIGN_OR_RAISE(std::optional> latest, + schema_manager->Latest()); + if (!latest) { + return Status::Invalid(fmt::format("branch {} does not exist", target_branch)); + } + // A branch other than main is named in the options the schema carries - the way a read + // context names it - so what reads through them serves the branch the managers point at. + std::map overridable; + if (target_branch != BranchManager::DEFAULT_MAIN_BRANCH) { + overridable[Options::BRANCH] = target_branch; + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr branch_schema, + (*latest)->MergeOptions(overridable)); + return std::shared_ptr
(std::make_shared( + file_system_, root_path_, std::shared_ptr(std::move(branch_schema)), + identifier_, CatalogUuid().value_or(""), target_branch, schema_manager)); +} + Result> FileStoreTable::NewWrite( std::unique_ptr ctx) const { - if (ctx == nullptr) { - return Status::Invalid("write context is null pointer"); - } - if (ctx->GetMemoryPool() == nullptr) { - return Status::Invalid("memory pool is null pointer"); - } - if (ctx->GetExecutor() == nullptr) { - return Status::Invalid("executor is null pointer"); - } + PAIMON_RETURN_NOT_OK(CheckOperatorContext(ctx, "write")); PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog())); std::shared_ptr specific_fs = ctx->GetSpecificFileSystem(); + // A catalog write names its table; the file system itself comes from the table, which its + // catalog's credentials already opened. if (specific_fs == nullptr && ctx->GetFileSystemSchemeToIdentifierMap().empty() && - ctx->GetCatalog() != nullptr) { - if (!ctx->GetIdentifier()) { - return Status::Invalid("a catalog write requires a table identifier"); - } - // A catalog issuing per-table temporary credentials only hands them out through - // GetTableFileSystem, so the write uses the credentials of this table. - PAIMON_ASSIGN_OR_RAISE(specific_fs, - ctx->GetCatalog()->GetTableFileSystem(ctx->GetIdentifier().value())); + ctx->GetCatalog() != nullptr && !ctx->GetIdentifier()) { + return Status::Invalid("a catalog write requires a table identifier"); } - PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, - CoreOptions::FromMap(ctx->GetOptions(), specific_fs, - ctx->GetFileSystemSchemeToIdentifierMap())); - std::string branch = ctx->GetBranch(); - std::optional catalog_identifier; - if (ctx->GetIdentifier()) { - PAIMON_ASSIGN_OR_RAISE( - Identifier branch_identifier, - CatalogUtils::BranchIdentifier(ctx->GetIdentifier().value(), branch)); - catalog_identifier.emplace(std::move(branch_identifier)); - } - auto schema_manager = - std::make_shared(tmp_options.GetFileSystem(), ctx->GetRootPath(), branch); + // The write goes to the branch the table serves: a table is the branch it was loaded for, + // so the context's own branch is not consulted. + PAIMON_ASSIGN_OR_RAISE(std::optional catalog_identifier, + MakeBranchCatalogIdentifier(ctx->GetIdentifier(), CurrentBranch())); // The schema the table was constructed with, so a write serves the table as it was loaded // rather than reading the schema again. - std::shared_ptr latest_schema = std::static_pointer_cast(schema_); - if (latest_schema == nullptr) { - return Status::Invalid(fmt::format("not found latest schema in branch {}", branch)); - } - const std::shared_ptr& schema = latest_schema; + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, + LatestTableSchemaOfBranch(schema_, CurrentBranch())); auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(schema->Fields()); - auto opts = schema->Options(); - for (const auto& [key, value] : ctx->GetOptions()) { - opts[key] = value; - } - // The metadata paths above are built for this branch, so what reads through these options, - // whose cache keys are scoped by it, has to read the same one. - opts[Options::BRANCH] = branch; + // The options the table was loaded with, with a runtime setting the context adds on top: an + // option the context would have changed is refused rather than honoured, and the branch is + // the table's - the metadata paths below are built for it, so what reads through these + // options, whose cache keys are scoped by it, has to read the same one. + std::map opts; + PAIMON_ASSIGN_OR_RAISE(opts, + MergedOperatorOptions(*schema, ctx->GetOptions(), CurrentBranch())); PAIMON_ASSIGN_OR_RAISE( CoreOptions options, - CoreOptions::FromMap(opts, specific_fs, ctx->GetFileSystemSchemeToIdentifierMap())); + CoreOptions::FromMap(opts, file_system_, ctx->GetFileSystemSchemeToIdentifierMap())); PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*schema, options)); PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateVectorStore(*schema, options)); if (options.GetVectorFileFormat() && !ctx->GetWriteSchema().empty()) { @@ -531,29 +617,14 @@ Result> FileStoreTable::NewWrite( PAIMON_ASSIGN_OR_RAISE(std::shared_ptr partition_schema, FieldMapping::GetPartitionSchema(arrow_schema, schema->PartitionKeys())); - PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, options.CreateExternalPaths()); - PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, - options.CreateGlobalIndexExternalPath()); - PAIMON_ASSIGN_OR_RAISE( std::shared_ptr file_store_path_factory, - FileStorePathFactory::Create(ctx->GetRootPath(), arrow_schema, schema->PartitionKeys(), - options.GetPartitionDefaultName(), - options.GetWriteFileFormat(/*level=*/0)->Identifier(), - options.DataFilePrefix(), options.LegacyPartitionNameEnabled(), - external_paths, global_index_external_path, - options.IndexFileInDataFileDir(), ctx->GetMemoryPool())); - auto snapshot_manager = - std::make_shared(options.GetFileSystem(), ctx->GetRootPath(), branch); - if (VersionManagedCatalog* versioned = AsVersionManaged(ctx->GetCatalog())) { - std::shared_ptr catalog = ctx->GetCatalog(); - Identifier identifier = catalog_identifier.value(); - // Keep the catalog alive while the writer uses its snapshot loader. - snapshot_manager->SetSnapshotLoader( - [catalog, versioned, identifier]() -> Result> { - return versioned->LoadSnapshot(identifier); - }); - } + store_->NewPathFactory(arrow_schema, options, + options.GetWriteFileFormat(/*level=*/0)->Identifier(), + ctx->GetMemoryPool())); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr snapshot_manager, + store_->NewSnapshotManager(options, ctx->GetCatalog(), catalog_identifier)); std::shared_ptr io_manager; const auto& io_temp_dir = ctx->GetTempDirectory(); if (!io_temp_dir.empty()) { @@ -633,8 +704,8 @@ Result> FileStoreTable::NewWrite( } auto file_store_write = std::make_unique( - file_store_path_factory, snapshot_manager, schema_manager, ctx->GetCommitUser(), - ctx->GetRootPath(), schema, arrow_schema, write_schema, realtime_schema_layout, + file_store_path_factory, snapshot_manager, GetSchemaManager(), ctx->GetCommitUser(), + root_path_, schema, arrow_schema, write_schema, realtime_schema_layout, partition_schema, dv_maintainer_factory, io_manager, options, ignore_previous_files, ctx->IsStreamingMode(), ctx->IgnoreNumBucketCheck(), ctx->GetRealtimeContext(), ctx->GetExecutor(), ctx->GetMemoryPool()); @@ -663,11 +734,11 @@ Result> FileStoreTable::NewWrite( } if (options.GetBucket() == BucketModeDefine::POSTPONE_BUCKET) { return PostponeBucketFileStoreWrite::Create( - snapshot_manager, schema_manager, ctx->GetCommitUser(), ctx->GetRootPath(), schema, + snapshot_manager, GetSchemaManager(), ctx->GetCommitUser(), root_path_, schema, arrow_schema, partition_schema, io_manager, options, ctx->IsStreamingMode(), ctx->IgnoreNumBucketCheck(), ctx->GetWriteId(), ctx->GetFileSystemSchemeToIdentifierMap(), ctx->GetExecutor(), ctx->GetMemoryPool(), - specific_fs); + file_system_); } if (options.GetBucket() <= 0) { return Status::Invalid( @@ -711,7 +782,7 @@ Result> FileStoreTable::NewWrite( if (!has_btree_index) { PAIMON_ASSIGN_OR_RAISE( bool has_historical_btree_index, - HasHistoricalPrimaryKeyBTreeDefinition(schema_manager, schema->Id())); + HasHistoricalPrimaryKeyBTreeDefinition(GetSchemaManager(), schema->Id())); if (has_historical_btree_index) { PAIMON_ASSIGN_OR_RAISE(latest_snapshot, snapshot_manager->LatestSnapshot()); has_existing_index_manifest = @@ -750,9 +821,10 @@ Result> FileStoreTable::NewWrite( if (has_btree_index || has_restored_btree_index) { primary_key_index_maintainer_factory = std::make_shared( - ctx->GetRootPath(), branch, schema, primary_key_index_definitions.Definitions(), - file_store_path_factory, index_file_handler, options, io_manager, - ctx->EnableMultiThreadSpill(), ctx->GetExecutor(), ctx->GetMemoryPool()); + root_path_, CurrentBranch(), schema, + primary_key_index_definitions.Definitions(), file_store_path_factory, + index_file_handler, options, io_manager, ctx->EnableMultiThreadSpill(), + ctx->GetExecutor(), ctx->GetMemoryPool()); } std::shared_ptr realtime_schema_layout; @@ -764,8 +836,8 @@ Result> FileStoreTable::NewWrite( } return std::make_unique( - file_store_path_factory, snapshot_manager, schema_manager, ctx->GetCommitUser(), - ctx->GetRootPath(), schema, arrow_schema, realtime_schema_layout, partition_schema, + file_store_path_factory, snapshot_manager, GetSchemaManager(), ctx->GetCommitUser(), + root_path_, schema, arrow_schema, realtime_schema_layout, partition_schema, dv_maintainer_factory, primary_key_index_maintainer_factory, io_manager, key_comparator, sequence_fields_comparator, merge_function_wrapper, options, ignore_previous_files, ctx->IsStreamingMode(), ctx->IgnoreNumBucketCheck(), ctx->EnableMultiThreadSpill(), @@ -774,15 +846,7 @@ Result> FileStoreTable::NewWrite( } Result> FileStoreWrite::Create(std::unique_ptr ctx) { - if (ctx == nullptr) { - return Status::Invalid("write context is null pointer"); - } - if (ctx->GetMemoryPool() == nullptr) { - return Status::Invalid("memory pool is null pointer"); - } - if (ctx->GetExecutor() == nullptr) { - return Status::Invalid("executor is null pointer"); - } + PAIMON_RETURN_NOT_OK(CheckOperatorContext(ctx, "write")); // First the table the context names, then the context handed to that table's `NewWrite`. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, TableOf(*ctx)); return table->NewWrite(std::move(ctx)); @@ -792,18 +856,19 @@ namespace { class TableScanImpl { public: + /// @param snapshot_manager The snapshot manager the scan reads through - the table's own. + /// @param schema_manager The schema manager the scan reads through, likewise. static Result> CreateFileStoreScan( const std::shared_ptr& path_factory, const std::shared_ptr& arrow_schema, const std::shared_ptr& table_schema, const CoreOptions& core_options, + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& schema_manager, const std::shared_ptr& executor, const std::shared_ptr& memory_pool, - const ScanContext* context) { + const ScanContext* context, const std::string& root_path) { auto fs = core_options.GetFileSystem(); auto manifest_file_format = core_options.GetManifestFormat(); - std::string branch = BranchManager::NormalizeBranch(core_options.GetBranch()); - auto snapshot_manager = std::make_shared(fs, context->GetPath(), branch); // TODO(liancheng.lsz): support fallback branch in scan - auto schema_manager = std::make_shared(fs, context->GetPath(), branch); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr manifest_list, ManifestList::Create(fs, manifest_file_format, core_options.GetManifestCompression(), @@ -838,12 +903,12 @@ class TableScanImpl { context->GetScanFilters(), core_options, executor, memory_pool)); } - return WithTablePath(std::move(scan), context); + return WithTablePath(std::move(scan), root_path); } static std::unique_ptr WithTablePath(std::unique_ptr&& scan, - const ScanContext* context) { - scan->WithTablePath(context->GetPath()); + const std::string& root_path) { + scan->WithTablePath(root_path); return std::move(scan); } @@ -939,15 +1004,7 @@ Status ValidateRealtimeScan(const TableSchema& table_schema, const CoreOptions& Result> FileStoreTable::NewScan( std::unique_ptr context) const { - if (context == nullptr) { - return Status::Invalid("scan context is null pointer"); - } - if (context->GetMemoryPool() == nullptr) { - return Status::Invalid("memory pool is null pointer"); - } - if (context->GetExecutor() == nullptr) { - return Status::Invalid("executor is null pointer"); - } + PAIMON_RETURN_NOT_OK(CheckOperatorContext(context, "scan")); // The schema the table was constructed with, so a scan serves the table as it was loaded // rather than reading the schema again. std::shared_ptr loaded_schema = std::static_pointer_cast(schema_); @@ -955,16 +1012,29 @@ Result> FileStoreTable::NewScan( return Status::Invalid("not found latest schema"); } const std::shared_ptr& table_schema = loaded_schema; + PAIMON_RETURN_NOT_OK(CheckOptionsAgreeWithTable(*table_schema, context->GetOptions())); PAIMON_ASSIGN_OR_RAISE(bool read_optimized, OptionsUtils::GetValueFromMap(context->GetOptions(), kReadOptimizedScanOption, false)); - // merge options + // The options the table was loaded with, with a runtime setting the context adds on top: an + // option the context would have changed is refused above, so the table's own stand as they + // are. The branch is the table's, not the context's: a table is the branch it was loaded + // for, so a branch the context carries is not honoured. auto options = table_schema->Options(); for (const auto& [key, value] : context->GetOptions()) { - options[key] = value; + if (key != Options::BRANCH) { + options[key] = value; + } + } + if (BranchManager::IsMainBranch(CurrentBranch())) { + options.erase(Options::BRANCH); + } else { + options[Options::BRANCH] = CurrentBranch(); } + // The file system and root path the table was opened with, so a scan serves the table as it + // was loaded rather than resolving them from the context. PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, - CoreOptions::FromMap(options, context->GetSpecificFileSystem(), {})); + CoreOptions::FromMap(options, file_system_, {})); core_options.WithCache(context->GetCache()); PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*table_schema, core_options)); @@ -990,24 +1060,17 @@ Result> FileStoreTable::NewScan( *arrow_schema, context->GetScanFilters()->GetPredicate(), /*validate_field_idx=*/false)); } - PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, - core_options.CreateExternalPaths()); - PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, - core_options.CreateGlobalIndexExternalPath()); - - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr path_factory, - FileStorePathFactory::Create( - context->GetPath(), arrow_schema, table_schema->PartitionKeys(), - core_options.GetPartitionDefaultName(), core_options.GetFileFormat()->Identifier(), - core_options.DataFilePrefix(), core_options.LegacyPartitionNameEnabled(), - external_paths, global_index_external_path, core_options.IndexFileInDataFileDir(), - context->GetMemoryPool())); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr path_factory, + store_->NewPathFactory(arrow_schema, core_options, + core_options.GetFileFormat()->Identifier(), + context->GetMemoryPool())); + // The managers the table carries, whose schema cache the dispatch already warmed. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr file_store_scan, TableScanImpl::CreateFileStoreScan( path_factory, arrow_schema, table_schema, core_options, - context->GetExecutor(), context->GetMemoryPool(), context.get())); + GetSnapshotManager(), GetSchemaManager(), context->GetExecutor(), + context->GetMemoryPool(), context.get(), root_path_)); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr split_generator, TableScanImpl::CreateSplitGenerator(table_schema, core_options, context.get())); @@ -1037,7 +1100,7 @@ Result> FileStoreTable::NewScan( } if (core_options.DataEvolutionEnabled()) { return std::make_unique( - context->GetPath(), snapshot_reader, std::move(batch_scan), table_schema, + root_path_, snapshot_reader, std::move(batch_scan), table_schema, context->GetGlobalIndexResult(), core_options, context->GetMemoryPool(), context->GetExecutor()); } @@ -1054,15 +1117,7 @@ Result> FileStoreTable::NewScan( } Result> TableScan::Create(std::unique_ptr context) { - if (context == nullptr) { - return Status::Invalid("scan context is null pointer"); - } - if (context->GetMemoryPool() == nullptr) { - return Status::Invalid("memory pool is null pointer"); - } - if (context->GetExecutor() == nullptr) { - return Status::Invalid("executor is null pointer"); - } + PAIMON_RETURN_NOT_OK(CheckOperatorContext(context, "scan")); // First the table the context names, then the context handed to that table's `NewScan`. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, TableOf(*context)); return table->NewScan(std::move(context)); @@ -1070,13 +1125,20 @@ Result> TableScan::Create(std::unique_ptr> CreateInternalReadContext( const std::shared_ptr& context, const std::string& branch, - const std::shared_ptr& loaded_schema) { - std::map tmp_options = context->GetOptions(); + const std::shared_ptr& schema_manager, + const std::shared_ptr& loaded_schema, + const std::shared_ptr& file_system, const std::string& root_path) { std::shared_ptr table_schema; const auto& specific_table_schema = context->GetSpecificTableSchema(); if (loaded_schema != nullptr) { @@ -1085,29 +1147,29 @@ Result> CreateInternalReadContext( PAIMON_ASSIGN_OR_RAISE(table_schema, TableSchema::CreateFromJson(specific_table_schema.value())); } else { - PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_core_options, - CoreOptions::FromMap(tmp_options, context->GetSpecificFileSystem(), - context->GetFileSystemSchemeToIdentifierMap())); - SchemaManager schema_manager(tmp_core_options.GetFileSystem(), context->GetPath(), branch); PAIMON_ASSIGN_OR_RAISE(std::optional> latest_schema, - schema_manager.Latest()); + schema_manager->Latest()); if (!latest_schema) { - return Status::Invalid(fmt::format("schema file not found in path {}, branch {}", - context->GetPath(), branch)); + return Status::Invalid( + fmt::format("schema file not found in path {}, branch {}", root_path, branch)); } table_schema = latest_schema.value(); } assert(table_schema); - // merge options + // The options the schema was loaded with, with a runtime setting the context adds on top - + // and the fallback below serves the ones its own schema was loaded with. The branch is the + // one named below, not the context's: a branch the context carries is not honoured here. auto options = table_schema->Options(); - for (const auto& [key, value] : tmp_options) { - options[key] = value; + for (const auto& [key, value] : context->GetOptions()) { + if (key != Options::BRANCH) { + options[key] = value; + } } if (branch != BranchManager::DEFAULT_MAIN_BRANCH) { options[Options::BRANCH] = branch; } - return InternalReadContext::Create(context, table_schema, options); + return InternalReadContext::Create(context, table_schema, options, file_system, &root_path); } Result> CreateTableRead( @@ -1126,19 +1188,11 @@ Result> CreateTableRead( } PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*table_schema, core_options)); auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); - PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, - core_options.CreateExternalPaths()); - PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, - core_options.CreateGlobalIndexExternalPath()); - PAIMON_ASSIGN_OR_RAISE( std::shared_ptr path_factory, - FileStorePathFactory::Create( - internal_context->GetPath(), arrow_schema, table_schema->PartitionKeys(), - core_options.GetPartitionDefaultName(), core_options.GetFileFormat()->Identifier(), - core_options.DataFilePrefix(), core_options.LegacyPartitionNameEnabled(), - external_paths, global_index_external_path, core_options.IndexFileInDataFileDir(), - memory_pool)); + FileStore::MakePathFactory(internal_context->GetPath(), arrow_schema, + table_schema->PartitionKeys(), core_options, + core_options.GetFileFormat()->Identifier(), memory_pool)); if (internal_context->GetPrimaryKeys().empty()) { return std::make_unique(path_factory, internal_context, memory_pool, @@ -1152,21 +1206,21 @@ Result> CreateTableRead( Result> FileStoreTable::NewRead( std::unique_ptr context) const { std::shared_ptr read_context = std::move(context); - if (read_context == nullptr) { - return Status::Invalid("read context is null pointer"); - } - if (read_context->GetMemoryPool() == nullptr) { - return Status::Invalid("memory pool is null pointer"); - } - if (read_context->GetExecutor() == nullptr) { - return Status::Invalid("executor is null pointer"); - } + PAIMON_RETURN_NOT_OK(CheckOperatorContext(read_context, "read")); // The schema the table was constructed with, so a read serves the table as it was loaded - // rather than reading the schema again. + // rather than reading the schema again. Its options are the ones the read serves: an option + // the context would have changed is refused rather than honoured. std::shared_ptr loaded_schema = std::static_pointer_cast(schema_); + if (loaded_schema == nullptr) { + return Status::Invalid("not found latest schema"); + } + PAIMON_RETURN_NOT_OK(CheckOptionsAgreeWithTable(*loaded_schema, read_context->GetOptions())); + // The branch the table serves, which the read reads on: a table is the branch it was loaded + // for, so the context's own branch is not consulted. PAIMON_ASSIGN_OR_RAISE( std::shared_ptr internal_context, - CreateInternalReadContext(read_context, read_context->GetBranch(), loaded_schema)); + CreateInternalReadContext(read_context, CurrentBranch(), GetSchemaManager(), loaded_schema, + file_system_, root_path_)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, CreateTableRead(internal_context, read_context->GetMemoryPool(), @@ -1179,10 +1233,14 @@ Result> FileStoreTable::NewRead( return std::move(table_read); } + // The fallback branch is not the table's, so its schema is read through a manager of its + // own rather than the table's. PAIMON_ASSIGN_OR_RAISE( std::shared_ptr fallback_context, - CreateInternalReadContext(read_context, /*branch=*/scan_fallback_branch.value(), - /*loaded_schema=*/nullptr)); + CreateInternalReadContext( + read_context, /*branch=*/scan_fallback_branch.value(), + std::make_shared(file_system_, root_path_, scan_fallback_branch.value()), + /*loaded_schema=*/nullptr, file_system_, root_path_)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr fallback_table_read, CreateTableRead(fallback_context, read_context->GetMemoryPool(), @@ -1193,15 +1251,7 @@ Result> FileStoreTable::NewRead( } Result> TableRead::Create(std::unique_ptr ctx) { - if (ctx == nullptr) { - return Status::Invalid("read context is null pointer"); - } - if (ctx->GetMemoryPool() == nullptr) { - return Status::Invalid("memory pool is null pointer"); - } - if (ctx->GetExecutor() == nullptr) { - return Status::Invalid("executor is null pointer"); - } + PAIMON_RETURN_NOT_OK(CheckOperatorContext(ctx, "read")); // First the table the context names, then the context handed to that table's `NewRead`. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, TableOf(*ctx)); return table->NewRead(std::move(ctx)); @@ -1293,41 +1343,19 @@ Result> NewSnapshotCommit( Result> FileStoreTable::NewCommit( std::unique_ptr ctx) const { - if (ctx == nullptr) { - return Status::Invalid("commit context is null pointer"); - } - if (ctx->GetMemoryPool() == nullptr) { - return Status::Invalid("memory pool is null pointer"); - } - if (ctx->GetExecutor() == nullptr) { - return Status::Invalid("executor is null pointer"); - } + PAIMON_RETURN_NOT_OK(CheckOperatorContext(ctx, "commit")); PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog())); - // Use catalog credentials unless the caller supplied a file system. std::shared_ptr specific_fs = ctx->GetSpecificFileSystem(); - if (specific_fs == nullptr && ctx->GetCatalog() != nullptr) { - if (!ctx->GetIdentifier()) { - return Status::Invalid("a catalog commit requires a table identifier"); - } - // Use the credentials of this table, which a catalog issuing per-table temporary - // ones only hands out through GetTableFileSystem. - PAIMON_ASSIGN_OR_RAISE(specific_fs, - ctx->GetCatalog()->GetTableFileSystem(ctx->GetIdentifier().value())); - } - PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, - CoreOptions::FromMap(ctx->GetOptions(), specific_fs)); - const std::string& root_path = ctx->GetRootPath(); - // The branch the context was built for, already validated by `CommitContextBuilder`. - PAIMON_ASSIGN_OR_RAISE(std::string branch, BranchManager::ResolveBranch( - ctx->GetIdentifier(), ctx->GetOptions(), - /*explicit_branch=*/std::nullopt, "commit")); - std::optional catalog_identifier; - if (ctx->GetIdentifier()) { - PAIMON_ASSIGN_OR_RAISE( - Identifier branch_identifier, - CatalogUtils::BranchIdentifier(ctx->GetIdentifier().value(), branch)); - catalog_identifier.emplace(std::move(branch_identifier)); - } + // A catalog commit names its table; the file system itself comes from the table, which its + // catalog's credentials already opened. + if (specific_fs == nullptr && ctx->GetCatalog() != nullptr && !ctx->GetIdentifier()) { + return Status::Invalid("a catalog commit requires a table identifier"); + } + // A commit commits to the root path the table was opened with and the branch the table + // serves, in place of the ones named by the context: a table is the branch it was loaded + // for, so the context's own branch is not consulted. + PAIMON_ASSIGN_OR_RAISE(std::optional catalog_identifier, + MakeBranchCatalogIdentifier(ctx->GetIdentifier(), CurrentBranch())); // The id a snapshot records the catalog's current schema by, read through the loader below // rather than trusted from a schema this commit will not re-read. FileStoreCommitImpl::SchemaIdLoader schema_id_loader; @@ -1349,33 +1377,30 @@ Result> FileStoreTable::NewCommit( return data_schema->Id(); }; } - auto schema_manager = - std::make_shared(tmp_options.GetFileSystem(), root_path, branch); // The schema the table was constructed with, so a commit serves the table as it was loaded // rather than reading the schema again. - std::shared_ptr latest_schema = std::static_pointer_cast(schema_); - if (latest_schema == nullptr) { - return Status::Invalid(fmt::format("not found latest schema in branch {}", branch)); - } - const std::shared_ptr& schema = latest_schema; - auto opts = schema->Options(); - for (const auto& [key, value] : ctx->GetOptions()) { - opts[key] = value; - } - // The metadata paths below are built for this branch, so what reads through these options, - // whose cache keys are scoped by it, has to read the same one. - opts[Options::BRANCH] = branch; + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, + LatestTableSchemaOfBranch(schema_, CurrentBranch())); std::shared_ptr arrow_schema = DataField::ConvertDataFieldsToArrowSchema(schema->Fields()); - PAIMON_ASSIGN_OR_RAISE(CoreOptions options, CoreOptions::FromMap(opts, specific_fs)); + // The options the table was loaded with, with a runtime setting the context adds on top: an + // option the context would have changed is refused rather than honoured, and the branch is + // the table's - the metadata paths below are built for it, so what reads through these + // options, whose cache keys are scoped by it, has to read the same one. + std::map opts; + PAIMON_ASSIGN_OR_RAISE(opts, + MergedOperatorOptions(*schema, ctx->GetOptions(), CurrentBranch())); + PAIMON_ASSIGN_OR_RAISE(CoreOptions options, CoreOptions::FromMap(opts, file_system_)); PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*schema, options)); assert(options.GetFileSystem()); assert(options.GetFileFormat()); PAIMON_RETURN_NOT_OK(FileStoreCommitImpl::ValidateCommitOptions(options)); - PAIMON_ASSIGN_OR_RAISE(bool is_object_store, FileSystem::IsObjectStore(root_path)); - // Catalog commits do not require atomic rename on the object store. + PAIMON_ASSIGN_OR_RAISE(bool is_object_store, FileSystem::IsObjectStore(root_path_)); + // Catalog commits do not require atomic rename on the object store. The inte-test marker is + // not a table option, so it is read from the context the commit was handed. if (is_object_store && !ctx->UseRESTCatalogCommit() && !IsCatalogCommit(*ctx) && - opts.find("enable-object-store-commit-in-inte-test") == opts.end()) { + ctx->GetOptions().find("enable-object-store-commit-in-inte-test") == + ctx->GetOptions().end()) { return Status::NotImplemented( "commit operation does not support object store file system for now"); } @@ -1384,30 +1409,14 @@ Result> FileStoreTable::NewCommit( options.GetPartitionDefaultName(), options.LegacyPartitionNameEnabled(), ctx->GetMemoryPool())); - PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, options.CreateExternalPaths()); - PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, - options.CreateGlobalIndexExternalPath()); - PAIMON_ASSIGN_OR_RAISE( std::shared_ptr path_factory, - FileStorePathFactory::Create( - root_path, arrow_schema, schema->PartitionKeys(), options.GetPartitionDefaultName(), - options.GetFileFormat()->Identifier(), options.DataFilePrefix(), - options.LegacyPartitionNameEnabled(), external_paths, global_index_external_path, - options.IndexFileInDataFileDir(), ctx->GetMemoryPool())); - - auto snapshot_manager = - std::make_shared(options.GetFileSystem(), root_path, branch); - if (IsCatalogCommit(*ctx)) { - std::shared_ptr catalog = ctx->GetCatalog(); - VersionManagedCatalog* versioned = AsVersionManaged(catalog); - Identifier identifier = catalog_identifier.value(); - // Capture the catalog to keep the versioned interface alive. - snapshot_manager->SetSnapshotLoader( - [catalog, versioned, identifier]() -> Result> { - return versioned->LoadSnapshot(identifier); - }); - } + store_->NewPathFactory(arrow_schema, options, options.GetFileFormat()->Identifier(), + ctx->GetMemoryPool())); + + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr snapshot_manager, + store_->NewSnapshotManager(options, ctx->GetCatalog(), catalog_identifier)); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr manifest_list, ManifestList::Create(options.GetFileSystem(), options.GetManifestFormat(), @@ -1435,11 +1444,11 @@ Result> FileStoreTable::NewCommit( CommitScanner::ScanSupplier scan_supplier; if (schema->PrimaryKeys().empty()) { - scan_supplier = CreateAppendScanSupplier(snapshot_manager, schema_manager, manifest_list, - manifest_file, schema, arrow_schema, options, - ctx->GetExecutor(), ctx->GetMemoryPool()); + scan_supplier = CreateAppendScanSupplier(snapshot_manager, GetSchemaManager(), + manifest_list, manifest_file, schema, arrow_schema, + options, ctx->GetExecutor(), ctx->GetMemoryPool()); } else { - scan_supplier = CreatePkScanSupplier(snapshot_manager, schema_manager, manifest_list, + scan_supplier = CreatePkScanSupplier(snapshot_manager, GetSchemaManager(), manifest_list, manifest_file, schema, arrow_schema, options, ctx->GetExecutor(), ctx->GetMemoryPool()); } @@ -1448,24 +1457,16 @@ Result> FileStoreTable::NewCommit( NewSnapshotCommit(*ctx, catalog_identifier, options, snapshot_manager)); return std::make_unique( - ctx->GetMemoryPool(), ctx->GetExecutor(), arrow_schema, root_path, ctx->GetCommitUser(), + ctx->GetMemoryPool(), ctx->GetExecutor(), arrow_schema, root_path_, ctx->GetCommitUser(), options, path_factory, std::move(partition_computer), snapshot_manager, snapshot_commit, ctx->IgnoreEmptyCommit(), ctx->AppendCommitCheckConflict(), schema, std::move(schema_id_loader), manifest_file, manifest_list, index_manifest_file, - expire_snapshots, schema_manager, std::move(scan_supplier)); + expire_snapshots, GetSchemaManager(), std::move(scan_supplier)); } Result> FileStoreCommit::Create( std::unique_ptr ctx) { - if (ctx == nullptr) { - return Status::Invalid("commit context is null pointer"); - } - if (ctx->GetMemoryPool() == nullptr) { - return Status::Invalid("memory pool is null pointer"); - } - if (ctx->GetExecutor() == nullptr) { - return Status::Invalid("executor is null pointer"); - } + PAIMON_RETURN_NOT_OK(CheckOperatorContext(ctx, "commit")); // First the table the context names, then the context handed to that table's `NewCommit`. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, TableOf(*ctx)); return table->NewCommit(std::move(ctx)); diff --git a/src/paimon/core/table/table_test.cpp b/src/paimon/core/table/table_test.cpp index 1ad8d9cd3..01a7ce27e 100644 --- a/src/paimon/core/table/table_test.cpp +++ b/src/paimon/core/table/table_test.cpp @@ -25,22 +25,29 @@ #include #include "arrow/api.h" +#include "arrow/c/abi.h" +#include "arrow/c/bridge.h" #include "gtest/gtest.h" #include "paimon/commit_context.h" +#include "paimon/commit_message.h" #include "paimon/common/utils/checked_cast.h" #include "paimon/core/catalog/file_system_catalog.h" #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/table_schema.h" #include "paimon/core/table/file_store_table.h" +#include "paimon/defs.h" #include "paimon/file_store_commit.h" #include "paimon/file_store_write.h" #include "paimon/read_context.h" +#include "paimon/record_batch.h" #include "paimon/scan_context.h" #include "paimon/schema/schema.h" #include "paimon/table/format/format_table.h" +#include "paimon/table/source/plan.h" #include "paimon/table/source/table_read.h" #include "paimon/table/source/table_scan.h" #include "paimon/testing/mock/mock_catalog.h" +#include "paimon/testing/mock/mock_file_system.h" #include "paimon/testing/utils/testharness.h" #include "paimon/write_context.h" @@ -49,23 +56,198 @@ namespace paimon::test { namespace { // Creates a managed table directory and a `Table` over it, holding the schema written to -// `branch`'s directory under `identifier`. -Result> CreateManagedTable(const std::shared_ptr& file_system, - const std::string& path, - const Identifier& identifier, - const std::string& branch = "main") { +// `branch`'s directory under `identifier` with `options`. +Result> CreateManagedTable( + const std::shared_ptr& file_system, const std::string& path, + const Identifier& identifier, const std::string& branch = "main", + const std::map& options = {}) { SchemaManager schema_manager(file_system, path, branch); auto schema = arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false)}); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr created, schema_manager.CreateTable(schema, /*partition_keys=*/{}, - /*primary_keys=*/{}, /*options=*/{})); + /*primary_keys=*/{}, options)); return std::make_shared( file_system, path, - checked_pointer_cast(std::shared_ptr(std::move(created))), identifier); + checked_pointer_cast(std::shared_ptr(std::move(created))), identifier, + /*uuid=*/"", branch); } } // namespace +TEST(TableTest, TestTableCarriesManagersOfItsBranch) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto fs = dir->GetFileSystem(); + const std::string path = dir->Str() + "/tbl"; + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
table, + CreateManagedTable(fs, path, Identifier(Identifier::kUnknownDatabase, "tbl"), + /*branch=*/"dev")); + auto file_store_table = std::dynamic_pointer_cast(table); + ASSERT_TRUE(file_store_table != nullptr); + // The branch the table was built for is the one its managers point at. + EXPECT_EQ(file_store_table->CurrentBranch(), "dev"); + ASSERT_TRUE(file_store_table->GetSchemaManager() != nullptr); + ASSERT_TRUE(file_store_table->GetSnapshotManager() != nullptr); + EXPECT_EQ(file_store_table->GetSnapshotManager()->Branch(), "dev"); + EXPECT_EQ(file_store_table->GetSnapshotManager()->RootPath(), path); + EXPECT_EQ(file_store_table->GetSnapshotManager()->Fs(), fs); + // A copy serves the same branch through the same managers, whose cache it shares. + ASSERT_OK_AND_ASSIGN(std::shared_ptr
copy, table->Copy({})); + auto file_store_copy = std::dynamic_pointer_cast(copy); + ASSERT_TRUE(file_store_copy != nullptr); + EXPECT_EQ(file_store_copy->CurrentBranch(), "dev"); + EXPECT_EQ(file_store_copy->GetSchemaManager(), file_store_table->GetSchemaManager()); + EXPECT_EQ(file_store_copy->GetSnapshotManager(), file_store_table->GetSnapshotManager()); +} + +TEST(TableTest, TestSwitchToBranch) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto fs = dir->GetFileSystem(); + const std::string path = dir->Str() + "/tbl"; + const Identifier identifier(Identifier::kUnknownDatabase, "tbl"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, CreateManagedTable(fs, path, identifier)); + // A branch of its own, with a schema of its own. + SchemaManager dev_schema_manager(fs, path, "dev"); + ASSERT_OK_AND_ASSIGN( + [[maybe_unused]] std::unique_ptr dev_schema, + dev_schema_manager.CreateTable(arrow::schema({arrow::field("c0", arrow::int32())}), + /*partition_keys=*/{}, /*primary_keys=*/{}, {})); + + // Switching to a branch serves that branch's schema and managers. + auto main_file_store_table = std::dynamic_pointer_cast(table); + ASSERT_TRUE(main_file_store_table != nullptr); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
dev_table, + main_file_store_table->SwitchToBranch("dev")); + auto dev_file_store_table = std::dynamic_pointer_cast(dev_table); + ASSERT_TRUE(dev_file_store_table != nullptr); + EXPECT_EQ("dev", dev_file_store_table->CurrentBranch()); + ASSERT_EQ((std::vector{"c0"}), dev_table->LatestSchema()->FieldNames()); + auto dev_table_schema = std::dynamic_pointer_cast(dev_table->LatestSchema()); + ASSERT_TRUE(dev_table_schema != nullptr); + ASSERT_EQ("dev", dev_table_schema->Options().at(Options::BRANCH)); + ASSERT_TRUE(dev_file_store_table->GetSchemaManager() != nullptr); + ASSERT_TRUE(dev_file_store_table->GetSnapshotManager() != nullptr); + EXPECT_EQ("dev", dev_file_store_table->GetSnapshotManager()->Branch()); + + // Switching to the branch already served hands back the same table, managers shared. + ASSERT_OK_AND_ASSIGN(std::shared_ptr
same_table, + dev_file_store_table->SwitchToBranch("dev")); + auto same_file_store_table = std::dynamic_pointer_cast(same_table); + ASSERT_TRUE(same_file_store_table != nullptr); + EXPECT_EQ(same_table->LatestSchema(), dev_table->LatestSchema()); + EXPECT_EQ(same_file_store_table->GetSchemaManager(), dev_file_store_table->GetSchemaManager()); + EXPECT_EQ(same_file_store_table->GetSnapshotManager(), + dev_file_store_table->GetSnapshotManager()); + + // Switching back serves the main branch's schema again. + ASSERT_OK_AND_ASSIGN(std::shared_ptr
main_again, + dev_file_store_table->SwitchToBranch("main")); + auto main_again_file_store_table = std::dynamic_pointer_cast(main_again); + ASSERT_TRUE(main_again_file_store_table != nullptr); + EXPECT_EQ("main", main_again_file_store_table->CurrentBranch()); + ASSERT_EQ((std::vector{"id"}), main_again->LatestSchema()->FieldNames()); + auto main_again_schema = std::dynamic_pointer_cast(main_again->LatestSchema()); + ASSERT_TRUE(main_again_schema != nullptr); + EXPECT_EQ(0, main_again_schema->Options().count(Options::BRANCH)); + + // A branch with no schema does not exist. + ASSERT_NOK_WITH_MSG(main_file_store_table->SwitchToBranch("missing"), "branch"); +} + +TEST(TableTest, TestNewWriteServesTheBranchTheTableWasLoadedWith) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::shared_ptr fs = dir->GetFileSystem(); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + CreateManagedTable(fs, dir->Str(), identifier, /*branch=*/"dev", + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + + // A context that names no branch is the one a caller hands straight to the table's write, + // which serves the branch the table was loaded with rather than the context's own main. + WriteContextBuilder write_builder(dir->Str(), "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write, + table->NewWrite(std::move(write_context))); + arrow::Int32Builder ids_builder; + ASSERT_TRUE(ids_builder.AppendValues({1, 2}).ok()); + std::shared_ptr ids_array; + ASSERT_TRUE(ids_builder.Finish(&ids_array).ok()); + auto struct_array = arrow::StructArray::Make({ids_array}, {arrow::field("id", arrow::int32(), + /*nullable=*/false)}) + .ValueOrDie(); + ::ArrowArray arrow_array; + ASSERT_TRUE(arrow::ExportArray(*struct_array, &arrow_array).ok()); + RecordBatchBuilder batch_builder(&arrow_array); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, batch_builder.SetBucket(0).Finish()); + ASSERT_OK(write->Write(std::move(batch))); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::vector> messages, + write->PrepareCommit()); + ASSERT_OK(write->Close()); + + // The files land under the branch's own directory, which a write of the main branch - the + // branch the context alone would name - would never touch. + std::vector branch_entries; + ASSERT_OK(fs->ListDir(BranchManager::BranchPath(dir->Str(), "dev"), &branch_entries)); + EXPECT_FALSE(branch_entries.empty()); +} + +TEST(TableTest, TestOperatorsServeTheBranchTheTableWasLoadedWith) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
table, + CreateManagedTable(dir->GetFileSystem(), dir->Str(), identifier, /*branch=*/"dev", + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + + // The write and the commit below carry no branch, and both serve the branch the table was + // loaded with, so the data they publish lands on it. + WriteContextBuilder write_builder(dir->Str(), "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write, + table->NewWrite(std::move(write_context))); + arrow::Int32Builder ids_builder; + ASSERT_TRUE(ids_builder.AppendValues({1, 2}).ok()); + std::shared_ptr ids_array; + ASSERT_TRUE(ids_builder.Finish(&ids_array).ok()); + auto struct_array = arrow::StructArray::Make({ids_array}, {arrow::field("id", arrow::int32(), + /*nullable=*/false)}) + .ValueOrDie(); + ::ArrowArray arrow_array; + ASSERT_TRUE(arrow::ExportArray(*struct_array, &arrow_array).ok()); + RecordBatchBuilder batch_builder(&arrow_array); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, batch_builder.SetBucket(0).Finish()); + ASSERT_OK(write->Write(std::move(batch))); + ASSERT_OK_AND_ASSIGN(std::vector> messages, + write->PrepareCommit()); + ASSERT_OK(write->Close()); + CommitContextBuilder commit_builder(dir->Str(), "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, + table->NewCommit(std::move(commit_context))); + ASSERT_OK(commit->Commit(messages)); + + // The scan finds what they wrote, reading through the table's own managers on its branch. + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_ctx, + ScanContextBuilder(dir->Str()).Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(scan_ctx))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + EXPECT_FALSE(plan->Splits().empty()); + + // A branch the scan context carries does not win either: the table's own does, so the data + // it finds is still the branch's. + ScanContextBuilder main_scan_builder(dir->Str()); + main_scan_builder.AddOption(Options::BRANCH, "main"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr main_scan_ctx, main_scan_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr main_scan, + table->NewScan(std::move(main_scan_ctx))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr main_plan, main_scan->CreatePlan()); + EXPECT_FALSE(main_plan->Splits().empty()); +} + TEST(TableTest, TestGetTableWithUnknownDatabase) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); @@ -166,9 +348,9 @@ TEST(TableTest, TestMainBranchAliasResolvesToTheMainBranch) { SCOPED_TRACE(main_identifier.ToString()); // A catalog that loaded the schema itself calls the constructor directly, which a // rest catalog is shaped like. The alias resolves to the main branch as well. - FileStoreTable table(fs, dir->Str(), schema, main_identifier); - ASSERT_OK_AND_ASSIGN(std::string branch, table.CurrentBranch()); - EXPECT_EQ("main", branch); + ASSERT_OK_AND_ASSIGN(std::string branch, main_identifier.GetBranchNameOrDefault()); + FileStoreTable table(fs, dir->Str(), schema, main_identifier, /*uuid=*/"", branch); + EXPECT_EQ("main", table.CurrentBranch()); } } @@ -180,15 +362,17 @@ TEST(TableTest, TestCurrentBranch) { ASSERT_OK_AND_ASSIGN( std::shared_ptr
main_table, CreateManagedTable(dir->GetFileSystem(), dir->Str(), Identifier("db", "tbl"))); - ASSERT_OK_AND_ASSIGN(std::string branch, main_table->CurrentBranch()); - EXPECT_EQ(branch, "main"); + auto main_file_store_table = std::dynamic_pointer_cast(main_table); + ASSERT_TRUE(main_file_store_table != nullptr); + EXPECT_EQ(main_file_store_table->CurrentBranch(), "main"); - // The identifier a table was opened with names the branch it is for. + // The branch a table was created on is the one it carries. ASSERT_OK_AND_ASSIGN(std::shared_ptr
branch_table, CreateManagedTable(dir->GetFileSystem(), dir->Str() + "/branch_tbl", Identifier("db", "tbl", "dev"), /*branch=*/"dev")); - ASSERT_OK_AND_ASSIGN(branch, branch_table->CurrentBranch()); - EXPECT_EQ(branch, "dev"); + auto branch_file_store_table = std::dynamic_pointer_cast(branch_table); + ASSERT_TRUE(branch_file_store_table != nullptr); + EXPECT_EQ(branch_file_store_table->CurrentBranch(), "dev"); } TEST(TableTest, TestCatalogOpenedTableServesThePerTableFileSystem) { @@ -314,6 +498,110 @@ TEST(TableTest, TestNewScanServesTheSchemaTheTableWasLoadedWith) { ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(ctx)), "not found latest schema"); } +TEST(TableTest, TestNewScanServesTheFileSystemTheTableWasOpenedWith) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::shared_ptr fs = dir->GetFileSystem(); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + CreateManagedTable(fs, dir->Str(), identifier, /*branch=*/"main", + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + + // A file system is only observable once a scan has data to read, so two rows are written and + // committed through the table's own write and commit. + WriteContextBuilder write_builder(dir->Str(), "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write, + table->NewWrite(std::move(write_context))); + arrow::Int32Builder ids_builder; + ASSERT_TRUE(ids_builder.AppendValues({1, 2}).ok()); + std::shared_ptr ids_array; + ASSERT_TRUE(ids_builder.Finish(&ids_array).ok()); + auto struct_array = arrow::StructArray::Make({ids_array}, {arrow::field("id", arrow::int32(), + /*nullable=*/false)}) + .ValueOrDie(); + ::ArrowArray arrow_array; + ASSERT_TRUE(arrow::ExportArray(*struct_array, &arrow_array).ok()); + RecordBatchBuilder batch_builder(&arrow_array); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, batch_builder.SetBucket(0).Finish()); + ASSERT_OK(write->Write(std::move(batch))); + ASSERT_OK_AND_ASSIGN(std::vector> messages, + write->PrepareCommit()); + CommitContextBuilder commit_builder(dir->Str(), "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, + table->NewCommit(std::move(commit_context))); + ASSERT_OK(commit->Commit(messages)); + ASSERT_OK(write->Close()); + + // The table was opened with the local file system, so a scan serves its data through that + // one even when the context hands over a mock that can read nothing. + ASSERT_OK_AND_ASSIGN( + std::unique_ptr held_ctx, + ScanContextBuilder(dir->Str()).WithFileSystem(std::make_shared()).Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(held_ctx))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_FALSE(plan->Splits().empty()); + + // A context without a table resolves the file system itself, which is the mock and reads no + // schema at all: the file system counterpart of the schema case above. + ASSERT_OK_AND_ASSIGN( + std::unique_ptr ctx, + ScanContextBuilder(dir->Str()).WithFileSystem(std::make_shared()).Finish()); + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(ctx)), "not found latest schema"); +} + +TEST(TableTest, TestNewScanServesTheRootPathTheTableWasOpenedWith) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::shared_ptr fs = dir->GetFileSystem(); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + CreateManagedTable(fs, dir->Str(), identifier, /*branch=*/"main", + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + + // Two rows are written and committed through the table's own write and commit, so the scan + // below has data to find. + WriteContextBuilder write_builder(dir->Str(), "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write, + table->NewWrite(std::move(write_context))); + arrow::Int32Builder ids_builder; + ASSERT_TRUE(ids_builder.AppendValues({1, 2}).ok()); + std::shared_ptr ids_array; + ASSERT_TRUE(ids_builder.Finish(&ids_array).ok()); + auto struct_array = arrow::StructArray::Make({ids_array}, {arrow::field("id", arrow::int32(), + /*nullable=*/false)}) + .ValueOrDie(); + ::ArrowArray arrow_array; + ASSERT_TRUE(arrow::ExportArray(*struct_array, &arrow_array).ok()); + RecordBatchBuilder batch_builder(&arrow_array); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, batch_builder.SetBucket(0).Finish()); + ASSERT_OK(write->Write(std::move(batch))); + ASSERT_OK_AND_ASSIGN(std::vector> messages, + write->PrepareCommit()); + CommitContextBuilder commit_builder(dir->Str(), "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, + table->NewCommit(std::move(commit_context))); + ASSERT_OK(commit->Commit(messages)); + ASSERT_OK(write->Close()); + + // The table was opened with this directory as its root path, so a scan serves its data from + // there even when the context names a different one. + ASSERT_OK_AND_ASSIGN(std::unique_ptr held_ctx, + ScanContextBuilder(dir->Str() + "/not-the-table").Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(held_ctx))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_FALSE(plan->Splits().empty()); + + // A context without a table reads the schema from the directory it names, which is not a + // table at all: the root path counterpart of the schema case above. + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, + ScanContextBuilder(dir->Str() + "/not-the-table").Finish()); + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(ctx)), "not found latest schema"); +} + TEST(TableTest, TestNewReadCreatesATableRead) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); @@ -383,6 +671,122 @@ TEST(TableTest, TestNewScanReadWriteCommitRejectANullContext) { ASSERT_NOK_WITH_MSG(table->NewCommit(nullptr), "commit context is null pointer"); } +TEST(TableTest, TestOperatorsRefuseAnOptionThatContradictsTheTable) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
table, + CreateManagedTable(dir->GetFileSystem(), dir->Str(), identifier, /*branch=*/"main", + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + + // The table was loaded with `bucket` = 1, so every operator refuses a context that names + // another value for it: the table serves the options it was loaded with, and a caller that + // means to change them copies the table. + { + ScanContextBuilder scan_builder(dir->Str()); + scan_builder.AddOption(Options::BUCKET, "2"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, scan_builder.Finish()); + ASSERT_NOK_WITH_MSG(table->NewScan(std::move(ctx)), + "option 'bucket' is '2' in the context but '1' on the table"); + } + { + ReadContextBuilder read_builder(dir->Str()); + read_builder.AddOption(Options::BUCKET, "2"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, read_builder.Finish()); + ASSERT_NOK_WITH_MSG(table->NewRead(std::move(ctx)), + "option 'bucket' is '2' in the context but '1' on the table"); + } + { + WriteContextBuilder write_builder(dir->Str(), "commit-user"); + write_builder.AddOption(Options::BUCKET, "2"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, write_builder.Finish()); + ASSERT_NOK_WITH_MSG(table->NewWrite(std::move(ctx)), + "option 'bucket' is '2' in the context but '1' on the table"); + } + { + CommitContextBuilder commit_builder(dir->Str(), "commit-user"); + commit_builder.AddOption(Options::BUCKET, "2"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, commit_builder.Finish()); + ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(ctx)), + "option 'bucket' is '2' in the context but '1' on the table"); + } + + // The copy made with the option in question agrees with the context, so it serves it. + ASSERT_OK_AND_ASSIGN(std::shared_ptr
copied, table->Copy({{Options::BUCKET, "2"}})); + ScanContextBuilder agreeing_scan_builder(dir->Str()); + agreeing_scan_builder.AddOption(Options::BUCKET, "2"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, agreeing_scan_builder.Finish()); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr scan, + copied->NewScan(std::move(ctx))); +} + +TEST(TableTest, TestCopyServesTheTableWithTheGivenOptionsOnTop) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::shared_ptr fs = dir->GetFileSystem(); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + CreateManagedTable(fs, dir->Str(), identifier, /*branch=*/"main", + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + + // Two rows are written and committed through the table's own write and commit, so the copy + // below has data to find. + WriteContextBuilder write_builder(dir->Str(), "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write, + table->NewWrite(std::move(write_context))); + arrow::Int32Builder ids_builder; + ASSERT_TRUE(ids_builder.AppendValues({1, 2}).ok()); + std::shared_ptr ids_array; + ASSERT_TRUE(ids_builder.Finish(&ids_array).ok()); + auto struct_array = arrow::StructArray::Make({ids_array}, {arrow::field("id", arrow::int32(), + /*nullable=*/false)}) + .ValueOrDie(); + ::ArrowArray arrow_array; + ASSERT_TRUE(arrow::ExportArray(*struct_array, &arrow_array).ok()); + RecordBatchBuilder batch_builder(&arrow_array); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, batch_builder.SetBucket(0).Finish()); + ASSERT_OK(write->Write(std::move(batch))); + ASSERT_OK_AND_ASSIGN(std::vector> messages, + write->PrepareCommit()); + CommitContextBuilder commit_builder(dir->Str(), "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, + table->NewCommit(std::move(commit_context))); + ASSERT_OK(commit->Commit(messages)); + ASSERT_OK(write->Close()); + + // The copy overlays what the call gives on the options the schema stored; the schema itself + // and the identity the table was opened with are copied as they are. + ASSERT_OK_AND_ASSIGN(std::shared_ptr
copied, + table->Copy({{Options::BUCKET, "2"}, {Options::TYPE, "format-table"}})); + std::shared_ptr copied_schema = + std::dynamic_pointer_cast(copied->LatestSchema()); + ASSERT_TRUE(copied_schema != nullptr); + EXPECT_EQ(copied_schema->Options().at(Options::BUCKET), "2"); + // `type` is structural and read from the schema alone, so the copy carries none of it: the + // option given at the call does not restyle a managed table. + EXPECT_EQ(copied_schema->Options().count(Options::TYPE), 0); + EXPECT_EQ(copied_schema->Options().at(Options::BUCKET_KEY), "id"); + EXPECT_EQ(copied_schema->Id(), + std::dynamic_pointer_cast(table->LatestSchema())->Id()); + EXPECT_EQ(copied->FullName(), table->FullName()); + EXPECT_EQ(copied->GetRootPath(), table->GetRootPath()); + EXPECT_EQ(copied->GetFileSystem(), table->GetFileSystem()); + // The original keeps the options it was loaded with. + EXPECT_EQ( + std::dynamic_pointer_cast(table->LatestSchema())->Options().at(Options::BUCKET), + "1"); + + // The copy is a table like the one it came from, and serves its data as one. + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_ctx, + ScanContextBuilder(dir->Str()).Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, copied->NewScan(std::move(scan_ctx))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_FALSE(plan->Splits().empty()); +} + TEST(TableTest, TestCreateBuildsTheOperator) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); diff --git a/src/paimon/rest/rest_catalog.cpp b/src/paimon/rest/rest_catalog.cpp index 1005b760d..dec534830 100644 --- a/src/paimon/rest/rest_catalog.cpp +++ b/src/paimon/rest/rest_catalog.cpp @@ -469,8 +469,12 @@ Result> RestCatalog::GetTable(const Identifier& identifie // tokens binds to this table. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table_file_system, GetTableFileSystem(identifier)); - return std::make_shared(table_file_system, table_path, schema, identifier, - uuid); + // The branch the identifier carries, which the schema and path above were read on, so the + // table and its managers serve it. + PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); + return std::make_shared( + table_file_system, table_path, schema, identifier, uuid, + NormalizeBranch(std::move(branch)).value_or(Identifier::kDefaultMainBranch)); } std::string RestCatalog::GetRootPath() const { diff --git a/src/paimon/rest/rest_catalog_test.cpp b/src/paimon/rest/rest_catalog_test.cpp index 43e579abb..4acd6796d 100644 --- a/src/paimon/rest/rest_catalog_test.cpp +++ b/src/paimon/rest/rest_catalog_test.cpp @@ -49,6 +49,7 @@ #include "paimon/core/core_options.h" #include "paimon/core/partition/partition_statistics.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/file_store_table.h" #include "paimon/core/table/system/system_table.h" #include "paimon/core/utils/snapshot_manager.h" #include "paimon/defs.h" @@ -1139,8 +1140,9 @@ TEST_F(RestCatalogTest, BranchTableLoadsBranchSchemaFromServer) { ASSERT_EQ(0, main_case_schema->Options().count(Options::BRANCH)); // the branch the alias spells resolves to the main branch, not to a branch // directory named after the alias - ASSERT_OK_AND_ASSIGN(std::string main_case_branch, main_case_table->CurrentBranch()); - ASSERT_EQ("main", main_case_branch); + auto main_case_file_store_table = std::dynamic_pointer_cast(main_case_table); + ASSERT_NE(nullptr, main_case_file_store_table); + ASSERT_EQ("main", main_case_file_store_table->CurrentBranch()); // a system table on a branch resolves against the branch's data table: the system // suffix is stripped while the branch stays in the identifier sent to the server diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp index 5de6c9fa7..fd9b49650 100644 --- a/test/inte/realtime_write_inte_test.cpp +++ b/test/inte/realtime_write_inte_test.cpp @@ -43,6 +43,7 @@ #include "gtest/gtest.h" #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" +#include "paimon/catalog/table.h" #include "paimon/commit_context.h" #include "paimon/common/data/variant/variant_type_utils.h" #include "paimon/common/table/special_fields.h" @@ -559,6 +560,41 @@ class RealtimeWriteInteTest : public ::testing::Test { return FileStoreWrite::Create(std::move(context)); } + // A writer that means to change an option the table was created with writes through a copy + // carrying it, which the context then agrees with. + Result> CopiedTable( + const std::map& overrides) const { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr catalog, Catalog::Create(dir_->Str(), {})); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, + catalog->GetTable(Identifier("foo", "bar"))); + return table->Copy(overrides); + } + + // Evolves the table schema to the current options with its fields unchanged, the way an + // ALTER TABLE changing an option would: the table itself then serves the changed option. + // The fields are carried over from the latest schema so their attributes survive: rebuilt + // raw fields would lose the non-nullable primary keys the table schema enforces. + Status EvolveSchemaWithOptions() const { + SchemaManager schema_manager(dir_->GetFileSystem(), table_path_); + PAIMON_ASSIGN_OR_RAISE(std::optional> latest, + schema_manager.Latest()); + if (!latest.has_value()) { + return Status::Invalid("table schema does not exist"); + } + return TestHelper::WriteNextSchema(dir_->GetFileSystem(), table_path_, + latest.value()->Fields(), + latest.value()->HighestFieldId(), options_); + } + + Result> CreateRealtimeWriter( + const std::shared_ptr& realtime_context, + const std::shared_ptr
& table) const { + WriteContextBuilder builder(table_path_, commit_user_); + builder.SetOptions(options_).WithStreamingMode(true).WithRealtimeContext(realtime_context); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); + return table->NewWrite(std::move(context)); + } + Result> CreateRealtimeWriter() const { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr realtime_context, RealtimeContext::Create()); @@ -1249,32 +1285,36 @@ TEST_F(RealtimeWriteInteTest, TestRealtimeOperationsRequireEnabledOption) { disabled_options[Options::REALTIME_ENABLED] = "false"; ASSERT_OK_AND_ASSIGN(std::shared_ptr realtime_context, RealtimeContext::Create()); + // The table was created enabled, so each operation below goes through a copy carrying it + // disabled, whose contexts then agree: each reports that real-time needs the option itself. + ASSERT_OK_AND_ASSIGN(std::shared_ptr
disabled_table, + CopiedTable({{Options::REALTIME_ENABLED, "false"}})); WriteContextBuilder write_builder(table_path_, commit_user_); write_builder.SetOptions(disabled_options) .WithStreamingMode(true) .WithRealtimeContext(realtime_context); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_context)), + ASSERT_NOK_WITH_MSG(disabled_table->NewWrite(std::move(write_context)), "real-time write requires realtime.enabled=true"); ScanContextBuilder scan_builder(table_path_); scan_builder.SetOptions(disabled_options).WithRealtimeContext(realtime_context); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), + ASSERT_NOK_WITH_MSG(disabled_table->NewScan(std::move(scan_context)), "real-time scan requires realtime.enabled=true"); ReadContextBuilder read_builder(table_path_); read_builder.SetOptions(disabled_options).WithRealtimeContext(realtime_context); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), + ASSERT_NOK_WITH_MSG(disabled_table->NewRead(std::move(read_context)), "real-time read requires realtime.enabled=true"); CommitContextBuilder commit_builder(table_path_, commit_user_); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.SetOptions(disabled_options).Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_context))); + disabled_table->NewCommit(std::move(commit_context))); ASSERT_NOK_WITH_MSG(commit->CommitWithProgress(/*realtime_commits=*/{}, /*commit_identifier=*/0, /*watermark=*/std::nullopt), @@ -2084,6 +2124,9 @@ TEST_F(RealtimeWriteInteTest, TestRealtimeDeduplicateAcrossMultipleFileIndexStor ASSERT_OK(embedded_writer->Close()); options_[Options::FILE_INDEX_IN_MANIFEST_THRESHOLD] = "1B"; + // The tighter threshold is a change the table itself carries, the way an ALTER TABLE would: + // the schema is evolved to it, so every later write, commit, and read serves it. + ASSERT_OK(EvolveSchemaWithOptions()); ASSERT_OK_AND_ASSIGN(std::unique_ptr external_writer, CreateRealtimeWriter(realtime_context)); ASSERT_OK_AND_ASSIGN(std::unique_ptr external_batch, @@ -3766,6 +3809,9 @@ TEST_F(RealtimeWriteInteTest, TestAppendRealtimeReadUsesEmbeddedAndExternalBitma ASSERT_OK(embedded_writer->Close()); options_[Options::FILE_INDEX_IN_MANIFEST_THRESHOLD] = "1B"; + // The tighter threshold is a change the table itself carries, the way an ALTER TABLE would: + // the schema is evolved to it, so every later write, commit, and read serves it. + ASSERT_OK(EvolveSchemaWithOptions()); ASSERT_OK_AND_ASSIGN(std::unique_ptr external_writer, CreateRealtimeWriter(realtime_context)); ASSERT_OK_AND_ASSIGN( @@ -4083,6 +4129,9 @@ TEST_F(RealtimeWriteInteTest, TestPkDvRealtimeReadUsesEmbeddedAndExternalBitmapI ASSERT_OK(embedded_writer->Close()); options_[Options::FILE_INDEX_IN_MANIFEST_THRESHOLD] = "1B"; + // The tighter threshold is a change the table itself carries, the way an ALTER TABLE would: + // the schema is evolved to it, so every later write, commit, and read serves it. + ASSERT_OK(EvolveSchemaWithOptions()); ASSERT_OK_AND_ASSIGN(std::unique_ptr external_writer, CreateRealtimeWriter(realtime_context)); ASSERT_OK_AND_ASSIGN(std::unique_ptr external_batch, diff --git a/test/inte/scan_and_read_inte_test.cpp b/test/inte/scan_and_read_inte_test.cpp index b7cc73b96..41d35de8d 100644 --- a/test/inte/scan_and_read_inte_test.cpp +++ b/test/inte/scan_and_read_inte_test.cpp @@ -36,6 +36,9 @@ #include "fmt/ranges.h" #include "gtest/gtest.h" #include "paimon/bucket/bucket_id_calculator.h" +#include "paimon/catalog/catalog.h" +#include "paimon/catalog/identifier.h" +#include "paimon/catalog/table.h" #include "paimon/common/factories/io_hook.h" #include "paimon/common/io/cache/lru_cache.h" #include "paimon/common/table/special_fields.h" @@ -796,19 +799,27 @@ TEST_P(ScanAndReadInteTest, TestWithAppendBucketKeyPointLookup) { } ASSERT_GE(other_index, 0); // These keys share a current two-bucket selection but need different historical buckets. + // The data was written with four buckets, so reading a rescaled layout reads through a + // copy of the table carrying the rescaled bucket count: an option on the context would + // contradict the table the catalog loaded. + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, + Catalog::Create(test_dir->Str(), {})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + catalog->GetTable(Identifier("foo", "bar"))); for (int32_t lookup_index : {32, other_index}) { - // The data was written with four buckets. Read it with rescaled table options too. for (const std::string current_bucket_count : {"2", "4", "8", "17"}) { SCOPED_TRACE(current_bucket_count); const std::string lookup_key = fmt::format("key{:03}", lookup_index); auto lookup_predicate = PredicateBuilder::Equal( 0, "rowkey", field_type, Literal(field_type, lookup_key.data(), lookup_key.size())); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
rescaled_table, + table->Copy({{Options::BUCKET, current_bucket_count}})); ScanContextBuilder scan_builder(table_path); scan_builder.SetPredicate(lookup_predicate) .AddOption(Options::BUCKET, current_bucket_count); ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_builder)); - ASSERT_OK_AND_ASSIGN(auto scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan, rescaled_table->NewScan(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto plan, scan->CreatePlan()); if (EnableSnapshotLiveManifestCache()) { ASSERT_OK_AND_ASSIGN( @@ -820,7 +831,7 @@ TEST_P(ScanAndReadInteTest, TestWithAppendBucketKeyPointLookup) { .AddOption(Options::BUCKET, current_bucket_count); ASSERT_OK_AND_ASSIGN(auto cached_context, FinishScanContext(cached_builder)); ASSERT_OK_AND_ASSIGN(auto cached_scan, - TableScan::Create(std::move(cached_context))); + rescaled_table->NewScan(std::move(cached_context))); ASSERT_OK_AND_ASSIGN(plan, cached_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(uint64_t hit, cached_scan->GetMetrics()->GetCounter( ScanMetrics::LAST_SNAPSHOT_CACHE_HIT)); @@ -3086,12 +3097,21 @@ TEST_P(ScanAndReadInteTest, TestScanAndReadWithDisableIndex) { PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, Literal(FieldType::STRING, "Alice0", 6)); + // The table was loaded with its index enabled, so the index-disabled pass below goes through + // a copy carrying it disabled: an option on the context would contradict the table. + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, + Catalog::Create(paimon::test::GetDataDir() + file_format, {})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + catalog->GetTable(Identifier("append_with_bitmap", "append_with_bitmap"))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
disabled_table, + table->Copy({{"file-index.read.enabled", "false"}})); + // scan ScanContextBuilder scan_context_builder(table_path); ASSERT_OK_AND_ASSIGN( auto scan_context, scan_context_builder.AddOption("file-index.read.enabled", "false").Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, disabled_table->NewScan(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -3102,7 +3122,7 @@ TEST_P(ScanAndReadInteTest, TestScanAndReadWithDisableIndex) { read_context_builder.AddOption("file-index.read.enabled", "false"); AddReadOptionsForPrefetch(&read_context_builder); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, disabled_table->NewRead(std::move(read_context))); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -3158,6 +3178,13 @@ TEST(ScanAndReadFileIndexInteTest, TestPkDvEmbeddedBitmapFiltersScanAndRead) { helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0, std::nullopt)); ASSERT_FALSE(commit_messages.empty()); + // The statistics-only baseline below disables the file index the table carries, so it scans + // through a copy with it disabled: an option on the context would contradict the table. + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, Catalog::Create(dir->Str(), {})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog->GetTable(Identifier("foo", "bar"))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
stats_only_table, + table->Copy({{Options::FILE_INDEX_READ_ENABLED, "false"}})); + // 15 falls inside the file's [10, 30] min/max range, so statistics alone must retain the // file. Disabling file-index reads verifies that baseline explicitly. const std::shared_ptr absent_predicate = PredicateBuilder::Equal( @@ -3169,7 +3196,7 @@ TEST(ScanAndReadFileIndexInteTest, TestPkDvEmbeddedBitmapFiltersScanAndRead) { ASSERT_OK_AND_ASSIGN(std::unique_ptr stats_only_scan_context, stats_only_scan_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr stats_only_scan, - TableScan::Create(std::move(stats_only_scan_context))); + stats_only_table->NewScan(std::move(stats_only_scan_context))); ASSERT_OK_AND_ASSIGN(std::shared_ptr stats_only_plan, stats_only_scan->CreatePlan()); ASSERT_FALSE(stats_only_plan->Splits().empty()); @@ -3262,6 +3289,13 @@ TEST(ScanAndReadFileIndexInteTest, TestPkDvReconstructsFileIndexPredicateAfterSc const std::shared_ptr predicate = PredicateBuilder::Equal( /*field_index=*/1, /*field_name=*/"renamed_value", FieldType::INT, Literal(15)); + // The statistics-only baseline below disables the file index the table carries, so it scans + // through a copy with it disabled: an option on the context would contradict the table. + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, Catalog::Create(dir->Str(), {})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog->GetTable(Identifier("foo", "bar"))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
stats_only_table, + table->Copy({{Options::FILE_INDEX_READ_ENABLED, "false"}})); + // The evolved min/max range is still [10, 30], so statistics alone retain the old file. ScanContextBuilder stats_only_scan_builder(table_path); stats_only_scan_builder.SetOptions(options) @@ -3270,7 +3304,7 @@ TEST(ScanAndReadFileIndexInteTest, TestPkDvReconstructsFileIndexPredicateAfterSc ASSERT_OK_AND_ASSIGN(std::unique_ptr stats_only_scan_context, stats_only_scan_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr stats_only_scan, - TableScan::Create(std::move(stats_only_scan_context))); + stats_only_table->NewScan(std::move(stats_only_scan_context))); ASSERT_OK_AND_ASSIGN(std::shared_ptr stats_only_plan, stats_only_scan->CreatePlan()); ASSERT_FALSE(stats_only_plan->Splits().empty()); @@ -4021,16 +4055,25 @@ TEST_P(ScanAndReadInteTest, TestWithPKBucketSelectByPredicate) { auto predicate = PredicateBuilder::Equal(/*field_index=*/2, /*field_name=*/"f2", FieldType::INT, Literal(static_cast(0))); - // Historical files use two buckets even when the current option has changed. + // Historical files use two buckets even when the current option has changed, so the scan + // goes through a copy of the table carrying the bucket count it selects by: an option on + // the context would contradict the table the catalog loaded. + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, + Catalog::Create(paimon::test::GetDataDir() + file_format, {})); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
table, + catalog->GetTable(Identifier("pk_table_scan_and_read_dv", "pk_table_scan_and_read_dv"))); for (const std::string current_bucket_count : {"2", "4", "8", "17"}) { SCOPED_TRACE(current_bucket_count); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
rescaled_table, + table->Copy({{Options::BUCKET, current_bucket_count}})); ScanContextBuilder scan_context_builder(table_path); scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "6"); scan_context_builder.AddOption(Options::BUCKET, current_bucket_count); scan_context_builder.SetPartitionFilter({{{"f1", "10"}}}); scan_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, rescaled_table->NewScan(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); AddReadOptionsForPrefetch(&read_context_builder); @@ -4049,7 +4092,8 @@ TEST_P(ScanAndReadInteTest, TestWithPKBucketSelectByPredicate) { .SetPartitionFilter({{{"f1", "10"}}}) .SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto cached_context, FinishScanContext(cached_builder)); - ASSERT_OK_AND_ASSIGN(auto cached_scan, TableScan::Create(std::move(cached_context))); + ASSERT_OK_AND_ASSIGN(auto cached_scan, + rescaled_table->NewScan(std::move(cached_context))); ASSERT_OK_AND_ASSIGN(result_plan, cached_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(uint64_t hit, cached_scan->GetMetrics()->GetCounter( ScanMetrics::LAST_SNAPSHOT_CACHE_HIT)); diff --git a/test/inte/scan_inte_test.cpp b/test/inte/scan_inte_test.cpp index 3f949484d..1502c39a7 100644 --- a/test/inte/scan_inte_test.cpp +++ b/test/inte/scan_inte_test.cpp @@ -32,6 +32,9 @@ #include "arrow/api.h" #include "gtest/gtest.h" +#include "paimon/catalog/catalog.h" +#include "paimon/catalog/identifier.h" +#include "paimon/catalog/table.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/data/data_define.h" #include "paimon/common/utils/decimal_utils.h" @@ -1161,10 +1164,18 @@ TEST_P(ScanInteTest, TestScanAppendWithInvalidOptions) { "scan.snapshot-id or scan.tag-name must be set when startup mode is FROM_SNAPSHOT"); } { + // `bucket` = -2 is refused by the scan itself, which a copy of the table carrying it + // reaches: an option on the context would contradict the table the catalog loaded, + // which a scan through that table refuses. + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, + Catalog::Create(paimon::test::GetDataDir() + "orc", {})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + catalog->GetTable(Identifier("append_09", "append_09"))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
copied, table->Copy({{Options::BUCKET, "-2"}})); ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::BUCKET, "-2").WithStreamingMode(true); + context_builder.WithStreamingMode(true); ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), + ASSERT_NOK_WITH_MSG(copied->NewScan(std::move(scan_context)), "do not support bucket=-2 in scan process"); } } @@ -1726,10 +1737,18 @@ TEST_P(ScanInteTest, TestScanPKWithSnapshot1WithBucketStats) { TEST_P(ScanInteTest, TestScanPKWithInvalidOptions) { std::string table_path = paimon::test::GetDataDir() + "orc/pk_table_with_total_buckets.db/pk_table_with_total_buckets"; + // Like the append case above, the invalid bucket count rides on a copy of the table: the + // scan refuses it where the bucket count is validated. + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, + Catalog::Create(paimon::test::GetDataDir() + "orc", {})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + catalog->GetTable(Identifier("pk_table_with_total_buckets", + "pk_table_with_total_buckets"))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
copied, table->Copy({{Options::BUCKET, "-1"}})); ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::BUCKET, "-1").WithStreamingMode(true); + context_builder.WithStreamingMode(true); ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), + ASSERT_NOK_WITH_MSG(copied->NewScan(std::move(scan_context)), "do not support pk table bucket=-1 in scan process"); } diff --git a/test/inte/write_and_read_inte_with_index_test.cpp b/test/inte/write_and_read_inte_with_index_test.cpp index efda3de03..9123a3385 100644 --- a/test/inte/write_and_read_inte_with_index_test.cpp +++ b/test/inte/write_and_read_inte_with_index_test.cpp @@ -1250,8 +1250,7 @@ TEST_P(WriteAndReadInteWithIndexTest, TestReadWithLimits) { PredicateBuilder::NotEqual(/*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, Literal(FieldType::STRING, "Alice", 5)); ReadContextBuilder context_builder(path); - context_builder.AddOption(Options::FILE_FORMAT, "orc") - .AddOption("test.enable-adaptive-prefetch-strategy", "false") + context_builder.AddOption("test.enable-adaptive-prefetch-strategy", "false") .AddOption(Options::READ_BATCH_SIZE, "1") .AddOption("orc.read.enable-metrics", "true") .SetPredicate(predicate); From 6eb4a363ed2c2dfdb96479971a4c1207ca0809c2 Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Thu, 22 Oct 2026 17:16:15 +0800 Subject: [PATCH 05/30] tmp --- src/paimon/CMakeLists.txt | 4 +- .../core/catalog/file_system_catalog.cpp | 5 +- .../table/append_only_file_store_table.cpp | 209 ++++++ .../core/table/append_only_file_store_table.h | 91 +++ src/paimon/core/table/file_store.cpp | 75 ++- src/paimon/core/table/file_store.h | 56 +- .../table/{table.cpp => file_store_table.cpp} | 596 ++++-------------- src/paimon/core/table/file_store_table.h | 124 +++- .../table/primary_key_file_store_table.cpp | 291 +++++++++ .../core/table/primary_key_file_store_table.h | 91 +++ src/paimon/core/table/table_test.cpp | 10 +- src/paimon/rest/rest_catalog.cpp | 2 +- src/paimon/testing/mock/mock_catalog.h | 2 +- 13 files changed, 1018 insertions(+), 538 deletions(-) create mode 100644 src/paimon/core/table/append_only_file_store_table.cpp create mode 100644 src/paimon/core/table/append_only_file_store_table.h rename src/paimon/core/table/{table.cpp => file_store_table.cpp} (66%) create mode 100644 src/paimon/core/table/primary_key_file_store_table.cpp create mode 100644 src/paimon/core/table/primary_key_file_store_table.h diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index b6ab5f92b..86ca0f444 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -443,9 +443,10 @@ set(PAIMON_CORE_SRCS core/stats/simple_stats_converter.cpp core/stats/simple_stats.cpp core/stats/simple_stats_evolution.cpp - core/table/table.cpp + core/table/append_only_file_store_table.cpp core/table/bucket_mode.cpp core/table/file_store.cpp + core/table/file_store_table.cpp core/table/format/format_file_listing.cpp core/table/format/format_file_naming.cpp core/table/format/format_path_validation.cpp @@ -457,6 +458,7 @@ set(PAIMON_CORE_SRCS core/table/format/format_table_scan.cpp core/table/format/format_table_write.cpp core/table/format/lazy_concat_batch_reader.cpp + core/table/primary_key_file_store_table.cpp core/table/sink/commit_message.cpp core/table/sink/commit_message_impl.cpp core/table/sink/commit_message_serializer.cpp diff --git a/src/paimon/core/catalog/file_system_catalog.cpp b/src/paimon/core/catalog/file_system_catalog.cpp index 36721a148..d3c6bd92e 100644 --- a/src/paimon/core/catalog/file_system_catalog.cpp +++ b/src/paimon/core/catalog/file_system_catalog.cpp @@ -350,9 +350,8 @@ Result> FileSystemCatalog::GetTable(const Identifier& ide // The branch the schema above was read on, which the table carries so its managers read // the same branch: `GetBranchNameOrDefault` folds any spelling of `main`. PAIMON_ASSIGN_OR_RAISE(std::string branch, identifier.GetBranchNameOrDefault()); - return std::make_shared(fs_, table_path, - checked_pointer_cast(*latest_schema), - identifier, /*uuid=*/"", branch); + return FileStoreTable::Create(fs_, table_path, checked_pointer_cast(*latest_schema), + identifier, /*uuid=*/"", branch); } Result> FileSystemCatalog::LoadSystemTable( diff --git a/src/paimon/core/table/append_only_file_store_table.cpp b/src/paimon/core/table/append_only_file_store_table.cpp new file mode 100644 index 000000000..0846aef90 --- /dev/null +++ b/src/paimon/core/table/append_only_file_store_table.cpp @@ -0,0 +1,209 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/table/append_only_file_store_table.h" + +#include +#include +#include +#include +#include + +#include "fmt/format.h" +#include "paimon/core/deletionvectors/bucketed_dv_maintainer.h" +#include "paimon/core/index/index_file_handler.h" +#include "paimon/core/operation/append_only_file_store_scan.h" +#include "paimon/core/operation/append_only_file_store_write.h" +#include "paimon/core/operation/commit/commit_scanner.h" +#include "paimon/core/operation/data_evolution_file_store_scan.h" +#include "paimon/core/operation/file_store_scan.h" +#include "paimon/core/realtime/realtime_schema_layout.h" +#include "paimon/core/snapshot.h" +#include "paimon/core/table/bucket_mode.h" +#include "paimon/core/table/file_store.h" +#include "paimon/core/table/source/append_only_split_generator.h" +#include "paimon/core/table/source/append_only_table_read.h" +#include "paimon/core/table/source/data_evolution_split_generator.h" +#include "paimon/core/table/source/split_generator.h" +#include "paimon/core/utils/snapshot_manager.h" +#include "paimon/result.h" +#include "paimon/status.h" +#include "paimon/write_context.h" + +namespace paimon { + +AppendOnlyFileStoreTable::AppendOnlyFileStoreTable( + const std::shared_ptr& file_system, const std::string& root_path, + const std::shared_ptr& schema, const Identifier& identifier, const std::string& uuid, + const std::string& branch, const std::shared_ptr& schema_manager, + const std::shared_ptr& snapshot_manager) + : FileStoreTable(file_system, root_path, schema, identifier, uuid, branch, schema_manager, + snapshot_manager) {} + +Result> AppendOnlyFileStoreTable::CreateWrite( + const WriteContext& ctx, const std::shared_ptr& schema, + const std::shared_ptr& arrow_schema, + const std::shared_ptr& partition_schema, const CoreOptions& options, + const std::shared_ptr& path_factory, + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& io_manager, bool ignore_previous_files) const { + // append table + const bool realtime_deduplicate = !options.GetRealtimeDeduplicateKeyFields().empty(); + bool need_dv_maintainer_factory = options.DeletionVectorsEnabled(); + if (options.GetBucket() == -1) { + need_dv_maintainer_factory = false; + ignore_previous_files = true; + } else if (options.GetBucket() <= 0) { + return Status::Invalid( + fmt::format("not support bucket {} in append table", options.GetBucket())); + } + if (ctx.GetRealtimeContext()) { + if (!ctx.GetWriteSchema().empty()) { + return Status::Invalid("real-time append write does not support a custom schema"); + } + if (!realtime_deduplicate) { + PAIMON_RETURN_NOT_OK(RestoreRealtimeCommittedProgress(ctx.GetRealtimeContext(), + snapshot_manager, options)); + } + } else if (realtime_deduplicate) { + return Status::Invalid("real-time deduplicate write requires a real-time context"); + } + std::shared_ptr write_schema = arrow_schema; + const auto& write_field_names = ctx.GetWriteSchema(); + if (!write_field_names.empty()) { + arrow::FieldVector write_fields; + write_fields.reserve(write_field_names.size()); + for (const auto& field_name : write_field_names) { + auto field = arrow_schema->GetFieldByName(field_name); + if (!field) { + // TODO(xinyu.lxy): support _ROW_ID and _SEQUENCE_NUMBER + return Status::Invalid( + fmt::format("write field {} does not exist in table schema", field_name)); + } + write_fields.push_back(field); + } + write_schema = arrow::schema(write_fields); + } + + std::shared_ptr realtime_schema_layout; + if (ctx.GetRealtimeContext()) { + const RealtimeStoreMode mode = + realtime_deduplicate ? RealtimeStoreMode::DEDUPLICATE : RealtimeStoreMode::APPEND_ONLY; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr layout, + RealtimeSchemaLayout::Create(mode, write_schema)); + realtime_schema_layout = std::move(layout); + } + + std::shared_ptr dv_maintainer_factory; + if (need_dv_maintainer_factory) { + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr index_file_handler, + store_->NewIndexFileHandler(options, path_factory, ctx.GetMemoryPool())); + dv_maintainer_factory = + std::make_shared(std::move(index_file_handler)); + } + + auto file_store_write = std::make_unique( + path_factory, snapshot_manager, GetSchemaManager(), ctx.GetCommitUser(), root_path_, schema, + arrow_schema, write_schema, realtime_schema_layout, partition_schema, dv_maintainer_factory, + io_manager, options, ignore_previous_files, ctx.IsStreamingMode(), + ctx.IgnoreNumBucketCheck(), ctx.GetRealtimeContext(), ctx.GetExecutor(), + ctx.GetMemoryPool()); + if (ctx.GetRealtimeContext() && realtime_deduplicate) { + PAIMON_ASSIGN_OR_RAISE(std::optional latest_snapshot, + snapshot_manager->LatestSnapshot()); + if (latest_snapshot) { + PAIMON_RETURN_NOT_OK(file_store_write->RefreshCommittedSnapshot(latest_snapshot->Id())); + } + } + return std::unique_ptr(std::move(file_store_write)); +} + +Result> AppendOnlyFileStoreTable::NewFileStoreScan( + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& arrow_schema, + const std::shared_ptr& table_schema, const CoreOptions& core_options, + const std::shared_ptr& executor, const std::shared_ptr& memory_pool, + const ScanContext* context) const { + // TODO(liancheng.lsz): support fallback branch in scan + std::unique_ptr scan; + if (core_options.DataEvolutionEnabled()) { + PAIMON_ASSIGN_OR_RAISE( + scan, DataEvolutionFileStoreScan::Create(GetSnapshotManager(), GetSchemaManager(), + manifest_list, manifest_file, table_schema, + arrow_schema, context->GetScanFilters(), + core_options, executor, memory_pool)); + } else { + PAIMON_ASSIGN_OR_RAISE( + scan, AppendOnlyFileStoreScan::Create(GetSnapshotManager(), GetSchemaManager(), + manifest_list, manifest_file, table_schema, + arrow_schema, context->GetScanFilters(), + core_options, executor, memory_pool)); + } + return WithTablePath(std::move(scan), root_path_); +} + +Result> AppendOnlyFileStoreTable::NewSplitGenerator( + const std::shared_ptr& table_schema, const CoreOptions& core_options) const { + auto source_split_target_size = core_options.GetSourceSplitTargetSize(); + auto source_split_open_file_cost = core_options.GetSourceSplitOpenFileCost(); + if (core_options.DataEvolutionEnabled()) { + return std::make_unique(source_split_target_size, + source_split_open_file_cost, + core_options.BlobSplitByFileSize()); + } + BucketMode bucket_mode = + (core_options.GetBucket() == -1 ? BucketMode::BUCKET_UNAWARE : BucketMode::HASH_FIXED); + return std::make_unique(source_split_target_size, + source_split_open_file_cost, bucket_mode); +} + +Result> AppendOnlyFileStoreTable::NewTableRead( + const std::shared_ptr& path_factory, + const std::shared_ptr& internal_context, + const std::shared_ptr& memory_pool, + const std::shared_ptr& executor) const { + return std::make_unique(path_factory, internal_context, memory_pool, + executor); +} + +CommitScanner::ScanSupplier AppendOnlyFileStoreTable::NewCommitScanSupplier( + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& table_schema, + const std::shared_ptr& arrow_schema, const CoreOptions& options, + const std::shared_ptr& executor, const std::shared_ptr& pool) const { + // The schema manager the table carries, which the scans this supplier hands out read the + // schema through. + const std::shared_ptr& schema_manager = GetSchemaManager(); + return [snapshot_manager, schema_manager, manifest_list, manifest_file, table_schema, + arrow_schema, options, executor, pool](const std::shared_ptr& scan_filter) + -> Result> { + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr scan, + AppendOnlyFileStoreScan::Create(snapshot_manager, schema_manager, manifest_list, + manifest_file, table_schema, arrow_schema, scan_filter, + options, executor, pool)); + return std::unique_ptr(std::move(scan)); + }; +} + +} // namespace paimon diff --git a/src/paimon/core/table/append_only_file_store_table.h b/src/paimon/core/table/append_only_file_store_table.h new file mode 100644 index 000000000..ea5374ba3 --- /dev/null +++ b/src/paimon/core/table/append_only_file_store_table.h @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include + +#include "paimon/catalog/identifier.h" +#include "paimon/core/table/file_store_table.h" +#include "paimon/core/utils/branch_manager.h" +#include "paimon/result.h" +#include "paimon/type_fwd.h" + +namespace paimon { + +class CoreOptions; +class FileStoreScan; +class IOManager; +class SplitGenerator; +class TableSchema; + +/// The managed table a schema without primary keys describes, Java Paimon's +/// `AppendOnlyFileStoreTable`. Its write appends the rows it is given, and its scan, read and +/// commit walk the files as they were written. The operator flows are the ones `FileStoreTable` +/// carries; the pieces those flows hand down are built here. +class AppendOnlyFileStoreTable : public FileStoreTable { + public: + /// The constructor `FileStoreTable::Create` builds an append-only table through; the + /// parameters are the ones it takes, and the base class carries them. + AppendOnlyFileStoreTable(const std::shared_ptr& file_system, + const std::string& root_path, const std::shared_ptr& schema, + const Identifier& identifier, const std::string& uuid = "", + const std::string& branch = BranchManager::DEFAULT_MAIN_BRANCH, + const std::shared_ptr& schema_manager = nullptr, + const std::shared_ptr& snapshot_manager = nullptr); + + private: + Result> CreateWrite( + const WriteContext& ctx, const std::shared_ptr& schema, + const std::shared_ptr& arrow_schema, + const std::shared_ptr& partition_schema, const CoreOptions& options, + const std::shared_ptr& path_factory, + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& io_manager, bool ignore_previous_files) const override; + + Result> NewFileStoreScan( + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& arrow_schema, + const std::shared_ptr& table_schema, const CoreOptions& core_options, + const std::shared_ptr& executor, const std::shared_ptr& memory_pool, + const ScanContext* context) const override; + + Result> NewSplitGenerator( + const std::shared_ptr& table_schema, + const CoreOptions& core_options) const override; + + Result> NewTableRead( + const std::shared_ptr& path_factory, + const std::shared_ptr& internal_context, + const std::shared_ptr& memory_pool, + const std::shared_ptr& executor) const override; + + CommitScanner::ScanSupplier NewCommitScanSupplier( + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& table_schema, + const std::shared_ptr& arrow_schema, const CoreOptions& options, + const std::shared_ptr& executor, + const std::shared_ptr& pool) const override; +}; + +} // namespace paimon diff --git a/src/paimon/core/table/file_store.cpp b/src/paimon/core/table/file_store.cpp index e70e85c1d..80b6afb05 100644 --- a/src/paimon/core/table/file_store.cpp +++ b/src/paimon/core/table/file_store.cpp @@ -26,12 +26,18 @@ #include "paimon/catalog/catalog.h" #include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/common/utils/binary_row_partition_computer.h" #include "paimon/core/catalog/version_managed_catalog.h" #include "paimon/core/core_options.h" +#include "paimon/core/index/index_file_handler.h" +#include "paimon/core/manifest/index_manifest_file.h" +#include "paimon/core/manifest/manifest_file.h" +#include "paimon/core/manifest/manifest_list.h" #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/table_schema.h" #include "paimon/core/utils/branch_manager.h" #include "paimon/core/utils/file_store_path_factory.h" +#include "paimon/core/utils/index_file_path_factories.h" #include "paimon/core/utils/snapshot_manager.h" #include "paimon/result.h" #include "paimon/status.h" @@ -62,8 +68,13 @@ Result> FileStore::NewPathFactory( const std::shared_ptr& arrow_schema, const CoreOptions& options, const std::string& file_format_identifier, const std::shared_ptr& memory_pool) const { - return MakePathFactory(root_path_, arrow_schema, schema_->PartitionKeys(), options, - file_format_identifier, memory_pool); + PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, options.CreateExternalPaths()); + PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, + options.CreateGlobalIndexExternalPath()); + return FileStorePathFactory::Create( + root_path_, arrow_schema, schema_->PartitionKeys(), options.GetPartitionDefaultName(), + file_format_identifier, options.DataFilePrefix(), options.LegacyPartitionNameEnabled(), + external_paths, global_index_external_path, options.IndexFileInDataFileDir(), memory_pool); } Result> FileStore::NewSnapshotManager( @@ -84,17 +95,55 @@ Result> FileStore::NewSnapshotManager( return snapshot_manager; } -Result> FileStore::MakePathFactory( - const std::string& root_path, const std::shared_ptr& arrow_schema, - const std::vector& partition_keys, const CoreOptions& options, - const std::string& file_format_identifier, const std::shared_ptr& memory_pool) { - PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, options.CreateExternalPaths()); - PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, - options.CreateGlobalIndexExternalPath()); - return FileStorePathFactory::Create( - root_path, arrow_schema, partition_keys, options.GetPartitionDefaultName(), - file_format_identifier, options.DataFilePrefix(), options.LegacyPartitionNameEnabled(), - external_paths, global_index_external_path, options.IndexFileInDataFileDir(), memory_pool); +Result> FileStore::NewManifestList( + const CoreOptions& options, const std::shared_ptr& path_factory, + const std::shared_ptr& memory_pool) const { + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr manifest_list, + ManifestList::Create(options.GetFileSystem(), options.GetManifestFormat(), + options.GetManifestCompression(), path_factory, options.GetCache(), + memory_pool)); + return manifest_list; +} + +Result> FileStore::NewManifestFile( + const CoreOptions& options, const std::shared_ptr& path_factory, + const std::shared_ptr& partition_schema, + const std::shared_ptr& memory_pool) const { + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr manifest_file, + ManifestFile::Create(options.GetFileSystem(), options.GetManifestFormat(), + options.GetManifestCompression(), path_factory, + options.GetManifestTargetFileSize(), memory_pool, options, + partition_schema)); + return manifest_file; +} + +Result> FileStore::NewIndexManifestFile( + const CoreOptions& options, const std::shared_ptr& path_factory, + const std::shared_ptr& memory_pool) const { + return IndexManifestFile::Create(options.GetFileSystem(), options.GetManifestFormat(), + options.GetManifestCompression(), path_factory, + options.GetBucket(), memory_pool, options); +} + +Result> FileStore::NewIndexFileHandler( + const CoreOptions& options, const std::shared_ptr& path_factory, + const std::shared_ptr& memory_pool) const { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr index_manifest_file, + NewIndexManifestFile(options, path_factory, memory_pool)); + return std::make_unique( + options.GetFileSystem(), std::move(index_manifest_file), + std::make_shared(path_factory), options.DeletionVectorsBitmap64(), + memory_pool); +} + +Result> FileStore::NewPartitionComputer( + const std::shared_ptr& arrow_schema, const CoreOptions& options, + const std::shared_ptr& memory_pool) const { + return BinaryRowPartitionComputer::Create(schema_->PartitionKeys(), arrow_schema, + options.GetPartitionDefaultName(), + options.LegacyPartitionNameEnabled(), memory_pool); } } // namespace paimon diff --git a/src/paimon/core/table/file_store.h b/src/paimon/core/table/file_store.h index ec3bc155c..83c34b1e7 100644 --- a/src/paimon/core/table/file_store.h +++ b/src/paimon/core/table/file_store.h @@ -22,11 +22,9 @@ #include #include #include -#include #include "paimon/catalog/identifier.h" #include "paimon/core/schema/schema_manager.h" -#include "paimon/core/utils/branch_manager.h" #include "paimon/core/utils/file_store_path_factory.h" #include "paimon/core/utils/snapshot_manager.h" #include "paimon/result.h" @@ -38,8 +36,13 @@ class Schema; namespace paimon { +class BinaryRowPartitionComputer; class Catalog; class CoreOptions; +class IndexFileHandler; +class IndexManifestFile; +class ManifestFile; +class ManifestList; class TableSchema; /// The storage a managed table is served through: the file system, root path, schema and branch @@ -86,9 +89,11 @@ class FileStore { } /// The path factory an operator builds its metadata paths through: over this store's root - /// path and the partition keys of its schema. The file format identifier is the caller's to - /// name: a write names the format its level-0 files are written in, which the options may - /// set per level, while a scan, a read and a commit name the table's format. + /// path and the partition keys of its schema - which no schema change can alter, so a read + /// serving a fallback branch's schema still names the same partitions. The file format + /// identifier is the caller's to name: a write names the format its level-0 files are + /// written in, which the options may set per level, while a scan, a read and a commit name + /// the table's format. Result> NewPathFactory( const std::shared_ptr& arrow_schema, const CoreOptions& options, const std::string& file_format_identifier, @@ -105,13 +110,40 @@ class FileStore { const CoreOptions& options, const std::shared_ptr& catalog, const std::optional& catalog_identifier) const; - /// The assembly the factory methods above share: a path factory over a root path and - /// partition keys named by the caller rather than the store's own, which the read path - /// needs - the schema it serves may be a fallback branch's, not the table's. - static Result> MakePathFactory( - const std::string& root_path, const std::shared_ptr& arrow_schema, - const std::vector& partition_keys, const CoreOptions& options, - const std::string& file_format_identifier, const std::shared_ptr& memory_pool); + /// The manifest list a scan or a commit reads the table's manifest entries through, over + /// the caller's path factory: an operator's manifests sit in the paths its path factory + /// names. + Result> NewManifestList( + const CoreOptions& options, const std::shared_ptr& path_factory, + const std::shared_ptr& memory_pool) const; + + /// The manifest file a scan or a commit reads a single manifest's entries through, over + /// the caller's path factory and the partition schema its entries are pruned by. + Result> NewManifestFile( + const CoreOptions& options, const std::shared_ptr& path_factory, + const std::shared_ptr& partition_schema, + const std::shared_ptr& memory_pool) const; + + /// The index manifest file a write, a scan or a commit reads the table's index manifest + /// through, over the caller's path factory. + Result> NewIndexManifestFile( + const CoreOptions& options, const std::shared_ptr& path_factory, + const std::shared_ptr& memory_pool) const; + + /// The index file handler a write's deletion vectors and indexes are maintained through, or + /// a scan reads them through: the index manifest file above and the path factories over the + /// caller's path factory, in one handler. + Result> NewIndexFileHandler( + const CoreOptions& options, const std::shared_ptr& path_factory, + const std::shared_ptr& memory_pool) const; + + /// The partition computer a commit names the partitions it writes with, over the partition + /// keys of this store's schema - which no schema change can alter, so a commit serving a + /// schema read after the table was loaded still names the same partitions - and the + /// caller's arrow schema and options. + Result> NewPartitionComputer( + const std::shared_ptr& arrow_schema, const CoreOptions& options, + const std::shared_ptr& memory_pool) const; private: std::shared_ptr file_system_; diff --git a/src/paimon/core/table/table.cpp b/src/paimon/core/table/file_store_table.cpp similarity index 66% rename from src/paimon/core/table/table.cpp rename to src/paimon/core/table/file_store_table.cpp index 33ff27d78..1c9769245 100644 --- a/src/paimon/core/table/table.cpp +++ b/src/paimon/core/table/file_store_table.cpp @@ -9,17 +9,16 @@ * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ -#include "paimon/catalog/table.h" +#include "paimon/core/table/file_store_table.h" -#include -#include #include #include #include @@ -27,18 +26,14 @@ #include #include -#include "arrow/c/bridge.h" #include "fmt/format.h" #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" #include "paimon/commit_context.h" -#include "paimon/common/global_index/btree/btree_defs.h" #include "paimon/common/predicate/predicate_validator.h" #include "paimon/common/types/data_field.h" -#include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/binary_row_partition_computer.h" #include "paimon/common/utils/checked_cast.h" -#include "paimon/common/utils/fields_comparator.h" #include "paimon/common/utils/options_utils.h" #include "paimon/common/utils/path_util.h" #include "paimon/common/utils/string_utils.h" @@ -51,57 +46,36 @@ #include "paimon/core/core_options.h" #include "paimon/core/disk/io_manager.h" #include "paimon/core/index/index_file_handler.h" -#include "paimon/core/index/pk/bucketed_primary_key_index_maintainer.h" #include "paimon/core/index/pk/primary_key_index_definitions.h" -#include "paimon/core/manifest/index_manifest_entry.h" #include "paimon/core/manifest/index_manifest_file.h" -#include "paimon/core/mergetree/compact/lookup_merge_function.h" -#include "paimon/core/mergetree/compact/merge_function.h" -#include "paimon/core/mergetree/compact/reducer_merge_function_wrapper.h" -#include "paimon/core/operation/append_only_file_store_scan.h" -#include "paimon/core/operation/append_only_file_store_write.h" #include "paimon/core/operation/commit/realtime_commit_properties.h" -#include "paimon/core/operation/data_evolution_file_store_scan.h" #include "paimon/core/operation/expire_snapshots.h" #include "paimon/core/operation/file_store_commit_impl.h" #include "paimon/core/operation/file_store_scan.h" #include "paimon/core/operation/internal_read_context.h" -#include "paimon/core/operation/key_value_file_store_scan.h" -#include "paimon/core/operation/key_value_file_store_write.h" -#include "paimon/core/options/merge_engine.h" #include "paimon/core/options/table_type.h" -#include "paimon/core/postpone/postpone_bucket_file_store_write.h" #include "paimon/core/realtime/realtime_context_impl.h" -#include "paimon/core/realtime/realtime_schema_layout.h" #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/schema_validation.h" #include "paimon/core/schema/table_schema.h" -#include "paimon/core/table/bucket_mode.h" +#include "paimon/core/snapshot.h" +#include "paimon/core/table/append_only_file_store_table.h" #include "paimon/core/table/file_store.h" -#include "paimon/core/table/file_store_table.h" -#include "paimon/core/table/source/abstract_table_scan.h" -#include "paimon/core/table/source/append_only_split_generator.h" -#include "paimon/core/table/source/append_only_table_read.h" +#include "paimon/core/table/primary_key_file_store_table.h" #include "paimon/core/table/source/data_evolution_batch_scan.h" -#include "paimon/core/table/source/data_evolution_split_generator.h" #include "paimon/core/table/source/data_table_batch_scan.h" #include "paimon/core/table/source/data_table_stream_scan.h" #include "paimon/core/table/source/fallback_table_read.h" -#include "paimon/core/table/source/key_value_table_read.h" -#include "paimon/core/table/source/merge_tree_split_generator.h" #include "paimon/core/table/source/primary_key_index_batch_scan.h" #include "paimon/core/table/source/read_optimized_scan_options.h" #include "paimon/core/table/source/realtime_table_scan.h" #include "paimon/core/table/source/snapshot/snapshot_reader.h" -#include "paimon/core/table/source/split_generator.h" #include "paimon/core/table/system/system_table.h" #include "paimon/core/utils/branch_manager.h" #include "paimon/core/utils/field_mapping.h" #include "paimon/core/utils/file_store_path_factory.h" -#include "paimon/core/utils/index_file_path_factories.h" #include "paimon/core/utils/primary_key_table_utils.h" #include "paimon/core/utils/snapshot_manager.h" -#include "paimon/defs.h" #include "paimon/file_store_commit.h" #include "paimon/file_store_write.h" #include "paimon/format/file_format.h" @@ -123,12 +97,9 @@ class Schema; } // namespace arrow namespace paimon { -class DataSplit; + class Executor; -struct KeyValue; class MemoryPool; -template -class MergeFunctionWrapper; namespace { @@ -171,46 +142,6 @@ Result> TryLoadFormatTable( /*location_carries_paimon_metadata=*/true, options); } -Status RestoreRealtimeCommittedProgress(const std::shared_ptr& realtime_context, - const std::shared_ptr& snapshot_manager, - const CoreOptions& options) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr realtime_context_impl, - RealtimeContextImpl::Cast(realtime_context)); - PAIMON_RETURN_NOT_OK(realtime_context_impl->CheckUsable()); - PAIMON_ASSIGN_OR_RAISE(std::optional latest_snapshot, - snapshot_manager->LatestSnapshot()); - if (latest_snapshot) { - PAIMON_ASSIGN_OR_RAISE( - RealtimeOffsetMap realtime_committed_offsets, - RealtimeCommitProperties::ReadOffsets(latest_snapshot, options.GetFileSystem())); - PAIMON_RETURN_NOT_OK(realtime_context_impl->AdvanceCommittedProgress( - latest_snapshot->Id(), realtime_committed_offsets, /*committed_data_files=*/{})); - } - return Status::OK(); -} - -Result HasHistoricalPrimaryKeyBTreeDefinition( - const std::shared_ptr& schema_manager, int64_t current_schema_id) { - if (current_schema_id <= TableSchema::FIRST_SCHEMA_ID) { - return false; - } - PAIMON_ASSIGN_OR_RAISE(std::vector schema_ids, schema_manager->ListAllIds()); - for (int64_t schema_id : schema_ids) { - if (schema_id >= current_schema_id) { - continue; - } - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr historical_schema, - schema_manager->ReadSchema(schema_id)); - const auto& historical_options = historical_schema->Options(); - auto option = historical_options.find(Options::PK_BTREE_INDEX_COLUMNS); - if (option != historical_options.end() && - !StringUtils::IsNullOrWhitespaceOnly(option->second)) { - return true; - } - } - return false; -} - /// The tail every `TableOf` ends with: the schema one of them read says what the path names. A /// format table is handed back as the `FormatTable` it is, everything else as the /// `FileStoreTable` a managed table is, carrying the schema for its `NewXxx` to build from. A @@ -231,8 +162,8 @@ Result> TableOfLoadedSchema( // it sits in - the same convention `TryLoadFormatTable` follows above. Identifier table_identifier = identifier.value_or(Identifier(Identifier::kUnknownDatabase, PathUtil::GetName(path))); - return std::make_shared(file_system, path, latest_schema, table_identifier, - uuid, branch, schema_manager); + return FileStoreTable::Create(file_system, path, latest_schema, table_identifier, uuid, branch, + schema_manager); } /// The managed table a context names behind a path: the schema is read through a manager of the @@ -431,8 +362,8 @@ Result> TableOf(const CommitContext& ctx) { MakeBranchCatalogIdentifier(ctx.GetIdentifier(), branch)); // Catalog-managed schemas may be absent from the table directory, but only on the main branch: // a branch publishes its own under `branch/branch-`, which is where the load below and a - // read of the branch both read it, so a catalog answering for no branch schema does not fail - // a commit to one. + // read of the branch both read it, so a catalog answering for no branch schema does not fail a + // commit to one. PAIMON_ASSIGN_OR_RAISE( std::optional catalog_table_schema, CatalogMainBranchSchemaJson(ctx.GetCatalog(), catalog_identifier, branch, "commit")); @@ -528,6 +459,25 @@ FileStoreTable::FileStoreTable(const std::shared_ptr& file_system, std::static_pointer_cast(schema), branch, schema_manager, snapshot_manager)) {} +Result> FileStoreTable::Create( + const std::shared_ptr& file_system, const std::string& root_path, + const std::shared_ptr& schema, const Identifier& identifier, const std::string& uuid, + const std::string& branch, const std::shared_ptr& schema_manager, + const std::shared_ptr& snapshot_manager) { + // The kind the schema's primary keys name, which no schema change can add or drop, so the + // kind the table is built as is the kind every schema its root path ever held describes. A + // caller without a schema gets the append-only kind, whose operators then report that no + // schema was found. + std::shared_ptr table_schema = std::static_pointer_cast(schema); + if (table_schema != nullptr && !table_schema->PrimaryKeys().empty()) { + return std::make_shared(file_system, root_path, schema, + identifier, uuid, branch, schema_manager, + snapshot_manager); + } + return std::make_shared( + file_system, root_path, schema, identifier, uuid, branch, schema_manager, snapshot_manager); +} + Result> FileStoreTable::Copy( const std::map& dynamic_options) const { std::shared_ptr loaded_schema = std::static_pointer_cast(schema_); @@ -542,10 +492,10 @@ Result> FileStoreTable::Copy( loaded_schema->MergeOptions(overridable)); // The managers are handed over, not rebuilt: the copy serves the same root path and branch, // so its operators read the schema cache this table already warmed. - return std::shared_ptr
(std::make_shared( - file_system_, root_path_, std::shared_ptr(std::move(merged_schema)), - identifier_, CatalogUuid().value_or(""), CurrentBranch(), GetSchemaManager(), - GetSnapshotManager())); + return FileStoreTable::Create(file_system_, root_path_, + std::shared_ptr(std::move(merged_schema)), + identifier_, CatalogUuid().value_or(""), CurrentBranch(), + GetSchemaManager(), GetSnapshotManager()); } Result> FileStoreTable::SwitchToBranch(const std::string& branch) const { @@ -553,9 +503,9 @@ Result> FileStoreTable::SwitchToBranch(const std::string& if (target_branch == CurrentBranch()) { // The table already serves this branch: a copy of it - schema and managers shared - is // the same table to the caller. - return std::shared_ptr
(std::make_shared( - file_system_, root_path_, schema_, identifier_, CatalogUuid().value_or(""), - CurrentBranch(), GetSchemaManager(), GetSnapshotManager())); + return FileStoreTable::Create(file_system_, root_path_, schema_, identifier_, + CatalogUuid().value_or(""), CurrentBranch(), + GetSchemaManager(), GetSnapshotManager()); } // The schema of the branch switched to, read from its own directory, which a branch without // one does not have. @@ -573,9 +523,33 @@ Result> FileStoreTable::SwitchToBranch(const std::string& } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr branch_schema, (*latest)->MergeOptions(overridable)); - return std::shared_ptr
(std::make_shared( + return FileStoreTable::Create( file_system_, root_path_, std::shared_ptr(std::move(branch_schema)), - identifier_, CatalogUuid().value_or(""), target_branch, schema_manager)); + identifier_, CatalogUuid().value_or(""), target_branch, schema_manager); +} + +Status FileStoreTable::RestoreRealtimeCommittedProgress( + const std::shared_ptr& realtime_context, + const std::shared_ptr& snapshot_manager, const CoreOptions& options) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr realtime_context_impl, + RealtimeContextImpl::Cast(realtime_context)); + PAIMON_RETURN_NOT_OK(realtime_context_impl->CheckUsable()); + PAIMON_ASSIGN_OR_RAISE(std::optional latest_snapshot, + snapshot_manager->LatestSnapshot()); + if (latest_snapshot) { + PAIMON_ASSIGN_OR_RAISE( + RealtimeOffsetMap realtime_committed_offsets, + RealtimeCommitProperties::ReadOffsets(latest_snapshot, options.GetFileSystem())); + PAIMON_RETURN_NOT_OK(realtime_context_impl->AdvanceCommittedProgress( + latest_snapshot->Id(), realtime_committed_offsets, /*committed_data_files=*/{})); + } + return Status::OK(); +} + +std::unique_ptr FileStoreTable::WithTablePath(std::unique_ptr&& scan, + const std::string& root_path) { + scan->WithTablePath(root_path); + return std::move(scan); } Result> FileStoreTable::NewWrite( @@ -640,209 +614,9 @@ Result> FileStoreTable::NewWrite( RealtimeContextImpl::Cast(ctx->GetRealtimeContext())); PAIMON_RETURN_NOT_OK(realtime_context_impl->CheckUsable()); } - if (schema->PrimaryKeys().empty()) { - // append table - const bool realtime_deduplicate = !options.GetRealtimeDeduplicateKeyFields().empty(); - bool need_dv_maintainer_factory = options.DeletionVectorsEnabled(); - if (options.GetBucket() == -1) { - need_dv_maintainer_factory = false; - ignore_previous_files = true; - } else if (options.GetBucket() <= 0) { - return Status::Invalid( - fmt::format("not support bucket {} in append table", options.GetBucket())); - } - if (ctx->GetRealtimeContext()) { - if (!ctx->GetWriteSchema().empty()) { - return Status::Invalid("real-time append write does not support a custom schema"); - } - if (!realtime_deduplicate) { - PAIMON_RETURN_NOT_OK(RestoreRealtimeCommittedProgress(ctx->GetRealtimeContext(), - snapshot_manager, options)); - } - } else if (realtime_deduplicate) { - return Status::Invalid("real-time deduplicate write requires a real-time context"); - } - std::shared_ptr write_schema = arrow_schema; - const auto& write_field_names = ctx->GetWriteSchema(); - if (!write_field_names.empty()) { - arrow::FieldVector write_fields; - write_fields.reserve(write_field_names.size()); - for (const auto& field_name : write_field_names) { - auto field = arrow_schema->GetFieldByName(field_name); - if (!field) { - // TODO(xinyu.lxy): support _ROW_ID and _SEQUENCE_NUMBER - return Status::Invalid( - fmt::format("write field {} does not exist in table schema", field_name)); - } - write_fields.push_back(field); - } - write_schema = arrow::schema(write_fields); - } - - std::shared_ptr realtime_schema_layout; - if (ctx->GetRealtimeContext()) { - const RealtimeStoreMode mode = realtime_deduplicate ? RealtimeStoreMode::DEDUPLICATE - : RealtimeStoreMode::APPEND_ONLY; - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr layout, - RealtimeSchemaLayout::Create(mode, write_schema)); - realtime_schema_layout = std::move(layout); - } - - std::shared_ptr dv_maintainer_factory; - if (need_dv_maintainer_factory) { - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr index_manifest_file, - IndexManifestFile::Create(options.GetFileSystem(), options.GetManifestFormat(), - options.GetManifestCompression(), file_store_path_factory, - options.GetBucket(), ctx->GetMemoryPool(), options)); - auto index_file_handler = std::make_shared( - options.GetFileSystem(), std::move(index_manifest_file), - std::make_shared(file_store_path_factory), - options.DeletionVectorsBitmap64(), ctx->GetMemoryPool()); - dv_maintainer_factory = - std::make_shared(index_file_handler); - } - - auto file_store_write = std::make_unique( - file_store_path_factory, snapshot_manager, GetSchemaManager(), ctx->GetCommitUser(), - root_path_, schema, arrow_schema, write_schema, realtime_schema_layout, - partition_schema, dv_maintainer_factory, io_manager, options, ignore_previous_files, - ctx->IsStreamingMode(), ctx->IgnoreNumBucketCheck(), ctx->GetRealtimeContext(), - ctx->GetExecutor(), ctx->GetMemoryPool()); - if (ctx->GetRealtimeContext() && realtime_deduplicate) { - PAIMON_ASSIGN_OR_RAISE(std::optional latest_snapshot, - snapshot_manager->LatestSnapshot()); - if (latest_snapshot) { - PAIMON_RETURN_NOT_OK( - file_store_write->RefreshCommittedSnapshot(latest_snapshot->Id())); - } - } - return std::unique_ptr(std::move(file_store_write)); - } else { - // pk table - if (ctx->GetRealtimeContext()) { - PAIMON_RETURN_NOT_OK(PrimaryKeyTableUtils::ValidateRealtimeOptions(options, *schema)); - if (ignore_previous_files) { - return Status::NotImplemented( - "PK realtime requires restore from the latest snapshot"); - } - if (!ctx->GetWriteSchema().empty()) { - return Status::NotImplemented("PK realtime does not support a custom write schema"); - } - PAIMON_RETURN_NOT_OK(RestoreRealtimeCommittedProgress(ctx->GetRealtimeContext(), - snapshot_manager, options)); - } - if (options.GetBucket() == BucketModeDefine::POSTPONE_BUCKET) { - return PostponeBucketFileStoreWrite::Create( - snapshot_manager, GetSchemaManager(), ctx->GetCommitUser(), root_path_, schema, - arrow_schema, partition_schema, io_manager, options, ctx->IsStreamingMode(), - ctx->IgnoreNumBucketCheck(), ctx->GetWriteId(), - ctx->GetFileSystemSchemeToIdentifierMap(), ctx->GetExecutor(), ctx->GetMemoryPool(), - file_system_); - } - if (options.GetBucket() <= 0) { - return Status::Invalid( - fmt::format("not support bucket {} in key value table", options.GetBucket())); - } - PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_keys, - schema->TrimmedPrimaryKeys()); - PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_key_fields, - schema->GetFields(trimmed_primary_keys)); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr key_comparator, - FieldsComparator::Create(trimmed_primary_key_fields, - options.SequenceFieldSortOrderIsAscending())); - auto primary_keys = schema->PrimaryKeys(); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merge_function, - PrimaryKeyTableUtils::CreateMergeFunction( - arrow_schema, primary_keys, options, ctx->GetMemoryPool())); - if (options.NeedLookup() && options.GetMergeEngine() != MergeEngine::FIRST_ROW) { - // don't wrap first row, it is already OK - merge_function = std::make_unique(std::move(merge_function)); - } - std::shared_ptr> merge_function_wrapper = - std::make_shared(std::move(merge_function)); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr sequence_fields_comparator, - PrimaryKeyTableUtils::CreateSequenceFieldsComparator(schema->Fields(), options)); - - std::shared_ptr dv_maintainer_factory; - PAIMON_ASSIGN_OR_RAISE(PrimaryKeyIndexDefinitions primary_key_index_definitions, - PrimaryKeyIndexDefinitions::Create(*schema)); - bool has_btree_index = false; - for (const PrimaryKeyIndexDefinition& definition : - primary_key_index_definitions.Definitions()) { - if (definition.GetFamily() == PrimaryKeyIndexDefinition::Family::BTREE) { - has_btree_index = true; - break; - } - } - std::optional latest_snapshot; - bool has_existing_index_manifest = false; - if (!has_btree_index) { - PAIMON_ASSIGN_OR_RAISE( - bool has_historical_btree_index, - HasHistoricalPrimaryKeyBTreeDefinition(GetSchemaManager(), schema->Id())); - if (has_historical_btree_index) { - PAIMON_ASSIGN_OR_RAISE(latest_snapshot, snapshot_manager->LatestSnapshot()); - has_existing_index_manifest = - latest_snapshot.has_value() && latest_snapshot->IndexManifest().has_value(); - } - } - std::shared_ptr index_file_handler; - if (options.DeletionVectorsEnabled() || has_btree_index || has_existing_index_manifest) { - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr index_manifest_file, - IndexManifestFile::Create(options.GetFileSystem(), options.GetManifestFormat(), - options.GetManifestCompression(), file_store_path_factory, - options.GetBucket(), ctx->GetMemoryPool(), options)); - index_file_handler = std::make_shared( - options.GetFileSystem(), std::move(index_manifest_file), - std::make_shared(file_store_path_factory), - options.DeletionVectorsBitmap64(), ctx->GetMemoryPool()); - } - bool has_restored_btree_index = false; - if (!has_btree_index && has_existing_index_manifest) { - PAIMON_ASSIGN_OR_RAISE( - std::vector restored_btree_entries, - index_file_handler->Scan( - latest_snapshot.value(), [](const IndexManifestEntry& entry) -> bool { - return entry.index_file->IndexType() == BtreeDefs::kIdentifier && - IndexFileHandler::IsPrimaryKeySourceIndex(*entry.index_file); - })); - has_restored_btree_index = !restored_btree_entries.empty(); - } - if (options.DeletionVectorsEnabled()) { - dv_maintainer_factory = - std::make_shared(index_file_handler); - } - std::shared_ptr - primary_key_index_maintainer_factory; - if (has_btree_index || has_restored_btree_index) { - primary_key_index_maintainer_factory = - std::make_shared( - root_path_, CurrentBranch(), schema, - primary_key_index_definitions.Definitions(), file_store_path_factory, - index_file_handler, options, io_manager, ctx->EnableMultiThreadSpill(), - ctx->GetExecutor(), ctx->GetMemoryPool()); - } - - std::shared_ptr realtime_schema_layout; - if (ctx->GetRealtimeContext()) { - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr layout, - RealtimeSchemaLayout::Create(RealtimeStoreMode::PRIMARY_KEY, arrow_schema)); - realtime_schema_layout = std::move(layout); - } - - return std::make_unique( - file_store_path_factory, snapshot_manager, GetSchemaManager(), ctx->GetCommitUser(), - root_path_, schema, arrow_schema, realtime_schema_layout, partition_schema, - dv_maintainer_factory, primary_key_index_maintainer_factory, io_manager, key_comparator, - sequence_fields_comparator, merge_function_wrapper, options, ignore_previous_files, - ctx->IsStreamingMode(), ctx->IgnoreNumBucketCheck(), ctx->EnableMultiThreadSpill(), - ctx->GetRealtimeContext(), ctx->GetExecutor(), ctx->GetMemoryPool()); - } + return CreateWrite(*ctx, schema, arrow_schema, partition_schema, options, + file_store_path_factory, snapshot_manager, io_manager, + ignore_previous_files); } Result> FileStoreWrite::Create(std::unique_ptr ctx) { @@ -854,110 +628,6 @@ Result> FileStoreWrite::Create(std::unique_ptr> CreateFileStoreScan( - const std::shared_ptr& path_factory, - const std::shared_ptr& arrow_schema, - const std::shared_ptr& table_schema, const CoreOptions& core_options, - const std::shared_ptr& snapshot_manager, - const std::shared_ptr& schema_manager, - const std::shared_ptr& executor, const std::shared_ptr& memory_pool, - const ScanContext* context, const std::string& root_path) { - auto fs = core_options.GetFileSystem(); - auto manifest_file_format = core_options.GetManifestFormat(); - // TODO(liancheng.lsz): support fallback branch in scan - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr manifest_list, - ManifestList::Create(fs, manifest_file_format, core_options.GetManifestCompression(), - path_factory, core_options.GetCache(), memory_pool)); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr partition_schema, - FieldMapping::GetPartitionSchema(arrow_schema, table_schema->PartitionKeys())); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr manifest_file, - ManifestFile::Create(fs, manifest_file_format, core_options.GetManifestCompression(), - path_factory, core_options.GetManifestTargetFileSize(), - memory_pool, core_options, partition_schema)); - std::unique_ptr scan; - if (table_schema->PrimaryKeys().empty()) { - if (core_options.DataEvolutionEnabled()) { - PAIMON_ASSIGN_OR_RAISE( - scan, DataEvolutionFileStoreScan::Create( - snapshot_manager, schema_manager, manifest_list, manifest_file, - table_schema, arrow_schema, context->GetScanFilters(), core_options, - executor, memory_pool)); - } else { - PAIMON_ASSIGN_OR_RAISE( - scan, AppendOnlyFileStoreScan::Create( - snapshot_manager, schema_manager, manifest_list, manifest_file, - table_schema, arrow_schema, context->GetScanFilters(), core_options, - executor, memory_pool)); - } - } else { - PAIMON_ASSIGN_OR_RAISE( - scan, KeyValueFileStoreScan::Create(snapshot_manager, schema_manager, manifest_list, - manifest_file, table_schema, arrow_schema, - context->GetScanFilters(), core_options, - executor, memory_pool)); - } - return WithTablePath(std::move(scan), root_path); - } - - static std::unique_ptr WithTablePath(std::unique_ptr&& scan, - const std::string& root_path) { - scan->WithTablePath(root_path); - return std::move(scan); - } - - static Result> CreateSplitGenerator( - const std::shared_ptr& table_schema, const CoreOptions& core_options, - const ScanContext* context) { - auto source_split_target_size = core_options.GetSourceSplitTargetSize(); - auto source_split_open_file_cost = core_options.GetSourceSplitOpenFileCost(); - if (table_schema->PrimaryKeys().empty()) { - if (core_options.DataEvolutionEnabled()) { - return std::make_unique( - source_split_target_size, source_split_open_file_cost, - core_options.BlobSplitByFileSize()); - } - BucketMode bucket_mode = (core_options.GetBucket() == -1 ? BucketMode::BUCKET_UNAWARE - : BucketMode::HASH_FIXED); - return std::make_unique( - source_split_target_size, source_split_open_file_cost, bucket_mode); - } else { - // TODO(liancheng.lsz): support evolution - PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_keys, - table_schema->TrimmedPrimaryKeys()); - PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_pk_fields, - table_schema->GetFields(trimmed_primary_keys)); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr key_comparator, - FieldsComparator::Create(trimmed_pk_fields, /*is_ascending_order=*/true)); - return std::make_unique( - source_split_target_size, source_split_open_file_cost, - core_options.DeletionVectorsEnabled(), core_options.GetMergeEngine(), - key_comparator); - } - } - - static Result> CreateIndexFileHandler( - const CoreOptions& core_options, const std::shared_ptr& path_factory, - const std::shared_ptr& memory_pool) { - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr index_manifest_file, - IndexManifestFile::Create( - core_options.GetFileSystem(), core_options.GetManifestFormat(), - core_options.GetManifestCompression(), path_factory, - core_options.GetBucket(), memory_pool, core_options)); - return std::make_unique( - core_options.GetFileSystem(), std::move(index_manifest_file), - std::make_shared(path_factory), - core_options.DeletionVectorsBitmap64(), memory_pool); - } -}; - Status ValidateRealtimeScan(const TableSchema& table_schema, const CoreOptions& core_options, const ScanContext& context, bool read_optimized) { if (!context.GetRealtimeContext()) { @@ -1065,18 +735,26 @@ Result> FileStoreTable::NewScan( core_options.GetFileFormat()->Identifier(), context->GetMemoryPool())); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr manifest_list, + store_->NewManifestList(core_options, path_factory, context->GetMemoryPool())); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr partition_schema, + FieldMapping::GetPartitionSchema(arrow_schema, table_schema->PartitionKeys())); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr manifest_file, + store_->NewManifestFile(core_options, path_factory, partition_schema, + context->GetMemoryPool())); + // The managers the table carries, whose schema cache the dispatch already warmed. - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr file_store_scan, - TableScanImpl::CreateFileStoreScan( - path_factory, arrow_schema, table_schema, core_options, - GetSnapshotManager(), GetSchemaManager(), context->GetExecutor(), - context->GetMemoryPool(), context.get(), root_path_)); PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr split_generator, - TableScanImpl::CreateSplitGenerator(table_schema, core_options, context.get())); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr index_file_handler, - TableScanImpl::CreateIndexFileHandler(core_options, path_factory, - context->GetMemoryPool())); + std::shared_ptr file_store_scan, + NewFileStoreScan(manifest_list, manifest_file, arrow_schema, table_schema, core_options, + context->GetExecutor(), context->GetMemoryPool(), context.get())); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr split_generator, + NewSplitGenerator(table_schema, core_options)); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr index_file_handler, + store_->NewIndexFileHandler(core_options, path_factory, context->GetMemoryPool())); auto snapshot_reader = std::make_shared( file_store_scan, path_factory, std::move(split_generator), std::move(index_file_handler)); const bool pk_table = !table_schema->PrimaryKeys().empty(); @@ -1172,9 +850,12 @@ Result> CreateInternalReadContext( return InternalReadContext::Create(context, table_schema, options, file_system, &root_path); } -Result> CreateTableRead( +} // namespace + +Result> FileStoreTable::CreateTableRead( const std::shared_ptr& internal_context, - const std::shared_ptr& memory_pool, const std::shared_ptr& executor) { + const std::shared_ptr& memory_pool, + const std::shared_ptr& executor) const { if (PredicateUtils::ContainsFullTextMatch(internal_context->GetPredicate())) { return Status::NotImplemented("FullTextMatch predicate execution is not implemented yet"); } @@ -1190,19 +871,11 @@ Result> CreateTableRead( auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr path_factory, - FileStore::MakePathFactory(internal_context->GetPath(), arrow_schema, - table_schema->PartitionKeys(), core_options, - core_options.GetFileFormat()->Identifier(), memory_pool)); - - if (internal_context->GetPrimaryKeys().empty()) { - return std::make_unique(path_factory, internal_context, memory_pool, - executor); - } - return KeyValueTableRead::Create(path_factory, internal_context, memory_pool, executor); + store_->NewPathFactory(arrow_schema, core_options, + core_options.GetFileFormat()->Identifier(), memory_pool)); + return NewTableRead(path_factory, internal_context, memory_pool, executor); } -} // namespace - Result> FileStoreTable::NewRead( std::unique_ptr context) const { std::shared_ptr read_context = std::move(context); @@ -1259,46 +932,6 @@ Result> TableRead::Create(std::unique_ptr& snapshot_manager, - const std::shared_ptr& schema_manager, - const std::shared_ptr& manifest_list, - const std::shared_ptr& manifest_file, - const std::shared_ptr& table_schema, - const std::shared_ptr& arrow_schema, const CoreOptions& options, - const std::shared_ptr& executor, const std::shared_ptr& pool) { - return [snapshot_manager, schema_manager, manifest_list, manifest_file, table_schema, - arrow_schema, options, executor, pool](const std::shared_ptr& scan_filter) - -> Result> { - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr scan, - AppendOnlyFileStoreScan::Create(snapshot_manager, schema_manager, manifest_list, - manifest_file, table_schema, arrow_schema, scan_filter, - options, executor, pool)); - return std::unique_ptr(std::move(scan)); - }; -} - -CommitScanner::ScanSupplier CreatePkScanSupplier( - const std::shared_ptr& snapshot_manager, - const std::shared_ptr& schema_manager, - const std::shared_ptr& manifest_list, - const std::shared_ptr& manifest_file, - const std::shared_ptr& table_schema, - const std::shared_ptr& arrow_schema, const CoreOptions& options, - const std::shared_ptr& executor, const std::shared_ptr& pool) { - return [snapshot_manager, schema_manager, manifest_list, manifest_file, table_schema, - arrow_schema, options, executor, pool](const std::shared_ptr& scan_filter) - -> Result> { - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr scan, - KeyValueFileStoreScan::Create(snapshot_manager, schema_manager, manifest_list, - manifest_file, table_schema, arrow_schema, scan_filter, - options, executor, pool)); - return std::unique_ptr(std::move(scan)); - }; -} - bool IsCatalogCommit(const CommitContext& ctx) { return AsVersionManaged(ctx.GetCatalog()) != nullptr; } @@ -1404,11 +1037,9 @@ Result> FileStoreTable::NewCommit( return Status::NotImplemented( "commit operation does not support object store file system for now"); } - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr partition_computer, - BinaryRowPartitionComputer::Create(schema->PartitionKeys(), arrow_schema, - options.GetPartitionDefaultName(), - options.LegacyPartitionNameEnabled(), - ctx->GetMemoryPool())); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr partition_computer, + store_->NewPartitionComputer(arrow_schema, options, ctx->GetMemoryPool())); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr path_factory, store_->NewPathFactory(arrow_schema, options, options.GetFileFormat()->Identifier(), @@ -1417,41 +1048,26 @@ Result> FileStoreTable::NewCommit( PAIMON_ASSIGN_OR_RAISE( std::shared_ptr snapshot_manager, store_->NewSnapshotManager(options, ctx->GetCatalog(), catalog_identifier)); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr manifest_list, - ManifestList::Create(options.GetFileSystem(), options.GetManifestFormat(), - options.GetManifestCompression(), path_factory, options.GetCache(), - ctx->GetMemoryPool())); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr manifest_list, + store_->NewManifestList(options, path_factory, ctx->GetMemoryPool())); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr partition_schema, FieldMapping::GetPartitionSchema(arrow_schema, schema->PartitionKeys())); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr manifest_file, - ManifestFile::Create(options.GetFileSystem(), options.GetManifestFormat(), - options.GetManifestCompression(), path_factory, - options.GetManifestTargetFileSize(), ctx->GetMemoryPool(), options, - partition_schema)); + store_->NewManifestFile(options, path_factory, partition_schema, ctx->GetMemoryPool())); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr index_manifest_file, - IndexManifestFile::Create(options.GetFileSystem(), options.GetManifestFormat(), - options.GetManifestCompression(), path_factory, - options.GetBucket(), ctx->GetMemoryPool(), options)); + store_->NewIndexManifestFile(options, path_factory, ctx->GetMemoryPool())); auto expire_snapshots = std::make_shared( snapshot_manager, path_factory, manifest_list, manifest_file, index_manifest_file, options.GetFileSystem(), options.GetExpireConfig(), options.RealtimeEnabled(), ctx->GetExecutor()); - CommitScanner::ScanSupplier scan_supplier; - if (schema->PrimaryKeys().empty()) { - scan_supplier = CreateAppendScanSupplier(snapshot_manager, GetSchemaManager(), - manifest_list, manifest_file, schema, arrow_schema, - options, ctx->GetExecutor(), ctx->GetMemoryPool()); - } else { - scan_supplier = CreatePkScanSupplier(snapshot_manager, GetSchemaManager(), manifest_list, - manifest_file, schema, arrow_schema, options, - ctx->GetExecutor(), ctx->GetMemoryPool()); - } + CommitScanner::ScanSupplier scan_supplier = + NewCommitScanSupplier(snapshot_manager, manifest_list, manifest_file, schema, arrow_schema, + options, ctx->GetExecutor(), ctx->GetMemoryPool()); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr snapshot_commit, NewSnapshotCommit(*ctx, catalog_identifier, options, snapshot_manager)); diff --git a/src/paimon/core/table/file_store_table.h b/src/paimon/core/table/file_store_table.h index 51c7c0b95..00959d257 100644 --- a/src/paimon/core/table/file_store_table.h +++ b/src/paimon/core/table/file_store_table.h @@ -25,6 +25,7 @@ #include "paimon/catalog/identifier.h" #include "paimon/catalog/table.h" +#include "paimon/core/operation/commit/commit_scanner.h" #include "paimon/core/schema/schema_manager.h" #include "paimon/core/table/file_store.h" #include "paimon/core/utils/branch_manager.h" @@ -34,26 +35,42 @@ namespace paimon { +class InternalReadContext; +class IOManager; +class ManifestFile; +class ManifestList; +class RealtimeContext; class Schema; +class SplitGenerator; /// The managed table a catalog hands out: one with snapshots and manifests, which the schema under /// its root path describes. A format table - a directory of plain data files - is a `FormatTable`, /// the other kind of `Table`. /// +/// This is the flow the four operators share, Java Paimon's `AbstractFileStoreTable`: what the +/// table was constructed with - the schema, file system, root path, branch and managers - decides +/// what every operator serves, so it is read here once and handed down. The schema's primary keys +/// decide the kind of managed table it is - `AppendOnlyFileStoreTable` or +/// `PrimaryKeyFileStoreTable`, the `Create` factory below - and that kind builds the pieces the +/// flows hand down: the write itself, the scans, the read and the split generator. +/// /// Its `NewScan`/`NewRead`/`NewWrite`/`NewCommit` build the operators here, from the context plus -/// the schema, file system, root path, branch and managers the table was constructed with, so -/// they serve the table as it was loaded. The +/// the state the table was constructed with, so they serve the table as it was loaded. The /// context-only entry points - `TableScan::Create()`, `TableRead::Create()`, /// `FileStoreWrite::Create()` and `FileStoreCommit::Create()` - load the table the context names /// and hand the context to that table's `NewXxx`, which is how a context built without a table /// reaches the same operators. class FileStoreTable : public Table { public: - /// The constructor a catalog builds a managed table through, so a `shared_ptr` can hold one; - /// the base constructor is protected, which an inherited constructor would stay. - /// @param file_system File system the table's data is accessed through. - /// @param root_path Root path of the table in the file system. - /// @param uuid Metastore UUID, or empty if unavailable. + /// The managed table the schema describes, built as the kind its primary keys name: an + /// `AppendOnlyFileStoreTable` when there are none, a `PrimaryKeyFileStoreTable` when there + /// are, the way Java Paimon's `FileStoreTableFactory` does. Primary keys cannot be added or + /// dropped by a schema change, so the kind the table was built as is the kind every schema + /// its root path ever held describes. + /// @param schema The schema the table was loaded with, whose primary keys name the kind and + /// whose options the operators serve. It is null when the caller had no schema, and + /// the table is built as the append-only kind - its operators then report that no + /// schema was found. /// @param branch The branch the table serves, which the managers below read and write on. /// @param schema_manager A schema manager to carry instead of building one - the one a /// context dispatch read the schema through, whose cache the operators then serve. @@ -61,12 +78,13 @@ class FileStoreTable : public Table { /// `Copy` hands its table's over so both share one. /// A null manager is built over `root_path` on `branch`, so the operators always read /// through the managers the table carries. - FileStoreTable(const std::shared_ptr& file_system, const std::string& root_path, - const std::shared_ptr& schema, const Identifier& identifier, - const std::string& uuid = "", - const std::string& branch = BranchManager::DEFAULT_MAIN_BRANCH, - const std::shared_ptr& schema_manager = nullptr, - const std::shared_ptr& snapshot_manager = nullptr); + static Result> Create( + const std::shared_ptr& file_system, const std::string& root_path, + const std::shared_ptr& schema, const Identifier& identifier, + const std::string& uuid = "", + const std::string& branch = BranchManager::DEFAULT_MAIN_BRANCH, + const std::shared_ptr& schema_manager = nullptr, + const std::shared_ptr& snapshot_manager = nullptr); /// The branch this table serves, as resolved when the table was built: the schema the table /// was loaded with came from it, and its managers point at it. @@ -120,6 +138,86 @@ class FileStoreTable : public Table { std::unique_ptr context) const override; private: + /// The read `NewRead` hands back, from the internal context it read the schema into: the + /// path factory comes from the store - a fallback branch's schema carries the table's + /// partition keys, which no schema change can alter - and the read itself is the kind's, + /// `NewTableRead`. + Result> CreateTableRead( + const std::shared_ptr& internal_context, + const std::shared_ptr& memory_pool, + const std::shared_ptr& executor) const; + + protected: + /// The base constructor the kinds of managed table build through, `Create` above. The + /// parameters are the ones it takes; see there for what each one is. + FileStoreTable(const std::shared_ptr& file_system, const std::string& root_path, + const std::shared_ptr& schema, const Identifier& identifier, + const std::string& uuid, const std::string& branch, + const std::shared_ptr& schema_manager, + const std::shared_ptr& snapshot_manager); + + /// The committed progress a real-time write restores, from the latest snapshot the snapshot + /// manager reads: where the write picks up after, which both kinds of real-time write ask + /// for the same way. + static Status RestoreRealtimeCommittedProgress( + const std::shared_ptr& realtime_context, + const std::shared_ptr& snapshot_manager, const CoreOptions& options); + + /// The scan the table's root path names, which the kinds of file store scan read from: the + /// kinds hand their scans back through it. + static std::unique_ptr WithTablePath(std::unique_ptr&& scan, + const std::string& root_path); + + /// The write of this kind of table, which `NewWrite` hands the shared flow's pieces to: the + /// kind's writer, built over the path factory, snapshot manager and options that flow + /// resolved. The branch the write serves is the table's, `CurrentBranch()`. + /// @param io_manager The spill manager the context's temp directory names, or null when the + /// context named none. + /// @param ignore_previous_files Whether the write starts over rather than appending, which + /// the flow resolved with this kind's own rules. + virtual Result> CreateWrite( + const WriteContext& ctx, const std::shared_ptr& schema, + const std::shared_ptr& arrow_schema, + const std::shared_ptr& partition_schema, const CoreOptions& options, + const std::shared_ptr& path_factory, + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& io_manager, bool ignore_previous_files) const = 0; + + /// The scan of this kind of table, which `NewScan` hands the manifest list and file it built + /// to. The snapshot and schema managers are the table's own, which the scan serves. + virtual Result> NewFileStoreScan( + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& arrow_schema, + const std::shared_ptr& table_schema, const CoreOptions& core_options, + const std::shared_ptr& executor, const std::shared_ptr& memory_pool, + const ScanContext* context) const = 0; + + /// The split generator of this kind of table, Java Paimon's `splitGenerator()`. + virtual Result> NewSplitGenerator( + const std::shared_ptr& table_schema, + const CoreOptions& core_options) const = 0; + + /// The read of this kind of table, which `NewRead` hands the internal context it read the + /// schema into and the path factory it built to. + virtual Result> NewTableRead( + const std::shared_ptr& path_factory, + const std::shared_ptr& internal_context, + const std::shared_ptr& memory_pool, + const std::shared_ptr& executor) const = 0; + + /// The manifest entry scan supplier of this kind of table, which `NewCommit` hands the + /// manifest list and file it built to. The schema manager the scans it hands out read through + /// is the table's own. + virtual CommitScanner::ScanSupplier NewCommitScanSupplier( + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& table_schema, + const std::shared_ptr& arrow_schema, const CoreOptions& options, + const std::shared_ptr& executor, + const std::shared_ptr& pool) const = 0; + /// The storage this table serves: the file system, root path, schema, branch and managers it /// was loaded with, and the factories that build the pieces every operator opens with. std::shared_ptr store_; diff --git a/src/paimon/core/table/primary_key_file_store_table.cpp b/src/paimon/core/table/primary_key_file_store_table.cpp new file mode 100644 index 000000000..564e015e3 --- /dev/null +++ b/src/paimon/core/table/primary_key_file_store_table.cpp @@ -0,0 +1,291 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/table/primary_key_file_store_table.h" + +#include +#include +#include +#include +#include +#include + +#include "fmt/format.h" +#include "paimon/common/global_index/btree/btree_defs.h" +#include "paimon/common/types/data_field.h" +#include "paimon/common/utils/fields_comparator.h" +#include "paimon/common/utils/string_utils.h" +#include "paimon/core/core_options.h" +#include "paimon/core/deletionvectors/bucketed_dv_maintainer.h" +#include "paimon/core/index/index_file_handler.h" +#include "paimon/core/index/pk/bucketed_primary_key_index_maintainer.h" +#include "paimon/core/index/pk/primary_key_index_definitions.h" +#include "paimon/core/manifest/index_manifest_entry.h" +#include "paimon/core/mergetree/compact/lookup_merge_function.h" +#include "paimon/core/mergetree/compact/merge_function.h" +#include "paimon/core/mergetree/compact/reducer_merge_function_wrapper.h" +#include "paimon/core/operation/commit/commit_scanner.h" +#include "paimon/core/operation/file_store_scan.h" +#include "paimon/core/operation/key_value_file_store_scan.h" +#include "paimon/core/operation/key_value_file_store_write.h" +#include "paimon/core/options/merge_engine.h" +#include "paimon/core/postpone/postpone_bucket_file_store_write.h" +#include "paimon/core/realtime/realtime_schema_layout.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/core/snapshot.h" +#include "paimon/core/table/bucket_mode.h" +#include "paimon/core/table/file_store.h" +#include "paimon/core/table/source/key_value_table_read.h" +#include "paimon/core/table/source/merge_tree_split_generator.h" +#include "paimon/core/table/source/split_generator.h" +#include "paimon/core/utils/primary_key_table_utils.h" +#include "paimon/core/utils/snapshot_manager.h" +#include "paimon/result.h" +#include "paimon/status.h" +#include "paimon/write_context.h" + +namespace paimon { + +struct KeyValue; +template +class MergeFunctionWrapper; + +namespace { + +/// Whether a schema older than `current_schema_id` carried a primary key b-tree index, which a +/// write has to keep maintaining: the index files an earlier schema wrote are still there to be +/// read, and dropping the definition now would leave them orphaned. +Result HasHistoricalPrimaryKeyBTreeDefinition( + const std::shared_ptr& schema_manager, int64_t current_schema_id) { + if (current_schema_id <= TableSchema::FIRST_SCHEMA_ID) { + return false; + } + PAIMON_ASSIGN_OR_RAISE(std::vector schema_ids, schema_manager->ListAllIds()); + for (int64_t schema_id : schema_ids) { + if (schema_id >= current_schema_id) { + continue; + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr historical_schema, + schema_manager->ReadSchema(schema_id)); + const auto& historical_options = historical_schema->Options(); + auto option = historical_options.find(Options::PK_BTREE_INDEX_COLUMNS); + if (option != historical_options.end() && + !StringUtils::IsNullOrWhitespaceOnly(option->second)) { + return true; + } + } + return false; +} + +} // namespace + +PrimaryKeyFileStoreTable::PrimaryKeyFileStoreTable( + const std::shared_ptr& file_system, const std::string& root_path, + const std::shared_ptr& schema, const Identifier& identifier, const std::string& uuid, + const std::string& branch, const std::shared_ptr& schema_manager, + const std::shared_ptr& snapshot_manager) + : FileStoreTable(file_system, root_path, schema, identifier, uuid, branch, schema_manager, + snapshot_manager) {} + +Result> PrimaryKeyFileStoreTable::CreateWrite( + const WriteContext& ctx, const std::shared_ptr& schema, + const std::shared_ptr& arrow_schema, + const std::shared_ptr& partition_schema, const CoreOptions& options, + const std::shared_ptr& path_factory, + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& io_manager, bool ignore_previous_files) const { + // pk table + if (ctx.GetRealtimeContext()) { + PAIMON_RETURN_NOT_OK(PrimaryKeyTableUtils::ValidateRealtimeOptions(options, *schema)); + if (ignore_previous_files) { + return Status::NotImplemented("PK realtime requires restore from the latest snapshot"); + } + if (!ctx.GetWriteSchema().empty()) { + return Status::NotImplemented("PK realtime does not support a custom write schema"); + } + PAIMON_RETURN_NOT_OK( + RestoreRealtimeCommittedProgress(ctx.GetRealtimeContext(), snapshot_manager, options)); + } + if (options.GetBucket() == BucketModeDefine::POSTPONE_BUCKET) { + return PostponeBucketFileStoreWrite::Create( + snapshot_manager, GetSchemaManager(), ctx.GetCommitUser(), root_path_, schema, + arrow_schema, partition_schema, io_manager, options, ctx.IsStreamingMode(), + ctx.IgnoreNumBucketCheck(), ctx.GetWriteId(), ctx.GetFileSystemSchemeToIdentifierMap(), + ctx.GetExecutor(), ctx.GetMemoryPool(), file_system_); + } + if (options.GetBucket() <= 0) { + return Status::Invalid( + fmt::format("not support bucket {} in key value table", options.GetBucket())); + } + PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_keys, + schema->TrimmedPrimaryKeys()); + PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_key_fields, + schema->GetFields(trimmed_primary_keys)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr key_comparator, + FieldsComparator::Create(trimmed_primary_key_fields, + options.SequenceFieldSortOrderIsAscending())); + auto primary_keys = schema->PrimaryKeys(); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merge_function, + PrimaryKeyTableUtils::CreateMergeFunction(arrow_schema, primary_keys, + options, ctx.GetMemoryPool())); + if (options.NeedLookup() && options.GetMergeEngine() != MergeEngine::FIRST_ROW) { + // don't wrap first row, it is already OK + merge_function = std::make_unique(std::move(merge_function)); + } + std::shared_ptr> merge_function_wrapper = + std::make_shared(std::move(merge_function)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr sequence_fields_comparator, + PrimaryKeyTableUtils::CreateSequenceFieldsComparator(schema->Fields(), options)); + + std::shared_ptr dv_maintainer_factory; + PAIMON_ASSIGN_OR_RAISE(PrimaryKeyIndexDefinitions primary_key_index_definitions, + PrimaryKeyIndexDefinitions::Create(*schema)); + bool has_btree_index = false; + for (const PrimaryKeyIndexDefinition& definition : + primary_key_index_definitions.Definitions()) { + if (definition.GetFamily() == PrimaryKeyIndexDefinition::Family::BTREE) { + has_btree_index = true; + break; + } + } + std::optional latest_snapshot; + bool has_existing_index_manifest = false; + if (!has_btree_index) { + PAIMON_ASSIGN_OR_RAISE( + bool has_historical_btree_index, + HasHistoricalPrimaryKeyBTreeDefinition(GetSchemaManager(), schema->Id())); + if (has_historical_btree_index) { + PAIMON_ASSIGN_OR_RAISE(latest_snapshot, snapshot_manager->LatestSnapshot()); + has_existing_index_manifest = + latest_snapshot.has_value() && latest_snapshot->IndexManifest().has_value(); + } + } + std::shared_ptr index_file_handler; + if (options.DeletionVectorsEnabled() || has_btree_index || has_existing_index_manifest) { + PAIMON_ASSIGN_OR_RAISE(index_file_handler, store_->NewIndexFileHandler( + options, path_factory, ctx.GetMemoryPool())); + } + bool has_restored_btree_index = false; + if (!has_btree_index && has_existing_index_manifest) { + PAIMON_ASSIGN_OR_RAISE( + std::vector restored_btree_entries, + index_file_handler->Scan( + latest_snapshot.value(), [](const IndexManifestEntry& entry) -> bool { + return entry.index_file->IndexType() == BtreeDefs::kIdentifier && + IndexFileHandler::IsPrimaryKeySourceIndex(*entry.index_file); + })); + has_restored_btree_index = !restored_btree_entries.empty(); + } + if (options.DeletionVectorsEnabled()) { + dv_maintainer_factory = std::make_shared(index_file_handler); + } + std::shared_ptr + primary_key_index_maintainer_factory; + if (has_btree_index || has_restored_btree_index) { + primary_key_index_maintainer_factory = + std::make_shared( + root_path_, CurrentBranch(), schema, primary_key_index_definitions.Definitions(), + path_factory, index_file_handler, options, io_manager, ctx.EnableMultiThreadSpill(), + ctx.GetExecutor(), ctx.GetMemoryPool()); + } + + std::shared_ptr realtime_schema_layout; + if (ctx.GetRealtimeContext()) { + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr layout, + RealtimeSchemaLayout::Create(RealtimeStoreMode::PRIMARY_KEY, arrow_schema)); + realtime_schema_layout = std::move(layout); + } + + return std::make_unique( + path_factory, snapshot_manager, GetSchemaManager(), ctx.GetCommitUser(), root_path_, schema, + arrow_schema, realtime_schema_layout, partition_schema, dv_maintainer_factory, + primary_key_index_maintainer_factory, io_manager, key_comparator, + sequence_fields_comparator, merge_function_wrapper, options, ignore_previous_files, + ctx.IsStreamingMode(), ctx.IgnoreNumBucketCheck(), ctx.EnableMultiThreadSpill(), + ctx.GetRealtimeContext(), ctx.GetExecutor(), ctx.GetMemoryPool()); +} + +Result> PrimaryKeyFileStoreTable::NewFileStoreScan( + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& arrow_schema, + const std::shared_ptr& table_schema, const CoreOptions& core_options, + const std::shared_ptr& executor, const std::shared_ptr& memory_pool, + const ScanContext* context) const { + // TODO(liancheng.lsz): support fallback branch in scan + std::unique_ptr scan; + PAIMON_ASSIGN_OR_RAISE( + scan, KeyValueFileStoreScan::Create(GetSnapshotManager(), GetSchemaManager(), manifest_list, + manifest_file, table_schema, arrow_schema, + context->GetScanFilters(), core_options, executor, + memory_pool)); + return WithTablePath(std::move(scan), root_path_); +} + +Result> PrimaryKeyFileStoreTable::NewSplitGenerator( + const std::shared_ptr& table_schema, const CoreOptions& core_options) const { + auto source_split_target_size = core_options.GetSourceSplitTargetSize(); + auto source_split_open_file_cost = core_options.GetSourceSplitOpenFileCost(); + // TODO(liancheng.lsz): support evolution + PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_keys, + table_schema->TrimmedPrimaryKeys()); + PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_pk_fields, + table_schema->GetFields(trimmed_primary_keys)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr key_comparator, + FieldsComparator::Create(trimmed_pk_fields, /*is_ascending_order=*/true)); + return std::make_unique( + source_split_target_size, source_split_open_file_cost, + core_options.DeletionVectorsEnabled(), core_options.GetMergeEngine(), key_comparator); +} + +Result> PrimaryKeyFileStoreTable::NewTableRead( + const std::shared_ptr& path_factory, + const std::shared_ptr& internal_context, + const std::shared_ptr& memory_pool, + const std::shared_ptr& executor) const { + return KeyValueTableRead::Create(path_factory, internal_context, memory_pool, executor); +} + +CommitScanner::ScanSupplier PrimaryKeyFileStoreTable::NewCommitScanSupplier( + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& table_schema, + const std::shared_ptr& arrow_schema, const CoreOptions& options, + const std::shared_ptr& executor, const std::shared_ptr& pool) const { + // The schema manager the table carries, which the scans this supplier hands out read the + // schema through. + const std::shared_ptr& schema_manager = GetSchemaManager(); + return [snapshot_manager, schema_manager, manifest_list, manifest_file, table_schema, + arrow_schema, options, executor, pool](const std::shared_ptr& scan_filter) + -> Result> { + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr scan, + KeyValueFileStoreScan::Create(snapshot_manager, schema_manager, manifest_list, + manifest_file, table_schema, arrow_schema, scan_filter, + options, executor, pool)); + return std::unique_ptr(std::move(scan)); + }; +} + +} // namespace paimon diff --git a/src/paimon/core/table/primary_key_file_store_table.h b/src/paimon/core/table/primary_key_file_store_table.h new file mode 100644 index 000000000..9c658758a --- /dev/null +++ b/src/paimon/core/table/primary_key_file_store_table.h @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include + +#include "paimon/catalog/identifier.h" +#include "paimon/core/table/file_store_table.h" +#include "paimon/core/utils/branch_manager.h" +#include "paimon/result.h" +#include "paimon/type_fwd.h" + +namespace paimon { + +class CoreOptions; +class FileStoreScan; +class IOManager; +class SplitGenerator; +class TableSchema; + +/// The managed table a schema with primary keys describes, Java Paimon's +/// `PrimaryKeyFileStoreTable`. Its write merges by key, and its scan, read and commit read +/// through the merge. The operator flows are the ones `FileStoreTable` carries; the pieces +/// those flows hand down are built here. +class PrimaryKeyFileStoreTable : public FileStoreTable { + public: + /// The constructor `FileStoreTable::Create` builds a primary key table through; the + /// parameters are the ones it takes, and the base class carries them. + PrimaryKeyFileStoreTable(const std::shared_ptr& file_system, + const std::string& root_path, const std::shared_ptr& schema, + const Identifier& identifier, const std::string& uuid = "", + const std::string& branch = BranchManager::DEFAULT_MAIN_BRANCH, + const std::shared_ptr& schema_manager = nullptr, + const std::shared_ptr& snapshot_manager = nullptr); + + private: + Result> CreateWrite( + const WriteContext& ctx, const std::shared_ptr& schema, + const std::shared_ptr& arrow_schema, + const std::shared_ptr& partition_schema, const CoreOptions& options, + const std::shared_ptr& path_factory, + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& io_manager, bool ignore_previous_files) const override; + + Result> NewFileStoreScan( + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& arrow_schema, + const std::shared_ptr& table_schema, const CoreOptions& core_options, + const std::shared_ptr& executor, const std::shared_ptr& memory_pool, + const ScanContext* context) const override; + + Result> NewSplitGenerator( + const std::shared_ptr& table_schema, + const CoreOptions& core_options) const override; + + Result> NewTableRead( + const std::shared_ptr& path_factory, + const std::shared_ptr& internal_context, + const std::shared_ptr& memory_pool, + const std::shared_ptr& executor) const override; + + CommitScanner::ScanSupplier NewCommitScanSupplier( + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& table_schema, + const std::shared_ptr& arrow_schema, const CoreOptions& options, + const std::shared_ptr& executor, + const std::shared_ptr& pool) const override; +}; + +} // namespace paimon diff --git a/src/paimon/core/table/table_test.cpp b/src/paimon/core/table/table_test.cpp index 01a7ce27e..fe99efe63 100644 --- a/src/paimon/core/table/table_test.cpp +++ b/src/paimon/core/table/table_test.cpp @@ -66,7 +66,7 @@ Result> CreateManagedTable( PAIMON_ASSIGN_OR_RAISE(std::unique_ptr created, schema_manager.CreateTable(schema, /*partition_keys=*/{}, /*primary_keys=*/{}, options)); - return std::make_shared( + return FileStoreTable::Create( file_system, path, checked_pointer_cast(std::shared_ptr(std::move(created))), identifier, /*uuid=*/"", branch); @@ -346,11 +346,13 @@ TEST(TableTest, TestMainBranchAliasResolvesToTheMainBranch) { {Identifier("db", "tbl"), Identifier("db", "tbl$branch_main"), Identifier("db", "tbl$branch_MAIN")}) { SCOPED_TRACE(main_identifier.ToString()); - // A catalog that loaded the schema itself calls the constructor directly, which a + // A catalog that loaded the schema itself builds the table through the factory, which a // rest catalog is shaped like. The alias resolves to the main branch as well. ASSERT_OK_AND_ASSIGN(std::string branch, main_identifier.GetBranchNameOrDefault()); - FileStoreTable table(fs, dir->Str(), schema, main_identifier, /*uuid=*/"", branch); - EXPECT_EQ("main", table.CurrentBranch()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr table, + FileStoreTable::Create(fs, dir->Str(), schema, main_identifier, + /*uuid=*/"", branch)); + EXPECT_EQ("main", table->CurrentBranch()); } } diff --git a/src/paimon/rest/rest_catalog.cpp b/src/paimon/rest/rest_catalog.cpp index dec534830..712a351cf 100644 --- a/src/paimon/rest/rest_catalog.cpp +++ b/src/paimon/rest/rest_catalog.cpp @@ -472,7 +472,7 @@ Result> RestCatalog::GetTable(const Identifier& identifie // The branch the identifier carries, which the schema and path above were read on, so the // table and its managers serve it. PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); - return std::make_shared( + return FileStoreTable::Create( table_file_system, table_path, schema, identifier, uuid, NormalizeBranch(std::move(branch)).value_or(Identifier::kDefaultMainBranch)); } diff --git a/src/paimon/testing/mock/mock_catalog.h b/src/paimon/testing/mock/mock_catalog.h index 88a00e67d..32a684c34 100644 --- a/src/paimon/testing/mock/mock_catalog.h +++ b/src/paimon/testing/mock/mock_catalog.h @@ -116,7 +116,7 @@ class MockVersionManagedCatalog : public Catalog, public VersionManagedCatalog { // Serves the per-table file system when one was set, matching `GetTableFileSystem` // without recording the request, so a test can still tell the two apart. No location: // this mock's `GetTableLocation` is the unsupported stub. - return std::make_shared( + return FileStoreTable::Create( table_file_system_ != nullptr ? table_file_system_ : file_system_, /*root_path=*/"", table_schema_, identifier, table_uuid_); } From b6678cfbbaea87ccb6482c6a050bf038b8be0403 Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Fri, 23 Oct 2026 13:39:56 +0800 Subject: [PATCH 06/30] tmp --- include/paimon/commit_context.h | 11 +- include/paimon/read_context.h | 11 +- include/paimon/scan_context.h | 12 +- include/paimon/write_context.h | 11 +- .../core/operation/internal_read_context.cpp | 24 +- .../core/operation/internal_read_context.h | 22 ++ .../table/append_only_file_store_table.cpp | 23 +- .../core/table/append_only_file_store_table.h | 22 +- src/paimon/core/table/file_store.cpp | 63 +++- src/paimon/core/table/file_store.h | 84 ++++- src/paimon/core/table/file_store_table.cpp | 312 +++++++++--------- src/paimon/core/table/file_store_table.h | 33 +- .../table/primary_key_file_store_table.cpp | 24 +- .../core/table/primary_key_file_store_table.h | 22 +- .../core/table/source/table_read_test.cpp | 17 +- .../table/system/audit_log_system_table.cpp | 14 +- .../system/read_optimized_system_table.cpp | 14 +- src/paimon/core/table/table_test.cpp | 145 +++++++- src/paimon/rest/rest_catalog.cpp | 18 + test/inte/scan_and_read_inte_test.cpp | 21 +- 20 files changed, 612 insertions(+), 291 deletions(-) diff --git a/include/paimon/commit_context.h b/include/paimon/commit_context.h index 1ec784f76..a70cacb7a 100644 --- a/include/paimon/commit_context.h +++ b/include/paimon/commit_context.h @@ -156,15 +156,18 @@ class PAIMON_EXPORT CommitContextBuilder { ~CommitContextBuilder(); - /// Set a configuration options map to set some option entries which are not defined in the - /// table schema or whose values you want to overwrite. + /// Set a configuration options map. The options take part in loading the table a commit + /// serves: the load merges them over the ones the table's schema carries and the table it + /// hands back serves the merged ones. A caller holding a table already loaded - a load + /// that is over - changes it through `Table::Copy()`; an option contradicting a table held + /// that way is refused when a commit is asked of it. A format table takes the options as + /// its own. /// @note The options map will clear the options added by `AddOption()` before. /// @param options The configuration options map. /// @return Reference to this builder for method chaining. CommitContextBuilder& SetOptions(const std::map& options); - /// Add a single configuration option which is not defined in the table schema or whose value - /// you want to overwrite. + /// Add a single configuration option, served the way `SetOptions()` describes. /// /// If you want to add multiple options, call `AddOption()` multiple times or use `SetOptions()` /// instead. diff --git a/include/paimon/read_context.h b/include/paimon/read_context.h index b9c534065..8cf565996 100644 --- a/include/paimon/read_context.h +++ b/include/paimon/read_context.h @@ -324,15 +324,18 @@ class PAIMON_EXPORT ReadContextBuilder { /// When set, read_field_ids and read_field_names are ignored. ReadContextBuilder& SetReadSchema(std::unique_ptr read_schema); - /// Set a configuration options map to set some option entries which are not defined in the - /// table schema or whose values you want to overwrite. + /// Set a configuration options map. The options take part in loading the table a read + /// serves: the load merges them over the ones the table's schema carries and the table it + /// hands back serves the merged ones. A caller holding a table already loaded - a load + /// that is over - changes it through `Table::Copy()`; an option contradicting a table held + /// that way is refused when a read is asked of it. A format table takes the options as its + /// own. /// @note The options map will clear the options added by `AddOption()` before. /// @param options The configuration options map. /// @return Reference to this builder for method chaining. ReadContextBuilder& SetOptions(const std::map& options); - /// Add a single configuration option which is not defined in the table schema or whose value - /// you want to overwrite. + /// Add a single configuration option, served the way `SetOptions()` describes. /// /// If you want to add multiple options, call `AddOption()` multiple times or use `SetOptions()` /// instead. diff --git a/include/paimon/scan_context.h b/include/paimon/scan_context.h index 5be7c9c4e..fe57afb58 100644 --- a/include/paimon/scan_context.h +++ b/include/paimon/scan_context.h @@ -208,11 +208,15 @@ class PAIMON_EXPORT ScanContextBuilder { ScanContextBuilder& WithRealtimeContext( const std::shared_ptr& realtime_context); - /// The options added or set in `ScanContextBuilder` have high priority and will be merged with - /// the options in table schema. + /// Add a single configuration option. See `SetOptions()` for how the options a context + /// carries are served. ScanContextBuilder& AddOption(const std::string& key, const std::string& value); - /// Set a configuration options map to set some option entries which are not defined in the - /// table schema or whose values you want to overwrite. + /// Set a configuration options map. The options take part in loading the table a scan + /// serves: the load merges them over the ones the table's schema carries and the table it + /// hands back serves the merged ones. A caller holding a table already loaded - a load + /// that is over - changes it through `Table::Copy()`; an option contradicting a table held + /// that way is refused when a scan is asked of it. A format table takes the options as its + /// own. /// @note The options map will clear the options added by `AddOption()` before. /// @param options The configuration options map. /// @return Reference to this builder for method chaining. diff --git a/include/paimon/write_context.h b/include/paimon/write_context.h index 58c0b3f97..23195793f 100644 --- a/include/paimon/write_context.h +++ b/include/paimon/write_context.h @@ -183,15 +183,18 @@ class PAIMON_EXPORT WriteContextBuilder { ~WriteContextBuilder(); - /// Set a configuration options map to set some option entries which are not defined in the - /// table schema or whose values you want to overwrite. + /// Set a configuration options map. The options take part in loading the table a write + /// serves: the load merges them over the ones the table's schema carries and the table it + /// hands back serves the merged ones. A caller holding a table already loaded - a load + /// that is over - changes it through `Table::Copy()`; an option contradicting a table held + /// that way is refused when a write is asked of it. A format table takes the options as its + /// own. /// @note The options map will clear the options added by `AddOption()` before. /// @param options The configuration options map. /// @return Reference to this builder for method chaining. WriteContextBuilder& SetOptions(const std::map& options); - /// Add a single configuration option which is not defined in the table schema or whose value - /// you want to overwrite. + /// Add a single configuration option, served the way `SetOptions()` describes. /// /// If you want to add multiple options, call `AddOption()` multiple times or use `SetOptions()` /// instead. diff --git a/src/paimon/core/operation/internal_read_context.cpp b/src/paimon/core/operation/internal_read_context.cpp index 6c0c43341..778976a46 100644 --- a/src/paimon/core/operation/internal_read_context.cpp +++ b/src/paimon/core/operation/internal_read_context.cpp @@ -273,18 +273,32 @@ std::optional InternalReadContext::TryResolveSpecialFieldByName( return std::nullopt; } +Result> InternalReadContext::Create( + const std::shared_ptr& context, const std::shared_ptr& table_schema, + CoreOptions core_options, const std::string* path) { + core_options.WithCache(context->GetCache()); + return CreateFromCoreOptions(context, table_schema, std::move(core_options), path); +} + Result> InternalReadContext::Create( const std::shared_ptr& context, const std::shared_ptr& table_schema, const std::map& options, const std::shared_ptr& file_system, const std::string* path) { - // The file system a table reads through, or null for the read context's own, which callers - // without a table hand over. + // The file system the caller named, or null for the read context's own, which callers + // without a table hand over. The scheme map the context carries does not take part: a file + // system handed over wins and one missing falls to the resolving one, the same way a scan + // resolves its own. PAIMON_ASSIGN_OR_RAISE( CoreOptions core_options, - CoreOptions::FromMap( - options, file_system != nullptr ? file_system : context->GetSpecificFileSystem(), - context->GetFileSystemSchemeToIdentifierMap())); + CoreOptions::FromMap(options, file_system != nullptr ? file_system + : context->GetSpecificFileSystem())); core_options.WithCache(context->GetCache()); + return CreateFromCoreOptions(context, table_schema, std::move(core_options), path); +} + +Result> InternalReadContext::CreateFromCoreOptions( + const std::shared_ptr& context, const std::shared_ptr& table_schema, + CoreOptions core_options, const std::string* path) { if (context->HasVectorOrFullTextSearch()) { if (!core_options.FileIndexReadEnabled()) { return Status::Invalid( diff --git a/src/paimon/core/operation/internal_read_context.h b/src/paimon/core/operation/internal_read_context.h index b1619c5bc..b7218a686 100644 --- a/src/paimon/core/operation/internal_read_context.h +++ b/src/paimon/core/operation/internal_read_context.h @@ -58,6 +58,21 @@ class InternalReadContext { std::string field_name; }; + /// Builds the context of a read a table serves: the options are the table's own, already + /// parsed - the store's `options()` - and the cache the context carries is armed on them + /// here. `path` names the root path the table was opened with, which the read reads from in + /// place of the one named by the context. + static Result> Create( + const std::shared_ptr& read_context, + const std::shared_ptr& table_schema, CoreOptions core_options, + const std::string* path = nullptr); + + /// Builds the context of a read whose options are composed on the spot rather than the + /// table's own - the fallback branch of a read, whose schema belongs to another branch, and + /// the reads a write or a lookup serve inside themselves. The options are parsed from + /// `options` here: the read's own file system is `file_system`, or the context's when null - + /// the scheme map the context carries does not take part. A cache the context carries is + /// armed on the options this parses. static Result> Create( const std::shared_ptr& read_context, const std::shared_ptr& table_schema, @@ -171,6 +186,13 @@ class InternalReadContext { const std::shared_ptr& arrow_pool, std::optional path = std::nullopt); + /// The tail both `Create` entries share: the options arrive parsed and cached, and the + /// validations and the read schema below are the same whichever way they were resolved. + static Result> CreateFromCoreOptions( + const std::shared_ptr& read_context, + const std::shared_ptr& table_schema, CoreOptions core_options, + const std::string* path); + static std::optional TryResolveSpecialFieldById(int32_t field_id, const CoreOptions& core_options, bool has_vector_or_full_text_search); diff --git a/src/paimon/core/table/append_only_file_store_table.cpp b/src/paimon/core/table/append_only_file_store_table.cpp index 0846aef90..cead79566 100644 --- a/src/paimon/core/table/append_only_file_store_table.cpp +++ b/src/paimon/core/table/append_only_file_store_table.cpp @@ -48,22 +48,19 @@ namespace paimon { -AppendOnlyFileStoreTable::AppendOnlyFileStoreTable( - const std::shared_ptr& file_system, const std::string& root_path, - const std::shared_ptr& schema, const Identifier& identifier, const std::string& uuid, - const std::string& branch, const std::shared_ptr& schema_manager, - const std::shared_ptr& snapshot_manager) - : FileStoreTable(file_system, root_path, schema, identifier, uuid, branch, schema_manager, - snapshot_manager) {} +AppendOnlyFileStoreTable::AppendOnlyFileStoreTable(const std::shared_ptr& store, + const Identifier& identifier, + const std::string& uuid) + : FileStoreTable(store, identifier, uuid) {} Result> AppendOnlyFileStoreTable::CreateWrite( - const WriteContext& ctx, const std::shared_ptr& schema, - const std::shared_ptr& arrow_schema, + const WriteContext& ctx, const std::shared_ptr& arrow_schema, const std::shared_ptr& partition_schema, const CoreOptions& options, const std::shared_ptr& path_factory, const std::shared_ptr& snapshot_manager, const std::shared_ptr& io_manager, bool ignore_previous_files) const { // append table + const std::shared_ptr& schema = store_->schema(); const bool realtime_deduplicate = !options.GetRealtimeDeduplicateKeyFields().empty(); bool need_dv_maintainer_factory = options.DeletionVectorsEnabled(); if (options.GetBucket() == -1) { @@ -138,11 +135,11 @@ Result> AppendOnlyFileStoreTable::CreateWrite( Result> AppendOnlyFileStoreTable::NewFileStoreScan( const std::shared_ptr& manifest_list, const std::shared_ptr& manifest_file, - const std::shared_ptr& arrow_schema, - const std::shared_ptr& table_schema, const CoreOptions& core_options, + const std::shared_ptr& arrow_schema, const CoreOptions& core_options, const std::shared_ptr& executor, const std::shared_ptr& memory_pool, const ScanContext* context) const { // TODO(liancheng.lsz): support fallback branch in scan + const std::shared_ptr& table_schema = store_->schema(); std::unique_ptr scan; if (core_options.DataEvolutionEnabled()) { PAIMON_ASSIGN_OR_RAISE( @@ -161,7 +158,7 @@ Result> AppendOnlyFileStoreTable::NewFileStoreSca } Result> AppendOnlyFileStoreTable::NewSplitGenerator( - const std::shared_ptr& table_schema, const CoreOptions& core_options) const { + const CoreOptions& core_options) const { auto source_split_target_size = core_options.GetSourceSplitTargetSize(); auto source_split_open_file_cost = core_options.GetSourceSplitOpenFileCost(); if (core_options.DataEvolutionEnabled()) { @@ -188,12 +185,12 @@ CommitScanner::ScanSupplier AppendOnlyFileStoreTable::NewCommitScanSupplier( const std::shared_ptr& snapshot_manager, const std::shared_ptr& manifest_list, const std::shared_ptr& manifest_file, - const std::shared_ptr& table_schema, const std::shared_ptr& arrow_schema, const CoreOptions& options, const std::shared_ptr& executor, const std::shared_ptr& pool) const { // The schema manager the table carries, which the scans this supplier hands out read the // schema through. const std::shared_ptr& schema_manager = GetSchemaManager(); + const std::shared_ptr& table_schema = store_->schema(); return [snapshot_manager, schema_manager, manifest_list, manifest_file, table_schema, arrow_schema, options, executor, pool](const std::shared_ptr& scan_filter) -> Result> { diff --git a/src/paimon/core/table/append_only_file_store_table.h b/src/paimon/core/table/append_only_file_store_table.h index ea5374ba3..499431eb0 100644 --- a/src/paimon/core/table/append_only_file_store_table.h +++ b/src/paimon/core/table/append_only_file_store_table.h @@ -24,13 +24,11 @@ #include "paimon/catalog/identifier.h" #include "paimon/core/table/file_store_table.h" -#include "paimon/core/utils/branch_manager.h" #include "paimon/result.h" #include "paimon/type_fwd.h" namespace paimon { -class CoreOptions; class FileStoreScan; class IOManager; class SplitGenerator; @@ -42,19 +40,14 @@ class TableSchema; /// carries; the pieces those flows hand down are built here. class AppendOnlyFileStoreTable : public FileStoreTable { public: - /// The constructor `FileStoreTable::Create` builds an append-only table through; the - /// parameters are the ones it takes, and the base class carries them. - AppendOnlyFileStoreTable(const std::shared_ptr& file_system, - const std::string& root_path, const std::shared_ptr& schema, - const Identifier& identifier, const std::string& uuid = "", - const std::string& branch = BranchManager::DEFAULT_MAIN_BRANCH, - const std::shared_ptr& schema_manager = nullptr, - const std::shared_ptr& snapshot_manager = nullptr); + /// The constructor `FileStoreTable::Create` builds an append-only table through: the store + /// carries every state, and the base class holds it. + AppendOnlyFileStoreTable(const std::shared_ptr& store, const Identifier& identifier, + const std::string& uuid = ""); private: Result> CreateWrite( - const WriteContext& ctx, const std::shared_ptr& schema, - const std::shared_ptr& arrow_schema, + const WriteContext& ctx, const std::shared_ptr& arrow_schema, const std::shared_ptr& partition_schema, const CoreOptions& options, const std::shared_ptr& path_factory, const std::shared_ptr& snapshot_manager, @@ -63,13 +56,11 @@ class AppendOnlyFileStoreTable : public FileStoreTable { Result> NewFileStoreScan( const std::shared_ptr& manifest_list, const std::shared_ptr& manifest_file, - const std::shared_ptr& arrow_schema, - const std::shared_ptr& table_schema, const CoreOptions& core_options, + const std::shared_ptr& arrow_schema, const CoreOptions& core_options, const std::shared_ptr& executor, const std::shared_ptr& memory_pool, const ScanContext* context) const override; Result> NewSplitGenerator( - const std::shared_ptr& table_schema, const CoreOptions& core_options) const override; Result> NewTableRead( @@ -82,7 +73,6 @@ class AppendOnlyFileStoreTable : public FileStoreTable { const std::shared_ptr& snapshot_manager, const std::shared_ptr& manifest_list, const std::shared_ptr& manifest_file, - const std::shared_ptr& table_schema, const std::shared_ptr& arrow_schema, const CoreOptions& options, const std::shared_ptr& executor, const std::shared_ptr& pool) const override; diff --git a/src/paimon/core/table/file_store.cpp b/src/paimon/core/table/file_store.cpp index 80b6afb05..2c5a19d96 100644 --- a/src/paimon/core/table/file_store.cpp +++ b/src/paimon/core/table/file_store.cpp @@ -19,9 +19,11 @@ #include "paimon/core/table/file_store.h" +#include #include #include #include +#include #include #include "paimon/catalog/catalog.h" @@ -39,30 +41,71 @@ #include "paimon/core/utils/file_store_path_factory.h" #include "paimon/core/utils/index_file_path_factories.h" #include "paimon/core/utils/snapshot_manager.h" +#include "paimon/defs.h" #include "paimon/result.h" #include "paimon/status.h" namespace paimon { +Result> FileStore::Create( + const std::shared_ptr& file_system, const std::string& root_path, + const std::shared_ptr& schema, const std::string& branch, + const std::shared_ptr& schema_manager, + const std::shared_ptr& snapshot_manager) { + auto store = std::shared_ptr( + new FileStore(file_system, root_path, schema, branch, schema_manager, snapshot_manager)); + if (schema != nullptr) { + // The options the schema was loaded with, on the branch this store serves: what reads + // through them - whose cache keys are scoped by the branch - reads the same one the + // managers point at. + std::map options = schema->Options(); + if (BranchManager::IsMainBranch(store->branch_)) { + options.erase(Options::BRANCH); + } else { + options[Options::BRANCH] = store->branch_; + } + PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, + CoreOptions::FromMap(options, file_system, {})); + store->core_options_ = std::move(core_options); + } + // The file system the table is served through: the one the options resolve - a schema + // carrying access configuration names its own, which the file system the caller opened the + // table through wins over - and without a schema, the one handed over, whose absence leaves + // the default local one. + if (store->core_options_.has_value()) { + store->file_system_ = store->core_options_->GetFileSystem(); + } else if (file_system != nullptr) { + store->file_system_ = file_system; + } else { + PAIMON_ASSIGN_OR_RAISE(CoreOptions default_options, CoreOptions::FromMap({})); + store->file_system_ = default_options.GetFileSystem(); + } + // The managers read through the same file system: the metadata belongs to the data, and + // both were written under the options the schema carries. A manager handed over already + // points at `root_path` on `branch`; a missing one is built here, so the operators always + // read through the managers the store carries. + if (store->schema_manager_ == nullptr) { + store->schema_manager_ = + std::make_shared(store->file_system_, root_path, store->branch_); + } + if (store->snapshot_manager_ == nullptr) { + store->snapshot_manager_ = + std::make_shared(store->file_system_, root_path, store->branch_); + } + return store; +} + FileStore::FileStore(const std::shared_ptr& file_system, const std::string& root_path, const std::shared_ptr& schema, const std::string& branch, const std::shared_ptr& schema_manager, const std::shared_ptr& snapshot_manager) : file_system_(file_system), + specified_file_system_(file_system), root_path_(root_path), schema_(schema), branch_(BranchManager::NormalizeBranch(branch)), schema_manager_(schema_manager), - snapshot_manager_(snapshot_manager) { - // A manager handed over points at `root_path` on `branch` already; a missing one is built - // here, so the operators always read through the managers the store carries. - if (schema_manager_ == nullptr) { - schema_manager_ = std::make_shared(file_system_, root_path_, branch_); - } - if (snapshot_manager_ == nullptr) { - snapshot_manager_ = std::make_shared(file_system_, root_path_, branch_); - } -} + snapshot_manager_(snapshot_manager) {} Result> FileStore::NewPathFactory( const std::shared_ptr& arrow_schema, const CoreOptions& options, diff --git a/src/paimon/core/table/file_store.h b/src/paimon/core/table/file_store.h index 83c34b1e7..6c2396bd6 100644 --- a/src/paimon/core/table/file_store.h +++ b/src/paimon/core/table/file_store.h @@ -24,6 +24,7 @@ #include #include "paimon/catalog/identifier.h" +#include "paimon/core/core_options.h" #include "paimon/core/schema/schema_manager.h" #include "paimon/core/utils/file_store_path_factory.h" #include "paimon/core/utils/snapshot_manager.h" @@ -38,38 +39,53 @@ namespace paimon { class BinaryRowPartitionComputer; class Catalog; -class CoreOptions; class IndexFileHandler; class IndexManifestFile; class ManifestFile; class ManifestList; class TableSchema; -/// The storage a managed table is served through: the file system, root path, schema and branch -/// it was loaded with, and the schema and snapshot managers that read its root path on that -/// branch - the state Java Paimon's `AbstractFileStore` carries. `FileStoreTable` hands the -/// state it was loaded with to one of these and asks it back for the pieces every operator +/// The storage a managed table is served through: the file system, root path, schema, branch, +/// options and the schema and snapshot managers it was loaded with - the state Java Paimon's +/// `AbstractFileStore` carries, parsed once here rather than per operator. `FileStoreTable` hands +/// the state it was loaded with to one of these and asks it back for the pieces every operator /// opens with. /// +/// The options are the ones the table was loaded with on this branch - the schema's own, or +/// the ones a load merged over them - the base every operator serves: `NewXxx` takes no +/// options of its own beyond them. A cache is not part of them - `WithCache()` arms a copy an +/// operator makes - so the factories below take the options of the call, not the store's own. +/// /// The factory methods build a fresh instance on every call, never a cached one: a path factory /// names each file it builds, and a snapshot manager a write or a commit arms with a catalog /// loader is not one a scan should be served through afterwards. class FileStore { public: - /// @param file_system File system the table's data is accessed through. + /// @param file_system The file system the caller opened the table through, which wins over + /// the access configuration the schema's options may carry when it is not null. A + /// null one hands the resolution to that configuration: the table is then served by + /// the file system its options name - the way a load that only had a path and a + /// schema to read names it - and by the default local one when they name none. /// @param root_path Root path of the table in the file system. /// @param schema The schema the table was loaded with, whose partition keys name the - /// partitions the paths below are built for. + /// partitions the paths below are built for and whose options `options()` parses. It + /// is null when the caller had no schema; the operators then report that none was + /// found. /// @param branch The branch the store serves, normalized here. /// @param schema_manager A schema manager to carry instead of building one - the one a /// context dispatch read the schema through, whose cache the operators then serve. /// @param snapshot_manager A snapshot manager to carry instead of building one. /// A null manager is built over `root_path` on `branch`, so the operators always read /// through the managers the store carries. - FileStore(const std::shared_ptr& file_system, const std::string& root_path, - const std::shared_ptr& schema, const std::string& branch, - const std::shared_ptr& schema_manager = nullptr, - const std::shared_ptr& snapshot_manager = nullptr); + /// + /// Building the store parses the schema's options, so it fails when they do not parse: the + /// error is the caller's to see, which is why a store is built through this factory rather + /// than the constructor. + static Result> Create( + const std::shared_ptr& file_system, const std::string& root_path, + const std::shared_ptr& schema, const std::string& branch, + const std::shared_ptr& schema_manager = nullptr, + const std::shared_ptr& snapshot_manager = nullptr); /// The branch this store serves, as resolved when it was built. const std::string& branch() const { @@ -88,6 +104,40 @@ class FileStore { return snapshot_manager_; } + /// The schema the table was loaded with, which every operator serves rather than reading the + /// schema again. It is null when the caller had no schema. + const std::shared_ptr& schema() const { + return schema_; + } + + /// The file system the table's data is accessed through, as the table was opened with it: + /// the one the options resolve - a schema carrying access configuration names its own, + /// which the file system the caller opened the table through wins over - rather than the + /// one that call handed over, which `specified_file_system()` hands back untouched. + const std::shared_ptr& file_system() const { + return file_system_; + } + + /// The file system the caller opened the table through, as it was handed to `Create`: null + /// when the call had none, so a copy of the table resolves from its own options again and + /// the runtime ones it merges in - a file-system option among them - can speak. + const std::shared_ptr& specified_file_system() const { + return specified_file_system_; + } + + /// The root path of the table in its file system, as the table was opened with it. + const std::string& root_path() const { + return root_path_; + } + + /// The options the schema was loaded with on `branch()`, which every operator serves. They + /// exist only over a schema: an operator reads them after reporting that no schema was + /// found when there is none. + const CoreOptions& options() const { + assert(core_options_.has_value()); + return *core_options_; + } + /// The path factory an operator builds its metadata paths through: over this store's root /// path and the partition keys of its schema - which no schema change can alter, so a read /// serving a fallback branch's schema still names the same partitions. The file format @@ -146,12 +196,24 @@ class FileStore { const std::shared_ptr& memory_pool) const; private: + FileStore(const std::shared_ptr& file_system, const std::string& root_path, + const std::shared_ptr& schema, const std::string& branch, + const std::shared_ptr& schema_manager, + const std::shared_ptr& snapshot_manager); + std::shared_ptr file_system_; + /// The file system the caller opened the table through, as handed to `Create` - null when + /// the call had none - kept for a copy to hand over again: what `file_system_` holds is + /// resolved, and would pin the copy to this table's resolution. + std::shared_ptr specified_file_system_; std::string root_path_; std::shared_ptr schema_; std::string branch_; std::shared_ptr schema_manager_; std::shared_ptr snapshot_manager_; + /// The options of `schema_` on `branch_`, or unset when there is no schema. `Create` parses + /// them: the constructor cannot report a parse failure. + std::optional core_options_; }; } // namespace paimon diff --git a/src/paimon/core/table/file_store_table.cpp b/src/paimon/core/table/file_store_table.cpp index 1c9769245..e409f4249 100644 --- a/src/paimon/core/table/file_store_table.cpp +++ b/src/paimon/core/table/file_store_table.cpp @@ -142,13 +142,45 @@ Result> TryLoadFormatTable( /*location_carries_paimon_metadata=*/true, options); } +/// The options a context carries, merged into the table the load hands back: a table is the +/// product of the load, so the options speak there - the way `Copy` merges them, over the ones +/// the schema carries - and `NewXxx` on the product serves them without merging again. A +/// caller holding a table already - a load that is over - changes it through `Copy` itself, +/// and an option that contradicts the table that way is refused. +Result> WithContextOptions( + const std::shared_ptr
& table, const std::shared_ptr& schema, + const std::map& options) { + if (table == nullptr || schema == nullptr || options.empty()) { + return table; + } + const std::map& schema_options = schema->Options(); + for (const auto& [key, value] : options) { + auto schema_option = schema_options.find(key); + // A key the schema does not carry, or carries differently: the merge has something to + // do. Every key agreeing with the schema leaves the table the load built as it is. + if (schema_option == schema_options.end() || schema_option->second != value) { + return table->Copy(options); + } + } + return table; +} + /// The tail every `TableOf` ends with: the schema one of them read says what the path names. A /// format table is handed back as the `FormatTable` it is, everything else as the /// `FileStoreTable` a managed table is, carrying the schema for its `NewXxx` to build from. A /// null schema is handed over as it is: what to say about a table that is not there is the /// managed path's to decide, and it says it in more detail. +/// +/// @param file_system The file system the schema above was read through, which a format table +/// is served by - it has no schema options of its own to name another one. +/// @param explicit_file_system The file system the context supplied, which a managed table is +/// opened through when there is one. A null one leaves the table to resolve from the +/// options its schema carries: access configuration the schema was stored with then +/// names the files it serves, rather than being pinned to the one this load happened to +/// read the schema through. Result> TableOfLoadedSchema( - const std::shared_ptr& file_system, const std::string& path, + const std::shared_ptr& file_system, + const std::shared_ptr& explicit_file_system, const std::string& path, const std::string& branch, const std::map& options, const std::shared_ptr& schema_manager, const std::shared_ptr& latest_schema, const std::optional& identifier, @@ -162,8 +194,10 @@ Result> TableOfLoadedSchema( // it sits in - the same convention `TryLoadFormatTable` follows above. Identifier table_identifier = identifier.value_or(Identifier(Identifier::kUnknownDatabase, PathUtil::GetName(path))); - return FileStoreTable::Create(file_system, path, latest_schema, table_identifier, uuid, branch, - schema_manager); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, + FileStoreTable::Create(explicit_file_system, path, latest_schema, + table_identifier, uuid, branch, schema_manager)); + return WithContextOptions(table, latest_schema, options); } /// The managed table a context names behind a path: the schema is read through a manager of the @@ -173,12 +207,14 @@ Result> TableOfLoadedSchema( /// @param schema_json The schema of the table, when the caller already read it, or null when it /// has to be read here. It is honoured on the main branch only: a branch publishes its own /// under `branch/branch-`, which is where the read below goes. -Result> ManagedTableOf(const std::shared_ptr& file_system, - const std::string& path, const std::string& branch, - const std::map& options, - const std::optional& schema_json, - const std::optional& identifier, - const std::string& uuid = "") { +/// @param explicit_file_system The file system the context supplied, which the managed table the +/// schema describes is opened through - see `TableOfLoadedSchema`. +Result> ManagedTableOf( + const std::shared_ptr& file_system, + const std::shared_ptr& explicit_file_system, const std::string& path, + const std::string& branch, const std::map& options, + const std::optional& schema_json, const std::optional& identifier, + const std::string& uuid = "") { auto schema_manager = std::make_shared(file_system, path, branch); std::shared_ptr latest_schema; if (branch == BranchManager::DEFAULT_MAIN_BRANCH && schema_json) { @@ -192,8 +228,8 @@ Result> ManagedTableOf(const std::shared_ptr& latest_schema = *latest; } } - return TableOfLoadedSchema(file_system, path, branch, options, schema_manager, latest_schema, - identifier, uuid); + return TableOfLoadedSchema(file_system, explicit_file_system, path, branch, options, + schema_manager, latest_schema, identifier, uuid); } /// The identifier a catalog knows the table by, on `branch`: a non-main branch is folded into @@ -253,8 +289,9 @@ Result> TableOf(const ScanContext& ctx) { // below. One `TableScan` interface serves both, as Java Paimon serves both through one // `ReadBuilder`. const std::string branch = BranchManager::NormalizeBranch(tmp_options.GetBranch()); - return ManagedTableOf(tmp_options.GetFileSystem(), ctx.GetPath(), branch, ctx.GetOptions(), - ctx.GetSpecificTableSchema(), ctx.GetIdentifier()); + return ManagedTableOf(tmp_options.GetFileSystem(), ctx.GetSpecificFileSystem(), ctx.GetPath(), + branch, ctx.GetOptions(), ctx.GetSpecificTableSchema(), + ctx.GetIdentifier()); } /// The table a read context names, read from what the context carries: a format table the @@ -289,8 +326,9 @@ Result> TableOf(const ReadContext& ctx) { // which files there are and how a row is put back together is its own. How a file is opened // is not: both paths go through `DataFileReaderFactory`. const std::string branch = ctx.GetBranch(); - return ManagedTableOf(tmp_core_options.GetFileSystem(), ctx.GetPath(), branch, ctx.GetOptions(), - ctx.GetSpecificTableSchema(), ctx.GetIdentifier()); + return ManagedTableOf(tmp_core_options.GetFileSystem(), ctx.GetSpecificFileSystem(), + ctx.GetPath(), branch, ctx.GetOptions(), ctx.GetSpecificTableSchema(), + ctx.GetIdentifier()); } /// The table a write context names, read from what the context carries: a format table the @@ -327,8 +365,8 @@ Result> TableOf(const WriteContext& ctx) { CatalogMainBranchSchemaJson(ctx.GetCatalog(), catalog_identifier, branch, "write")); // A format table writes plain data files into a directory, so it never reaches the manifest // path below. - return ManagedTableOf(tmp_options.GetFileSystem(), ctx.GetRootPath(), branch, ctx.GetOptions(), - catalog_table_schema, ctx.GetIdentifier()); + return ManagedTableOf(tmp_options.GetFileSystem(), specific_fs, ctx.GetRootPath(), branch, + ctx.GetOptions(), catalog_table_schema, ctx.GetIdentifier()); } /// The table a commit context names, read from what the context carries: a format table the @@ -370,15 +408,18 @@ Result> TableOf(const CommitContext& ctx) { // A format table commits by renaming files into place, so it never reaches the snapshot path // below. The managed path here reads the commit's branch, so this reads the same one: the two // must not dispatch on different schemas. - return ManagedTableOf(tmp_options.GetFileSystem(), root_path, branch, ctx.GetOptions(), - catalog_table_schema, ctx.GetIdentifier(), ctx.GetTableId().value_or("")); + return ManagedTableOf(tmp_options.GetFileSystem(), specific_fs, root_path, branch, + ctx.GetOptions(), catalog_table_schema, ctx.GetIdentifier(), + ctx.GetTableId().value_or("")); } /// The options a context carries against the ones the table was loaded with. A table serves the /// options it was loaded with, so an option the context contradicts is refused rather than -/// honoured - a caller that means to change them copies the table. A key the table was not loaded -/// with is no contradiction: it names a runtime setting - a scan mode, a batch size - which is -/// none of the table's business, and is served as given. +/// honoured - a caller that means to change them copies the table. A table a load hands back +/// carries the context's own options merged in, so it agrees with the context by construction; +/// the refusal belongs to a table the caller held already, whose load is over. A key the table +/// was not loaded with is no contradiction: it names a runtime setting - a scan mode, a batch +/// size - which is none of the table's business, and is served as given. Status CheckOptionsAgreeWithTable(const TableSchema& table_schema, const std::map& context_options) { const std::map& table_options = table_schema.Options(); @@ -410,33 +451,6 @@ Status CheckOperatorContext(const ContextPointer& context, const char* kind) { return Status::OK(); } -/// The schema the table was constructed with, which a write or a commit serves as the table was -/// loaded rather than reading the schema again. -Result> LatestTableSchemaOfBranch( - const std::shared_ptr& schema, const std::string& branch) { - std::shared_ptr latest_schema = std::static_pointer_cast(schema); - if (latest_schema == nullptr) { - return Status::Invalid(fmt::format("not found latest schema in branch {}", branch)); - } - return latest_schema; -} - -/// The options a write or a commit serves: the ones the table was loaded with - an option the -/// context contradicts is refused by `CheckOptionsAgreeWithTable` above - the ones the context -/// adds on top, and the table's branch, which the metadata paths below are built for: what reads -/// through these options, whose cache keys are scoped by it, has to read the same one. -Result> MergedOperatorOptions( - const TableSchema& table_schema, const std::map& context_options, - const std::string& branch) { - PAIMON_RETURN_NOT_OK(CheckOptionsAgreeWithTable(table_schema, context_options)); - auto opts = table_schema.Options(); - for (const auto& [key, value] : context_options) { - opts[key] = value; - } - opts[Options::BRANCH] = branch; - return opts; -} - } // namespace Table::Table(const std::shared_ptr& file_system, const std::string& root_path, @@ -448,16 +462,10 @@ Table::Table(const std::shared_ptr& file_system, const std::string& identifier_(identifier), uuid_(uuid) {} -FileStoreTable::FileStoreTable(const std::shared_ptr& file_system, - const std::string& root_path, const std::shared_ptr& schema, - const Identifier& identifier, const std::string& uuid, - const std::string& branch, - const std::shared_ptr& schema_manager, - const std::shared_ptr& snapshot_manager) - : Table(file_system, root_path, schema, identifier, uuid), - store_(std::make_shared(file_system, root_path, - std::static_pointer_cast(schema), branch, - schema_manager, snapshot_manager)) {} +FileStoreTable::FileStoreTable(const std::shared_ptr& store, + const Identifier& identifier, const std::string& uuid) + : Table(store->file_system(), store->root_path(), store->schema(), identifier, uuid), + store_(store) {} Result> FileStoreTable::Create( const std::shared_ptr& file_system, const std::string& root_path, @@ -467,15 +475,25 @@ Result> FileStoreTable::Create( // The kind the schema's primary keys name, which no schema change can add or drop, so the // kind the table is built as is the kind every schema its root path ever held describes. A // caller without a schema gets the append-only kind, whose operators then report that no - // schema was found. - std::shared_ptr table_schema = std::static_pointer_cast(schema); + // schema was found. A schema that is not a table schema - the read-only one a system table + // exposes, say - is none to serve a managed table through, so it is refused rather than cast. + std::shared_ptr table_schema; + if (schema != nullptr) { + table_schema = std::dynamic_pointer_cast(schema); + if (table_schema == nullptr) { + return Status::Invalid( + fmt::format("{} is not a managed table schema", identifier.ToString())); + } + } + // The store parses the schema's options, so a schema whose options do not parse is refused + // here: the table it names cannot be served, not by one operator more than another. + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr store, + FileStore::Create(file_system, root_path, table_schema, branch, + schema_manager, snapshot_manager)); if (table_schema != nullptr && !table_schema->PrimaryKeys().empty()) { - return std::make_shared(file_system, root_path, schema, - identifier, uuid, branch, schema_manager, - snapshot_manager); + return std::make_shared(store, identifier, uuid); } - return std::make_shared( - file_system, root_path, schema, identifier, uuid, branch, schema_manager, snapshot_manager); + return std::make_shared(store, identifier, uuid); } Result> FileStoreTable::Copy( @@ -491,8 +509,11 @@ Result> FileStoreTable::Copy( PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merged_schema, loaded_schema->MergeOptions(overridable)); // The managers are handed over, not rebuilt: the copy serves the same root path and branch, - // so its operators read the schema cache this table already warmed. - return FileStoreTable::Create(file_system_, root_path_, + // so its operators read the schema cache this table already warmed. The file system is the + // one the caller opened this table through, not the one it resolved to: the copy resolves + // from its own merged options again, so a file-system option among the ones it was given + // speaks rather than being pinned to this table's resolution. + return FileStoreTable::Create(store_->specified_file_system(), root_path_, std::shared_ptr(std::move(merged_schema)), identifier_, CatalogUuid().value_or(""), CurrentBranch(), GetSchemaManager(), GetSnapshotManager()); @@ -503,8 +524,8 @@ Result> FileStoreTable::SwitchToBranch(const std::string& if (target_branch == CurrentBranch()) { // The table already serves this branch: a copy of it - schema and managers shared - is // the same table to the caller. - return FileStoreTable::Create(file_system_, root_path_, schema_, identifier_, - CatalogUuid().value_or(""), CurrentBranch(), + return FileStoreTable::Create(store_->specified_file_system(), root_path_, schema_, + identifier_, CatalogUuid().value_or(""), CurrentBranch(), GetSchemaManager(), GetSnapshotManager()); } // The schema of the branch switched to, read from its own directory, which a branch without @@ -523,9 +544,10 @@ Result> FileStoreTable::SwitchToBranch(const std::string& } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr branch_schema, (*latest)->MergeOptions(overridable)); - return FileStoreTable::Create( - file_system_, root_path_, std::shared_ptr(std::move(branch_schema)), - identifier_, CatalogUuid().value_or(""), target_branch, schema_manager); + return FileStoreTable::Create(store_->specified_file_system(), root_path_, + std::shared_ptr(std::move(branch_schema)), + identifier_, CatalogUuid().value_or(""), target_branch, + schema_manager); } Status FileStoreTable::RestoreRealtimeCommittedProgress( @@ -569,19 +591,18 @@ Result> FileStoreTable::NewWrite( MakeBranchCatalogIdentifier(ctx->GetIdentifier(), CurrentBranch())); // The schema the table was constructed with, so a write serves the table as it was loaded // rather than reading the schema again. - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, - LatestTableSchemaOfBranch(schema_, CurrentBranch())); + const std::shared_ptr& schema = store_->schema(); + if (schema == nullptr) { + return Status::Invalid( + fmt::format("not found latest schema in branch {}", CurrentBranch())); + } + // The options the store was built with are the ones this write serves: an option the + // context contradicts is refused rather than honoured, and a key the table was not loaded + // with names a runtime setting the write does not take - `Copy` derives the table that one + // belongs to. + PAIMON_RETURN_NOT_OK(CheckOptionsAgreeWithTable(*schema, ctx->GetOptions())); + CoreOptions options = store_->options(); auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(schema->Fields()); - // The options the table was loaded with, with a runtime setting the context adds on top: an - // option the context would have changed is refused rather than honoured, and the branch is - // the table's - the metadata paths below are built for it, so what reads through these - // options, whose cache keys are scoped by it, has to read the same one. - std::map opts; - PAIMON_ASSIGN_OR_RAISE(opts, - MergedOperatorOptions(*schema, ctx->GetOptions(), CurrentBranch())); - PAIMON_ASSIGN_OR_RAISE( - CoreOptions options, - CoreOptions::FromMap(opts, file_system_, ctx->GetFileSystemSchemeToIdentifierMap())); PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*schema, options)); PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateVectorStore(*schema, options)); if (options.GetVectorFileFormat() && !ctx->GetWriteSchema().empty()) { @@ -614,9 +635,8 @@ Result> FileStoreTable::NewWrite( RealtimeContextImpl::Cast(ctx->GetRealtimeContext())); PAIMON_RETURN_NOT_OK(realtime_context_impl->CheckUsable()); } - return CreateWrite(*ctx, schema, arrow_schema, partition_schema, options, - file_store_path_factory, snapshot_manager, io_manager, - ignore_previous_files); + return CreateWrite(*ctx, arrow_schema, partition_schema, options, file_store_path_factory, + snapshot_manager, io_manager, ignore_previous_files); } Result> FileStoreWrite::Create(std::unique_ptr ctx) { @@ -677,34 +697,22 @@ Result> FileStoreTable::NewScan( PAIMON_RETURN_NOT_OK(CheckOperatorContext(context, "scan")); // The schema the table was constructed with, so a scan serves the table as it was loaded // rather than reading the schema again. - std::shared_ptr loaded_schema = std::static_pointer_cast(schema_); - if (loaded_schema == nullptr) { + const std::shared_ptr& table_schema = store_->schema(); + if (table_schema == nullptr) { return Status::Invalid("not found latest schema"); } - const std::shared_ptr& table_schema = loaded_schema; PAIMON_RETURN_NOT_OK(CheckOptionsAgreeWithTable(*table_schema, context->GetOptions())); PAIMON_ASSIGN_OR_RAISE(bool read_optimized, OptionsUtils::GetValueFromMap(context->GetOptions(), kReadOptimizedScanOption, false)); - // The options the table was loaded with, with a runtime setting the context adds on top: an - // option the context would have changed is refused above, so the table's own stand as they - // are. The branch is the table's, not the context's: a table is the branch it was loaded - // for, so a branch the context carries is not honoured. - auto options = table_schema->Options(); - for (const auto& [key, value] : context->GetOptions()) { - if (key != Options::BRANCH) { - options[key] = value; - } - } - if (BranchManager::IsMainBranch(CurrentBranch())) { - options.erase(Options::BRANCH); - } else { - options[Options::BRANCH] = CurrentBranch(); - } - // The file system and root path the table was opened with, so a scan serves the table as it - // was loaded rather than resolving them from the context. - PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, - CoreOptions::FromMap(options, file_system_, {})); + // The options the store was built with are the ones this scan serves: an option the context + // contradicts is refused above, and a key the table was not loaded with names a runtime + // setting the scan does not take - `Copy` derives the table that one belongs to. The branch + // in them is the table's, not the context's: a table is the branch it was loaded for, so a + // branch the context carries is not honoured. + CoreOptions core_options = store_->options(); + // The cache is a runtime setting of this scan, not of the table: the copy the scan serves + // carries it. core_options.WithCache(context->GetCache()); PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*table_schema, core_options)); @@ -748,10 +756,10 @@ Result> FileStoreTable::NewScan( // The managers the table carries, whose schema cache the dispatch already warmed. PAIMON_ASSIGN_OR_RAISE( std::shared_ptr file_store_scan, - NewFileStoreScan(manifest_list, manifest_file, arrow_schema, table_schema, core_options, + NewFileStoreScan(manifest_list, manifest_file, arrow_schema, core_options, context->GetExecutor(), context->GetMemoryPool(), context.get())); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr split_generator, - NewSplitGenerator(table_schema, core_options)); + NewSplitGenerator(core_options)); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr index_file_handler, store_->NewIndexFileHandler(core_options, path_factory, context->GetMemoryPool())); @@ -803,28 +811,24 @@ Result> TableScan::Create(std::unique_ptr> CreateInternalReadContext( +/// The read context of a fallback branch, whose schema is not the table's own: it is read +/// through a manager of that branch, and the options the read serves are the ones that schema +/// was loaded with - the same way the table's own read serves the store's - with the branch it +/// was read from in them. A branch the context carries is not honoured here. +/// @param file_system The file system the table was opened with, which the fallback reads +/// through in place of one resolved from the context. +/// @param root_path The root path the table was opened with, which the fallback reads from in +/// place of the one named by the context. +Result> CreateFallbackReadContext( const std::shared_ptr& context, const std::string& branch, - const std::shared_ptr& schema_manager, - const std::shared_ptr& loaded_schema, const std::shared_ptr& file_system, const std::string& root_path) { std::shared_ptr table_schema; const auto& specific_table_schema = context->GetSpecificTableSchema(); - if (loaded_schema != nullptr) { - table_schema = loaded_schema; - } else if (branch == BranchManager::DEFAULT_MAIN_BRANCH && specific_table_schema) { + if (branch == BranchManager::DEFAULT_MAIN_BRANCH && specific_table_schema) { PAIMON_ASSIGN_OR_RAISE(table_schema, TableSchema::CreateFromJson(specific_table_schema.value())); } else { + auto schema_manager = std::make_shared(file_system, root_path, branch); PAIMON_ASSIGN_OR_RAISE(std::optional> latest_schema, schema_manager->Latest()); if (!latest_schema) { @@ -834,16 +838,7 @@ Result> CreateInternalReadContext( table_schema = latest_schema.value(); } assert(table_schema); - - // The options the schema was loaded with, with a runtime setting the context adds on top - - // and the fallback below serves the ones its own schema was loaded with. The branch is the - // one named below, not the context's: a branch the context carries is not honoured here. auto options = table_schema->Options(); - for (const auto& [key, value] : context->GetOptions()) { - if (key != Options::BRANCH) { - options[key] = value; - } - } if (branch != BranchManager::DEFAULT_MAIN_BRANCH) { options[Options::BRANCH] = branch; } @@ -882,18 +877,20 @@ Result> FileStoreTable::NewRead( PAIMON_RETURN_NOT_OK(CheckOperatorContext(read_context, "read")); // The schema the table was constructed with, so a read serves the table as it was loaded // rather than reading the schema again. Its options are the ones the read serves: an option - // the context would have changed is refused rather than honoured. - std::shared_ptr loaded_schema = std::static_pointer_cast(schema_); + // the context would have changed is refused rather than honoured, and a key the table was + // not loaded with names a runtime setting the read does not take - `Copy` derives the table + // that one belongs to. + const std::shared_ptr& loaded_schema = store_->schema(); if (loaded_schema == nullptr) { return Status::Invalid("not found latest schema"); } PAIMON_RETURN_NOT_OK(CheckOptionsAgreeWithTable(*loaded_schema, read_context->GetOptions())); - // The branch the table serves, which the read reads on: a table is the branch it was loaded - // for, so the context's own branch is not consulted. + // The options the store was built with, which the read reads through on the branch the + // table serves: a table is the branch it was loaded for, so the context's own branch is + // not consulted. PAIMON_ASSIGN_OR_RAISE( std::shared_ptr internal_context, - CreateInternalReadContext(read_context, CurrentBranch(), GetSchemaManager(), loaded_schema, - file_system_, root_path_)); + InternalReadContext::Create(read_context, loaded_schema, store_->options(), &root_path_)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, CreateTableRead(internal_context, read_context->GetMemoryPool(), @@ -910,10 +907,8 @@ Result> FileStoreTable::NewRead( // own rather than the table's. PAIMON_ASSIGN_OR_RAISE( std::shared_ptr fallback_context, - CreateInternalReadContext( - read_context, /*branch=*/scan_fallback_branch.value(), - std::make_shared(file_system_, root_path_, scan_fallback_branch.value()), - /*loaded_schema=*/nullptr, file_system_, root_path_)); + CreateFallbackReadContext(read_context, /*branch=*/scan_fallback_branch.value(), + file_system_, root_path_)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr fallback_table_read, CreateTableRead(fallback_context, read_context->GetMemoryPool(), @@ -1012,18 +1007,19 @@ Result> FileStoreTable::NewCommit( } // The schema the table was constructed with, so a commit serves the table as it was loaded // rather than reading the schema again. - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, - LatestTableSchemaOfBranch(schema_, CurrentBranch())); + const std::shared_ptr& schema = store_->schema(); + if (schema == nullptr) { + return Status::Invalid( + fmt::format("not found latest schema in branch {}", CurrentBranch())); + } std::shared_ptr arrow_schema = DataField::ConvertDataFieldsToArrowSchema(schema->Fields()); - // The options the table was loaded with, with a runtime setting the context adds on top: an - // option the context would have changed is refused rather than honoured, and the branch is - // the table's - the metadata paths below are built for it, so what reads through these - // options, whose cache keys are scoped by it, has to read the same one. - std::map opts; - PAIMON_ASSIGN_OR_RAISE(opts, - MergedOperatorOptions(*schema, ctx->GetOptions(), CurrentBranch())); - PAIMON_ASSIGN_OR_RAISE(CoreOptions options, CoreOptions::FromMap(opts, file_system_)); + // The options the store was built with are the ones this commit serves: an option the + // context contradicts is refused rather than honoured, and a key the table was not loaded + // with names a runtime setting the commit does not take - `Copy` derives the table that one + // belongs to. + PAIMON_RETURN_NOT_OK(CheckOptionsAgreeWithTable(*schema, ctx->GetOptions())); + CoreOptions options = store_->options(); PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*schema, options)); assert(options.GetFileSystem()); assert(options.GetFileFormat()); @@ -1066,8 +1062,8 @@ Result> FileStoreTable::NewCommit( ctx->GetExecutor()); CommitScanner::ScanSupplier scan_supplier = - NewCommitScanSupplier(snapshot_manager, manifest_list, manifest_file, schema, arrow_schema, - options, ctx->GetExecutor(), ctx->GetMemoryPool()); + NewCommitScanSupplier(snapshot_manager, manifest_list, manifest_file, arrow_schema, options, + ctx->GetExecutor(), ctx->GetMemoryPool()); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr snapshot_commit, NewSnapshotCommit(*ctx, catalog_identifier, options, snapshot_manager)); diff --git a/src/paimon/core/table/file_store_table.h b/src/paimon/core/table/file_store_table.h index 00959d257..8df9c4d86 100644 --- a/src/paimon/core/table/file_store_table.h +++ b/src/paimon/core/table/file_store_table.h @@ -148,13 +148,11 @@ class FileStoreTable : public Table { const std::shared_ptr& executor) const; protected: - /// The base constructor the kinds of managed table build through, `Create` above. The - /// parameters are the ones it takes; see there for what each one is. - FileStoreTable(const std::shared_ptr& file_system, const std::string& root_path, - const std::shared_ptr& schema, const Identifier& identifier, - const std::string& uuid, const std::string& branch, - const std::shared_ptr& schema_manager, - const std::shared_ptr& snapshot_manager); + /// The base constructor the kinds of managed table build through, `Create` above: the store + /// the table was loaded with carries every state the kinds hand down - see `Create` for what + /// each piece of it is. + FileStoreTable(const std::shared_ptr& store, const Identifier& identifier, + const std::string& uuid); /// The committed progress a real-time write restores, from the latest snapshot the snapshot /// manager reads: where the write picks up after, which both kinds of real-time write ask @@ -170,32 +168,32 @@ class FileStoreTable : public Table { /// The write of this kind of table, which `NewWrite` hands the shared flow's pieces to: the /// kind's writer, built over the path factory, snapshot manager and options that flow - /// resolved. The branch the write serves is the table's, `CurrentBranch()`. + /// resolved. The schema is the store's own, the one the table was loaded with, and the + /// branch the write serves is the table's, `CurrentBranch()`. /// @param io_manager The spill manager the context's temp directory names, or null when the /// context named none. /// @param ignore_previous_files Whether the write starts over rather than appending, which /// the flow resolved with this kind's own rules. virtual Result> CreateWrite( - const WriteContext& ctx, const std::shared_ptr& schema, - const std::shared_ptr& arrow_schema, + const WriteContext& ctx, const std::shared_ptr& arrow_schema, const std::shared_ptr& partition_schema, const CoreOptions& options, const std::shared_ptr& path_factory, const std::shared_ptr& snapshot_manager, const std::shared_ptr& io_manager, bool ignore_previous_files) const = 0; /// The scan of this kind of table, which `NewScan` hands the manifest list and file it built - /// to. The snapshot and schema managers are the table's own, which the scan serves. + /// to. The schema is the store's own and the snapshot and schema managers are the table's + /// own, which the scan serves. virtual Result> NewFileStoreScan( const std::shared_ptr& manifest_list, const std::shared_ptr& manifest_file, - const std::shared_ptr& arrow_schema, - const std::shared_ptr& table_schema, const CoreOptions& core_options, + const std::shared_ptr& arrow_schema, const CoreOptions& core_options, const std::shared_ptr& executor, const std::shared_ptr& memory_pool, const ScanContext* context) const = 0; - /// The split generator of this kind of table, Java Paimon's `splitGenerator()`. + /// The split generator of this kind of table, Java Paimon's `splitGenerator()`. The schema + /// it reads its kind's keys from is the store's own. virtual Result> NewSplitGenerator( - const std::shared_ptr& table_schema, const CoreOptions& core_options) const = 0; /// The read of this kind of table, which `NewRead` hands the internal context it read the @@ -207,13 +205,12 @@ class FileStoreTable : public Table { const std::shared_ptr& executor) const = 0; /// The manifest entry scan supplier of this kind of table, which `NewCommit` hands the - /// manifest list and file it built to. The schema manager the scans it hands out read through - /// is the table's own. + /// manifest list and file it built to. The schema is the store's own, and the schema manager + /// the scans it hands out read through is the table's own. virtual CommitScanner::ScanSupplier NewCommitScanSupplier( const std::shared_ptr& snapshot_manager, const std::shared_ptr& manifest_list, const std::shared_ptr& manifest_file, - const std::shared_ptr& table_schema, const std::shared_ptr& arrow_schema, const CoreOptions& options, const std::shared_ptr& executor, const std::shared_ptr& pool) const = 0; diff --git a/src/paimon/core/table/primary_key_file_store_table.cpp b/src/paimon/core/table/primary_key_file_store_table.cpp index 564e015e3..06e27e0d1 100644 --- a/src/paimon/core/table/primary_key_file_store_table.cpp +++ b/src/paimon/core/table/primary_key_file_store_table.cpp @@ -96,22 +96,19 @@ Result HasHistoricalPrimaryKeyBTreeDefinition( } // namespace -PrimaryKeyFileStoreTable::PrimaryKeyFileStoreTable( - const std::shared_ptr& file_system, const std::string& root_path, - const std::shared_ptr& schema, const Identifier& identifier, const std::string& uuid, - const std::string& branch, const std::shared_ptr& schema_manager, - const std::shared_ptr& snapshot_manager) - : FileStoreTable(file_system, root_path, schema, identifier, uuid, branch, schema_manager, - snapshot_manager) {} +PrimaryKeyFileStoreTable::PrimaryKeyFileStoreTable(const std::shared_ptr& store, + const Identifier& identifier, + const std::string& uuid) + : FileStoreTable(store, identifier, uuid) {} Result> PrimaryKeyFileStoreTable::CreateWrite( - const WriteContext& ctx, const std::shared_ptr& schema, - const std::shared_ptr& arrow_schema, + const WriteContext& ctx, const std::shared_ptr& arrow_schema, const std::shared_ptr& partition_schema, const CoreOptions& options, const std::shared_ptr& path_factory, const std::shared_ptr& snapshot_manager, const std::shared_ptr& io_manager, bool ignore_previous_files) const { // pk table + const std::shared_ptr& schema = store_->schema(); if (ctx.GetRealtimeContext()) { PAIMON_RETURN_NOT_OK(PrimaryKeyTableUtils::ValidateRealtimeOptions(options, *schema)); if (ignore_previous_files) { @@ -227,11 +224,11 @@ Result> PrimaryKeyFileStoreTable::CreateWrite( Result> PrimaryKeyFileStoreTable::NewFileStoreScan( const std::shared_ptr& manifest_list, const std::shared_ptr& manifest_file, - const std::shared_ptr& arrow_schema, - const std::shared_ptr& table_schema, const CoreOptions& core_options, + const std::shared_ptr& arrow_schema, const CoreOptions& core_options, const std::shared_ptr& executor, const std::shared_ptr& memory_pool, const ScanContext* context) const { // TODO(liancheng.lsz): support fallback branch in scan + const std::shared_ptr& table_schema = store_->schema(); std::unique_ptr scan; PAIMON_ASSIGN_OR_RAISE( scan, KeyValueFileStoreScan::Create(GetSnapshotManager(), GetSchemaManager(), manifest_list, @@ -242,10 +239,11 @@ Result> PrimaryKeyFileStoreTable::NewFileStoreSca } Result> PrimaryKeyFileStoreTable::NewSplitGenerator( - const std::shared_ptr& table_schema, const CoreOptions& core_options) const { + const CoreOptions& core_options) const { auto source_split_target_size = core_options.GetSourceSplitTargetSize(); auto source_split_open_file_cost = core_options.GetSourceSplitOpenFileCost(); // TODO(liancheng.lsz): support evolution + const std::shared_ptr& table_schema = store_->schema(); PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_keys, table_schema->TrimmedPrimaryKeys()); PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_pk_fields, @@ -270,12 +268,12 @@ CommitScanner::ScanSupplier PrimaryKeyFileStoreTable::NewCommitScanSupplier( const std::shared_ptr& snapshot_manager, const std::shared_ptr& manifest_list, const std::shared_ptr& manifest_file, - const std::shared_ptr& table_schema, const std::shared_ptr& arrow_schema, const CoreOptions& options, const std::shared_ptr& executor, const std::shared_ptr& pool) const { // The schema manager the table carries, which the scans this supplier hands out read the // schema through. const std::shared_ptr& schema_manager = GetSchemaManager(); + const std::shared_ptr& table_schema = store_->schema(); return [snapshot_manager, schema_manager, manifest_list, manifest_file, table_schema, arrow_schema, options, executor, pool](const std::shared_ptr& scan_filter) -> Result> { diff --git a/src/paimon/core/table/primary_key_file_store_table.h b/src/paimon/core/table/primary_key_file_store_table.h index 9c658758a..9b0ed5c9a 100644 --- a/src/paimon/core/table/primary_key_file_store_table.h +++ b/src/paimon/core/table/primary_key_file_store_table.h @@ -24,13 +24,11 @@ #include "paimon/catalog/identifier.h" #include "paimon/core/table/file_store_table.h" -#include "paimon/core/utils/branch_manager.h" #include "paimon/result.h" #include "paimon/type_fwd.h" namespace paimon { -class CoreOptions; class FileStoreScan; class IOManager; class SplitGenerator; @@ -42,19 +40,14 @@ class TableSchema; /// those flows hand down are built here. class PrimaryKeyFileStoreTable : public FileStoreTable { public: - /// The constructor `FileStoreTable::Create` builds a primary key table through; the - /// parameters are the ones it takes, and the base class carries them. - PrimaryKeyFileStoreTable(const std::shared_ptr& file_system, - const std::string& root_path, const std::shared_ptr& schema, - const Identifier& identifier, const std::string& uuid = "", - const std::string& branch = BranchManager::DEFAULT_MAIN_BRANCH, - const std::shared_ptr& schema_manager = nullptr, - const std::shared_ptr& snapshot_manager = nullptr); + /// The constructor `FileStoreTable::Create` builds a primary key table through: the store + /// carries every state, and the base class holds it. + PrimaryKeyFileStoreTable(const std::shared_ptr& store, const Identifier& identifier, + const std::string& uuid = ""); private: Result> CreateWrite( - const WriteContext& ctx, const std::shared_ptr& schema, - const std::shared_ptr& arrow_schema, + const WriteContext& ctx, const std::shared_ptr& arrow_schema, const std::shared_ptr& partition_schema, const CoreOptions& options, const std::shared_ptr& path_factory, const std::shared_ptr& snapshot_manager, @@ -63,13 +56,11 @@ class PrimaryKeyFileStoreTable : public FileStoreTable { Result> NewFileStoreScan( const std::shared_ptr& manifest_list, const std::shared_ptr& manifest_file, - const std::shared_ptr& arrow_schema, - const std::shared_ptr& table_schema, const CoreOptions& core_options, + const std::shared_ptr& arrow_schema, const CoreOptions& core_options, const std::shared_ptr& executor, const std::shared_ptr& memory_pool, const ScanContext* context) const override; Result> NewSplitGenerator( - const std::shared_ptr& table_schema, const CoreOptions& core_options) const override; Result> NewTableRead( @@ -82,7 +73,6 @@ class PrimaryKeyFileStoreTable : public FileStoreTable { const std::shared_ptr& snapshot_manager, const std::shared_ptr& manifest_list, const std::shared_ptr& manifest_file, - const std::shared_ptr& table_schema, const std::shared_ptr& arrow_schema, const CoreOptions& options, const std::shared_ptr& executor, const std::shared_ptr& pool) const override; diff --git a/src/paimon/core/table/source/table_read_test.cpp b/src/paimon/core/table/source/table_read_test.cpp index 3d580d64d..b6aef3992 100644 --- a/src/paimon/core/table/source/table_read_test.cpp +++ b/src/paimon/core/table/source/table_read_test.cpp @@ -251,7 +251,7 @@ TEST(TableReadTest, TestCreateAppendOnlyTableRead) { TEST(TableReadTest, TestMergeOptions) { std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; { - // The options the table was loaded with stand as they are, and a runtime setting the + // The options the schema carries stand in the table the load builds, and a setting the // context adds on top - the table was loaded with neither of these keys - is served as // given. ReadContextBuilder context_builder(path); @@ -276,14 +276,21 @@ TEST(TableReadTest, TestMergeOptions) { ASSERT_EQ(expected_options, core_options.ToMap()); } { - // An option that contradicts the table is refused: the read serves the options the table - // was loaded with, and a caller that means to change them copies the table. + // An option the schema also carries speaks at the load: the table the context is + // handed back is built with the context's value over the schema's, and the read serves + // the merged options. A caller holding a table already - a load that is over - is the + // one that gets a refusal instead, which `TableTest` covers. ReadContextBuilder context_builder(path); context_builder.SetReadFieldNames({"f0", "f1", "f2", "f3"}); context_builder.AddOption("bucket", "10"); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), - "option 'bucket' is '10' in the context but '2' on the table"); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + auto append_only_table_read = dynamic_cast(table_read.get()); + ASSERT_TRUE(append_only_table_read); + auto* typed_split_read = + dynamic_cast(append_only_table_read->split_reads_[0].get()); + ASSERT_TRUE(typed_split_read); + ASSERT_EQ(10, typed_split_read->options_.GetBucket()); } } diff --git a/src/paimon/core/table/system/audit_log_system_table.cpp b/src/paimon/core/table/system/audit_log_system_table.cpp index 3d3f7b3e5..7254dff60 100644 --- a/src/paimon/core/table/system/audit_log_system_table.cpp +++ b/src/paimon/core/table/system/audit_log_system_table.cpp @@ -395,7 +395,12 @@ Result> AuditLogSystemTable::NewScan( .WithStreamingMode(context->IsStreamingMode()) .WithMemoryPool(context->GetMemoryPool()) .WithExecutor(context->GetExecutor()) - .WithFileSystem(context->GetSpecificFileSystem()) + // The data table behind this view is served through the file system the view itself was + // opened with, so a context without one of its own falls back to it rather than the + // default backend. + .WithFileSystem(context->GetSpecificFileSystem() != nullptr + ? context->GetSpecificFileSystem() + : GetFileSystem()) .WithCache(context->GetCache()); if (scan_filter) { if (scan_filter->GetBucketFilter()) { @@ -458,7 +463,12 @@ Result> AuditLogSystemTable::CreateDataReadContext( .WithBranch(core_options.GetBranch()) .WithMemoryPool(context->GetMemoryPool()) .WithExecutor(context->GetExecutor()) - .WithFileSystem(context->GetSpecificFileSystem()) + // The data table behind this view is served through the file system the view itself was + // opened with, so a context without one of its own falls back to it rather than the + // default backend. + .WithFileSystem(context->GetSpecificFileSystem() != nullptr + ? context->GetSpecificFileSystem() + : GetFileSystem()) .WithFileSystemSchemeToIdentifierMap(context->GetFileSystemSchemeToIdentifierMap()) .EnablePrefetch(context->EnablePrefetch()) .SetPrefetchBatchCount(context->GetPrefetchBatchCount()) diff --git a/src/paimon/core/table/system/read_optimized_system_table.cpp b/src/paimon/core/table/system/read_optimized_system_table.cpp index 172fc964c..0d44b5989 100644 --- a/src/paimon/core/table/system/read_optimized_system_table.cpp +++ b/src/paimon/core/table/system/read_optimized_system_table.cpp @@ -71,7 +71,12 @@ Result> ReadOptimizedSystemTable::NewScan( .WithStreamingMode(context->IsStreamingMode()) .WithMemoryPool(context->GetMemoryPool()) .WithExecutor(context->GetExecutor()) - .WithFileSystem(context->GetSpecificFileSystem()) + // The data table behind this view is served through the file system the view itself was + // opened with, so a context without one of its own falls back to it rather than the + // default backend. + .WithFileSystem(context->GetSpecificFileSystem() != nullptr + ? context->GetSpecificFileSystem() + : GetFileSystem()) .WithCache(context->GetCache()); if (context->GetLimit().has_value()) { builder.SetLimit(context->GetLimit().value()); @@ -114,7 +119,12 @@ Result> ReadOptimizedSystemTable::CreateDataReadCon .SetRowToBatchThreadNumber(context->GetRowToBatchThreadNumber()) .WithMemoryPool(context->GetMemoryPool()) .WithExecutor(context->GetExecutor()) - .WithFileSystem(context->GetSpecificFileSystem()) + // The data table behind this view is served through the file system the view itself was + // opened with, so a context without one of its own falls back to it rather than the + // default backend. + .WithFileSystem(context->GetSpecificFileSystem() != nullptr + ? context->GetSpecificFileSystem() + : GetFileSystem()) .WithFileSystemSchemeToIdentifierMap(context->GetFileSystemSchemeToIdentifierMap()) .SetReadAheadCacheEnabled(context->ReadAheadCacheEnabled()) .WithCacheConfig(context->GetCacheConfig()) diff --git a/src/paimon/core/table/table_test.cpp b/src/paimon/core/table/table_test.cpp index fe99efe63..7fde349d9 100644 --- a/src/paimon/core/table/table_test.cpp +++ b/src/paimon/core/table/table_test.cpp @@ -28,6 +28,7 @@ #include "arrow/c/abi.h" #include "arrow/c/bridge.h" #include "gtest/gtest.h" +#include "paimon/cache/cache.h" #include "paimon/commit_context.h" #include "paimon/commit_message.h" #include "paimon/common/utils/checked_cast.h" @@ -72,6 +73,57 @@ Result> CreateManagedTable( /*uuid=*/"", branch); } +/// Counts the lookups a scan makes, which is enough to tell that the cache a context carries +/// reached the scan's manifest reading rather than being dropped on the way. +class CountingCache : public Cache { + public: + Result> Get( + const std::shared_ptr& key, + std::function>(const std::shared_ptr&)> + supplier) override { + gets++; + return supplier(key); + } + Status Put(const std::shared_ptr&, const std::shared_ptr&) override { + return Status::OK(); + } + void Invalidate(const std::shared_ptr&) override {} + void InvalidateAll() override {} + size_t Size() const override { + return 0; + } + + int32_t gets = 0; +}; + +/// Writes two rows through the table's own write and commit, so a scan below has data to find. +void WriteAndCommitTwoRows(const std::shared_ptr
& table, const std::string& root_path) { + WriteContextBuilder write_builder(root_path, "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write, + table->NewWrite(std::move(write_context))); + arrow::Int32Builder ids_builder; + ASSERT_TRUE(ids_builder.AppendValues({1, 2}).ok()); + std::shared_ptr ids_array; + ASSERT_TRUE(ids_builder.Finish(&ids_array).ok()); + auto struct_array = arrow::StructArray::Make({ids_array}, {arrow::field("id", arrow::int32(), + /*nullable=*/false)}) + .ValueOrDie(); + ::ArrowArray arrow_array; + ASSERT_TRUE(arrow::ExportArray(*struct_array, &arrow_array).ok()); + RecordBatchBuilder batch_builder(&arrow_array); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, batch_builder.SetBucket(0).Finish()); + ASSERT_OK(write->Write(std::move(batch))); + ASSERT_OK_AND_ASSIGN(std::vector> messages, + write->PrepareCommit()); + CommitContextBuilder commit_builder(root_path, "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, + table->NewCommit(std::move(commit_context))); + ASSERT_OK(commit->Commit(messages)); + ASSERT_OK(write->Close()); +} + } // namespace TEST(TableTest, TestTableCarriesManagersOfItsBranch) { @@ -387,10 +439,11 @@ TEST(TableTest, TestCatalogOpenedTableServesThePerTableFileSystem) { auto catalog = std::make_shared(); catalog->SetTableSchema(table->LatestSchema()); - // A catalog that hands out no per-table file system opens the table without one, and one - // that does opens it with the file system issued for that table. + // A catalog that hands out no per-table file system opens the table through the one its + // schema's options resolve - never through none, the default local one when they name no + // other - and one that does opens it with the file system issued for that table. ASSERT_OK_AND_ASSIGN(std::shared_ptr
opened, catalog->GetTable(identifier)); - EXPECT_EQ(opened->GetFileSystem(), nullptr); + EXPECT_NE(opened->GetFileSystem(), nullptr); catalog->SetTableFileSystem(dir->GetFileSystem()); ASSERT_OK_AND_ASSIGN(opened, catalog->GetTable(identifier)); @@ -723,6 +776,92 @@ TEST(TableTest, TestOperatorsRefuseAnOptionThatContradictsTheTable) { copied->NewScan(std::move(ctx))); } +TEST(TableTest, TestOperatorsIgnoreARuntimeKeyTheTableWasNotLoadedWith) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + CreateManagedTable(dir->GetFileSystem(), dir->Str(), identifier, + /*branch=*/"main", + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + WriteAndCommitTwoRows(table, dir->Str()); + + // `scan.mode` and `scan.snapshot-id` name no option the table was loaded with, so they name + // a runtime setting the scan does not take: the scan serves the table's own options and + // reads its latest snapshot - a merged context would have gone looking for snapshot 999, + // which was never committed. + ScanContextBuilder scan_builder(dir->Str()); + scan_builder.AddOption(Options::SCAN_MODE, "from-snapshot") + .AddOption(Options::SCAN_SNAPSHOT_ID, "999"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_ctx, scan_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(scan_ctx))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_FALSE(plan->Splits().empty()); + ASSERT_EQ(plan->SnapshotId(), std::optional(1)); + + // A read takes a runtime key the same way: the key names nothing the table carries, so the + // read is served rather than refused. + ReadContextBuilder read_builder(dir->Str()); + read_builder.AddOption(Options::SCAN_MODE, "from-snapshot") + .AddOption(Options::SCAN_SNAPSHOT_ID, "999"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_ctx, read_builder.Finish()); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr read, + table->NewRead(std::move(read_ctx))); + + // The runtime setting a caller means to serve is the copy's: `Table::Copy` derives the table + // whose options carry it, and that table's scan reads the snapshot it names - which is not + // there, as this table only ever committed snapshot 1. + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
copied, + table->Copy({{Options::SCAN_MODE, "from-snapshot"}, {Options::SCAN_SNAPSHOT_ID, "999"}})); + ASSERT_OK_AND_ASSIGN(std::unique_ptr copied_scan_ctx, + ScanContextBuilder(dir->Str()).Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr copied_scan, + copied->NewScan(std::move(copied_scan_ctx))); + ASSERT_NOK_WITH_MSG(copied_scan->CreatePlan(), "snapshot"); +} + +TEST(TableTest, TestCreateFailsWhenTheSchemaOptionsDoNotParse) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + // The schema itself carries no validation, so a schema written with options no + // `CoreOptions` accepts surfaces when the store parses them - at the load, not at the first + // operator built on the table. + ASSERT_OK_AND_ASSIGN( + std::unique_ptr created, + TableSchema::Create( + /*schema_id=*/0, + arrow::schema({arrow::field("id", arrow::int32(), + /*nullable=*/false)}), + /*partition_keys=*/{}, /*primary_keys=*/{}, {{Options::TARGET_FILE_ROW_NUM, "0"}})); + auto schema = checked_pointer_cast(std::shared_ptr(std::move(created))); + ASSERT_NOK_WITH_MSG( + FileStoreTable::Create(dir->GetFileSystem(), dir->Str(), schema, Identifier("db", "tbl")), + "target-file-row-num should be at least 1"); +} + +TEST(TableTest, TestTheCacheAScanContextCarriesReachesTheScan) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + CreateManagedTable(dir->GetFileSystem(), dir->Str(), identifier, + /*branch=*/"main", + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + WriteAndCommitTwoRows(table, dir->Str()); + + // The cache is a runtime setting the scan serves from the context rather than from the + // table's own options, so a scan built with one reads its manifests through it. + auto cache = std::make_shared(); + ScanContextBuilder scan_builder(dir->Str()); + scan_builder.WithCache(cache); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_ctx, scan_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(scan_ctx))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_FALSE(plan->Splits().empty()); + ASSERT_GT(cache->gets, 0) << "the cache the scan context carries never reached the scan"; +} + TEST(TableTest, TestCopyServesTheTableWithTheGivenOptionsOnTop) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); diff --git a/src/paimon/rest/rest_catalog.cpp b/src/paimon/rest/rest_catalog.cpp index 712a351cf..26a976d94 100644 --- a/src/paimon/rest/rest_catalog.cpp +++ b/src/paimon/rest/rest_catalog.cpp @@ -435,6 +435,24 @@ Result> RestCatalog::LoadSystemTable( } Result> RestCatalog::GetTable(const Identifier& identifier) const { + // The global system tables of the "sys" database expose the catalog itself rather than any + // table the server stands behind, so they are served locally - the way `LoadTableSchema` + // serves them their schema - and never reach the managed-table load below, whose schema + // cast a read-only system table schema would not survive. + if (CatalogUtils::IsSystemDatabase(identifier.GetDatabaseName())) { + PAIMON_ASSIGN_OR_RAISE(bool supported, + GlobalSystemTableLoader::IsSupported(identifier.GetObjectName(), + api_->GetMergedOptions())); + if (!supported) { + return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); + } + GlobalSystemTableContext context; + context.catalog = this; + context.fs = fs_; + context.warehouse = warehouse_; + context.catalog_options = api_->GetMergedOptions(); + return GlobalSystemTableLoader::Load(identifier.GetObjectName(), context); + } // A `T$system` identifier opens the read-only view it names, which reads the data table it // belongs to and answers for itself through the same `Table` interface. PAIMON_ASSIGN_OR_RAISE(bool is_system_table, identifier.IsSystemTable()); diff --git a/test/inte/scan_and_read_inte_test.cpp b/test/inte/scan_and_read_inte_test.cpp index 41d35de8d..4fa4e1da1 100644 --- a/test/inte/scan_and_read_inte_test.cpp +++ b/test/inte/scan_and_read_inte_test.cpp @@ -813,8 +813,17 @@ TEST_P(ScanAndReadInteTest, TestWithAppendBucketKeyPointLookup) { auto lookup_predicate = PredicateBuilder::Equal( 0, "rowkey", field_type, Literal(field_type, lookup_key.data(), lookup_key.size())); + // The rescaled bucket count and, with it, the cache settings `FinishScanContext` + // also puts on the context: a table already held takes its runtime settings + // through the copy it is made of, and the context carries them only to agree + // with it. + std::map dynamic_options{ + {Options::BUCKET, current_bucket_count}}; + if (EnableSnapshotLiveManifestCache()) { + dynamic_options.emplace(Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, "3"); + } ASSERT_OK_AND_ASSIGN(std::shared_ptr
rescaled_table, - table->Copy({{Options::BUCKET, current_bucket_count}})); + table->Copy(dynamic_options)); ScanContextBuilder scan_builder(table_path); scan_builder.SetPredicate(lookup_predicate) .AddOption(Options::BUCKET, current_bucket_count); @@ -4065,8 +4074,14 @@ TEST_P(ScanAndReadInteTest, TestWithPKBucketSelectByPredicate) { catalog->GetTable(Identifier("pk_table_scan_and_read_dv", "pk_table_scan_and_read_dv"))); for (const std::string current_bucket_count : {"2", "4", "8", "17"}) { SCOPED_TRACE(current_bucket_count); - ASSERT_OK_AND_ASSIGN(std::shared_ptr
rescaled_table, - table->Copy({{Options::BUCKET, current_bucket_count}})); + // The rescaled bucket count and, with it, the cache settings `FinishScanContext` also + // puts on the context: a table already held takes its runtime settings through the copy + // it is made of, and the context carries them only to agree with it. + std::map dynamic_options{{Options::BUCKET, current_bucket_count}}; + if (EnableSnapshotLiveManifestCache()) { + dynamic_options.emplace(Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, "3"); + } + ASSERT_OK_AND_ASSIGN(std::shared_ptr
rescaled_table, table->Copy(dynamic_options)); ScanContextBuilder scan_context_builder(table_path); scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "6"); scan_context_builder.AddOption(Options::BUCKET, current_bucket_count); From 135c1b4d375f6075b9df289ee9e1da371f082849 Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Fri, 23 Oct 2026 14:23:42 +0800 Subject: [PATCH 07/30] fix --- .../core/operation/internal_read_context.cpp | 4 +- src/paimon/core/table/file_store.cpp | 16 ++-- src/paimon/core/table/file_store.h | 21 ++++- src/paimon/core/table/file_store_table.cpp | 87 +++++++++++++++---- src/paimon/core/table/file_store_table.h | 6 +- .../table/system/audit_log_system_table.cpp | 15 ++-- .../system/read_optimized_system_table.cpp | 15 ++-- src/paimon/rest/rest_catalog.cpp | 14 +-- 8 files changed, 131 insertions(+), 47 deletions(-) diff --git a/src/paimon/core/operation/internal_read_context.cpp b/src/paimon/core/operation/internal_read_context.cpp index 778976a46..784123828 100644 --- a/src/paimon/core/operation/internal_read_context.cpp +++ b/src/paimon/core/operation/internal_read_context.cpp @@ -290,8 +290,8 @@ Result> InternalReadContext::Create( // resolves its own. PAIMON_ASSIGN_OR_RAISE( CoreOptions core_options, - CoreOptions::FromMap(options, file_system != nullptr ? file_system - : context->GetSpecificFileSystem())); + CoreOptions::FromMap( + options, file_system != nullptr ? file_system : context->GetSpecificFileSystem())); core_options.WithCache(context->GetCache()); return CreateFromCoreOptions(context, table_schema, std::move(core_options), path); } diff --git a/src/paimon/core/table/file_store.cpp b/src/paimon/core/table/file_store.cpp index 2c5a19d96..472626a01 100644 --- a/src/paimon/core/table/file_store.cpp +++ b/src/paimon/core/table/file_store.cpp @@ -51,13 +51,15 @@ Result> FileStore::Create( const std::shared_ptr& file_system, const std::string& root_path, const std::shared_ptr& schema, const std::string& branch, const std::shared_ptr& schema_manager, - const std::shared_ptr& snapshot_manager) { - auto store = std::shared_ptr( - new FileStore(file_system, root_path, schema, branch, schema_manager, snapshot_manager)); + const std::shared_ptr& snapshot_manager, + const std::map& fs_scheme_map) { + auto store = std::shared_ptr(new FileStore( + file_system, root_path, schema, branch, schema_manager, snapshot_manager, fs_scheme_map)); if (schema != nullptr) { // The options the schema was loaded with, on the branch this store serves: what reads // through them - whose cache keys are scoped by the branch - reads the same one the - // managers point at. + // managers point at. The scheme map is the one the schema was read through, so a scheme + // it mapped resolves here the way it resolved there. std::map options = schema->Options(); if (BranchManager::IsMainBranch(store->branch_)) { options.erase(Options::BRANCH); @@ -65,7 +67,7 @@ Result> FileStore::Create( options[Options::BRANCH] = store->branch_; } PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, - CoreOptions::FromMap(options, file_system, {})); + CoreOptions::FromMap(options, file_system, fs_scheme_map)); store->core_options_ = std::move(core_options); } // The file system the table is served through: the one the options resolve - a schema @@ -98,9 +100,11 @@ Result> FileStore::Create( FileStore::FileStore(const std::shared_ptr& file_system, const std::string& root_path, const std::shared_ptr& schema, const std::string& branch, const std::shared_ptr& schema_manager, - const std::shared_ptr& snapshot_manager) + const std::shared_ptr& snapshot_manager, + const std::map& fs_scheme_map) : file_system_(file_system), specified_file_system_(file_system), + fs_scheme_map_(fs_scheme_map), root_path_(root_path), schema_(schema), branch_(BranchManager::NormalizeBranch(branch)), diff --git a/src/paimon/core/table/file_store.h b/src/paimon/core/table/file_store.h index 6c2396bd6..66ccc09c2 100644 --- a/src/paimon/core/table/file_store.h +++ b/src/paimon/core/table/file_store.h @@ -19,6 +19,7 @@ #pragma once +#include #include #include #include @@ -77,6 +78,11 @@ class FileStore { /// @param snapshot_manager A snapshot manager to carry instead of building one. /// A null manager is built over `root_path` on `branch`, so the operators always read /// through the managers the store carries. + /// @param fs_scheme_map The scheme-to-identifier map the caller's contexts resolve custom + /// schemes through. It takes part in building the store's options the way it took + /// part in reading the schema: without it a schema naming a custom scheme would + /// resolve to the default backend once its operators run. It is kept for a copy to + /// hand over again, so the copy resolves as this store does. /// /// Building the store parses the schema's options, so it fails when they do not parse: the /// error is the caller's to see, which is why a store is built through this factory rather @@ -85,7 +91,8 @@ class FileStore { const std::shared_ptr& file_system, const std::string& root_path, const std::shared_ptr& schema, const std::string& branch, const std::shared_ptr& schema_manager = nullptr, - const std::shared_ptr& snapshot_manager = nullptr); + const std::shared_ptr& snapshot_manager = nullptr, + const std::map& fs_scheme_map = {}); /// The branch this store serves, as resolved when it was built. const std::string& branch() const { @@ -125,6 +132,12 @@ class FileStore { return specified_file_system_; } + /// The scheme-to-identifier map the options were resolved with, kept for a copy to resolve + /// through again: see `Create`'s `fs_scheme_map`. + const std::map& fs_scheme_map() const { + return fs_scheme_map_; + } + /// The root path of the table in its file system, as the table was opened with it. const std::string& root_path() const { return root_path_; @@ -199,13 +212,17 @@ class FileStore { FileStore(const std::shared_ptr& file_system, const std::string& root_path, const std::shared_ptr& schema, const std::string& branch, const std::shared_ptr& schema_manager, - const std::shared_ptr& snapshot_manager); + const std::shared_ptr& snapshot_manager, + const std::map& fs_scheme_map); std::shared_ptr file_system_; /// The file system the caller opened the table through, as handed to `Create` - null when /// the call had none - kept for a copy to hand over again: what `file_system_` holds is /// resolved, and would pin the copy to this table's resolution. std::shared_ptr specified_file_system_; + /// The scheme-to-identifier map the options were resolved with, kept for a copy to resolve + /// through again. + std::map fs_scheme_map_; std::string root_path_; std::shared_ptr schema_; std::string branch_; diff --git a/src/paimon/core/table/file_store_table.cpp b/src/paimon/core/table/file_store_table.cpp index e409f4249..cbf2c41d7 100644 --- a/src/paimon/core/table/file_store_table.cpp +++ b/src/paimon/core/table/file_store_table.cpp @@ -36,6 +36,7 @@ #include "paimon/common/utils/checked_cast.h" #include "paimon/common/utils/options_utils.h" #include "paimon/common/utils/path_util.h" +#include "paimon/common/utils/sensitive_config_utils.h" #include "paimon/common/utils/string_utils.h" #include "paimon/core/catalog/catalog_snapshot_commit.h" #include "paimon/core/catalog/catalog_utils.h" @@ -142,6 +143,26 @@ Result> TryLoadFormatTable( /*location_carries_paimon_metadata=*/true, options); } +/// Whether an option of a copy names the file system its store resolves: the backend +/// `file-system` picks, or the access configuration a `fs.` key carries. A copy whose dynamic +/// options change one of these resolves to another file system, so the managers of the table it +/// was copied from - built over that table's resolution - are not its to inherit: they would +/// keep reading metadata through the backend the copy just left. +bool FileSystemResolutionMayChange(const TableSchema& schema, + const std::map& dynamic_options) { + const std::map& schema_options = schema.Options(); + for (const auto& [key, value] : dynamic_options) { + if (key != Options::FILE_SYSTEM && key.rfind("fs.", 0) != 0) { + continue; + } + auto schema_option = schema_options.find(key); + if (schema_option == schema_options.end() || schema_option->second != value) { + return true; + } + } + return false; +} + /// The options a context carries, merged into the table the load hands back: a table is the /// product of the load, so the options speak there - the way `Copy` merges them, over the ones /// the schema carries - and `NewXxx` on the product serves them without merging again. A @@ -184,7 +205,7 @@ Result> TableOfLoadedSchema( const std::string& branch, const std::map& options, const std::shared_ptr& schema_manager, const std::shared_ptr& latest_schema, const std::optional& identifier, - const std::string& uuid = "") { + const std::string& uuid = "", const std::map& fs_scheme_map = {}) { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, TryLoadFormatTable(file_system, path, branch, options, latest_schema)); if (format_table != nullptr) { @@ -196,7 +217,8 @@ Result> TableOfLoadedSchema( identifier.value_or(Identifier(Identifier::kUnknownDatabase, PathUtil::GetName(path))); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, FileStoreTable::Create(explicit_file_system, path, latest_schema, - table_identifier, uuid, branch, schema_manager)); + table_identifier, uuid, branch, schema_manager, + /*snapshot_manager=*/nullptr, fs_scheme_map)); return WithContextOptions(table, latest_schema, options); } @@ -214,7 +236,7 @@ Result> ManagedTableOf( const std::shared_ptr& explicit_file_system, const std::string& path, const std::string& branch, const std::map& options, const std::optional& schema_json, const std::optional& identifier, - const std::string& uuid = "") { + const std::string& uuid = "", const std::map& fs_scheme_map = {}) { auto schema_manager = std::make_shared(file_system, path, branch); std::shared_ptr latest_schema; if (branch == BranchManager::DEFAULT_MAIN_BRANCH && schema_json) { @@ -229,7 +251,7 @@ Result> ManagedTableOf( } } return TableOfLoadedSchema(file_system, explicit_file_system, path, branch, options, - schema_manager, latest_schema, identifier, uuid); + schema_manager, latest_schema, identifier, uuid, fs_scheme_map); } /// The identifier a catalog knows the table by, on `branch`: a non-main branch is folded into @@ -326,9 +348,12 @@ Result> TableOf(const ReadContext& ctx) { // which files there are and how a row is put back together is its own. How a file is opened // is not: both paths go through `DataFileReaderFactory`. const std::string branch = ctx.GetBranch(); + // The scheme map the schema was read through, so the table it describes resolves custom + // schemes the same way - the data belongs to the metadata, both sit behind the map. return ManagedTableOf(tmp_core_options.GetFileSystem(), ctx.GetSpecificFileSystem(), ctx.GetPath(), branch, ctx.GetOptions(), ctx.GetSpecificTableSchema(), - ctx.GetIdentifier()); + ctx.GetIdentifier(), /*uuid=*/"", + ctx.GetFileSystemSchemeToIdentifierMap()); } /// The table a write context names, read from what the context carries: a format table the @@ -366,7 +391,8 @@ Result> TableOf(const WriteContext& ctx) { // A format table writes plain data files into a directory, so it never reaches the manifest // path below. return ManagedTableOf(tmp_options.GetFileSystem(), specific_fs, ctx.GetRootPath(), branch, - ctx.GetOptions(), catalog_table_schema, ctx.GetIdentifier()); + ctx.GetOptions(), catalog_table_schema, ctx.GetIdentifier(), /*uuid=*/"", + ctx.GetFileSystemSchemeToIdentifierMap()); } /// The table a commit context names, read from what the context carries: a format table the @@ -426,9 +452,12 @@ Status CheckOptionsAgreeWithTable(const TableSchema& table_schema, for (const auto& [key, value] : context_options) { auto table_option = table_options.find(key); if (table_option != table_options.end() && table_option->second != value) { + // A value may be a credential, and the error travels into logs: it is redacted the + // way every surface that exposes options redacts them. return Status::Invalid( - fmt::format("option '{}' is '{}' in the context but '{}' on the table", key, value, - table_option->second)); + fmt::format("option '{}' is '{}' in the context but '{}' on the table", key, + SensitiveConfigUtils::RedactValue(key, value), + SensitiveConfigUtils::RedactValue(key, table_option->second))); } } return Status::OK(); @@ -471,7 +500,8 @@ Result> FileStoreTable::Create( const std::shared_ptr& file_system, const std::string& root_path, const std::shared_ptr& schema, const Identifier& identifier, const std::string& uuid, const std::string& branch, const std::shared_ptr& schema_manager, - const std::shared_ptr& snapshot_manager) { + const std::shared_ptr& snapshot_manager, + const std::map& fs_scheme_map) { // The kind the schema's primary keys name, which no schema change can add or drop, so the // kind the table is built as is the kind every schema its root path ever held describes. A // caller without a schema gets the append-only kind, whose operators then report that no @@ -489,7 +519,7 @@ Result> FileStoreTable::Create( // here: the table it names cannot be served, not by one operator more than another. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr store, FileStore::Create(file_system, root_path, table_schema, branch, - schema_manager, snapshot_manager)); + schema_manager, snapshot_manager, fs_scheme_map)); if (table_schema != nullptr && !table_schema->PrimaryKeys().empty()) { return std::make_shared(store, identifier, uuid); } @@ -508,15 +538,22 @@ Result> FileStoreTable::Copy( overridable.erase(Options::TYPE); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merged_schema, loaded_schema->MergeOptions(overridable)); - // The managers are handed over, not rebuilt: the copy serves the same root path and branch, - // so its operators read the schema cache this table already warmed. The file system is the - // one the caller opened this table through, not the one it resolved to: the copy resolves - // from its own merged options again, so a file-system option among the ones it was given - // speaks rather than being pinned to this table's resolution. + // The managers are handed over, not rebuilt, when the copy resolves the same file system + // this table does: its operators then read the schema cache this table already warmed. The + // file system is the one the caller opened this table through, not the one it resolved to: + // the copy resolves from its own merged options again, so a file-system option among the + // ones it was given speaks rather than being pinned to this table's resolution - and when + // one does change the resolution, fresh managers are built over it instead, so the metadata + // is read through the same backend the data is. The scheme map travels the same way, so the + // copy resolves custom schemes as this table does. + const bool fs_resolution_may_change = + FileSystemResolutionMayChange(*loaded_schema, overridable); return FileStoreTable::Create(store_->specified_file_system(), root_path_, std::shared_ptr(std::move(merged_schema)), identifier_, CatalogUuid().value_or(""), CurrentBranch(), - GetSchemaManager(), GetSnapshotManager()); + fs_resolution_may_change ? nullptr : GetSchemaManager(), + fs_resolution_may_change ? nullptr : GetSnapshotManager(), + store_->fs_scheme_map()); } Result> FileStoreTable::SwitchToBranch(const std::string& branch) const { @@ -526,7 +563,8 @@ Result> FileStoreTable::SwitchToBranch(const std::string& // the same table to the caller. return FileStoreTable::Create(store_->specified_file_system(), root_path_, schema_, identifier_, CatalogUuid().value_or(""), CurrentBranch(), - GetSchemaManager(), GetSnapshotManager()); + GetSchemaManager(), GetSnapshotManager(), + store_->fs_scheme_map()); } // The schema of the branch switched to, read from its own directory, which a branch without // one does not have. @@ -547,7 +585,8 @@ Result> FileStoreTable::SwitchToBranch(const std::string& return FileStoreTable::Create(store_->specified_file_system(), root_path_, std::shared_ptr(std::move(branch_schema)), identifier_, CatalogUuid().value_or(""), target_branch, - schema_manager); + schema_manager, /*snapshot_manager=*/nullptr, + store_->fs_scheme_map()); } Status FileStoreTable::RestoreRealtimeCommittedProgress( @@ -839,8 +878,20 @@ Result> CreateFallbackReadContext( } assert(table_schema); auto options = table_schema->Options(); + // The dynamic options the load merged into the table this read serves - a sequence-number + // flag, a batch size - speak for the read, and the fallback branch serves the same read: + // they are merged over the fallback branch's own schema options the way the load merged + // them over the table's, so the fallback reader recognises what the main one does. `type` + // stays the branch's own: a copy never restyles the table, and neither does a fallback. + for (const auto& [key, value] : context->GetOptions()) { + if (key != Options::TYPE) { + options[key] = value; + } + } if (branch != BranchManager::DEFAULT_MAIN_BRANCH) { options[Options::BRANCH] = branch; + } else { + options.erase(Options::BRANCH); } return InternalReadContext::Create(context, table_schema, options, file_system, &root_path); } diff --git a/src/paimon/core/table/file_store_table.h b/src/paimon/core/table/file_store_table.h index 8df9c4d86..df841d548 100644 --- a/src/paimon/core/table/file_store_table.h +++ b/src/paimon/core/table/file_store_table.h @@ -78,13 +78,17 @@ class FileStoreTable : public Table { /// `Copy` hands its table's over so both share one. /// A null manager is built over `root_path` on `branch`, so the operators always read /// through the managers the table carries. + /// @param fs_scheme_map The scheme-to-identifier map the caller's contexts resolve custom + /// schemes through, which the store's options resolve through as well - see + /// `FileStore::Create`. `Copy` hands its table's map over. static Result> Create( const std::shared_ptr& file_system, const std::string& root_path, const std::shared_ptr& schema, const Identifier& identifier, const std::string& uuid = "", const std::string& branch = BranchManager::DEFAULT_MAIN_BRANCH, const std::shared_ptr& schema_manager = nullptr, - const std::shared_ptr& snapshot_manager = nullptr); + const std::shared_ptr& snapshot_manager = nullptr, + const std::map& fs_scheme_map = {}); /// The branch this table serves, as resolved when the table was built: the schema the table /// was loaded with came from it, and its managers point at it. diff --git a/src/paimon/core/table/system/audit_log_system_table.cpp b/src/paimon/core/table/system/audit_log_system_table.cpp index 7254dff60..e191b653a 100644 --- a/src/paimon/core/table/system/audit_log_system_table.cpp +++ b/src/paimon/core/table/system/audit_log_system_table.cpp @@ -458,17 +458,20 @@ Result> AuditLogSystemTable::CreateDataReadContext( PAIMON_ASSIGN_OR_RAISE(std::shared_ptr base_read_schema, BaseReadSchema()); PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(read_options)); ReadContextBuilder builder(table_path_); + // The data table behind this view is served through the file system the view itself was + // opened with, but only when the context names neither a file system of its own nor a + // scheme map to resolve through - a file system and a scheme map cannot both be named, and + // the context's own answer always wins. + std::shared_ptr data_table_fs = context->GetSpecificFileSystem(); + if (data_table_fs == nullptr && context->GetFileSystemSchemeToIdentifierMap().empty()) { + data_table_fs = GetFileSystem(); + } builder.SetOptions(read_options) .SetReadFieldNames(base_read_schema->field_names()) .WithBranch(core_options.GetBranch()) .WithMemoryPool(context->GetMemoryPool()) .WithExecutor(context->GetExecutor()) - // The data table behind this view is served through the file system the view itself was - // opened with, so a context without one of its own falls back to it rather than the - // default backend. - .WithFileSystem(context->GetSpecificFileSystem() != nullptr - ? context->GetSpecificFileSystem() - : GetFileSystem()) + .WithFileSystem(data_table_fs) .WithFileSystemSchemeToIdentifierMap(context->GetFileSystemSchemeToIdentifierMap()) .EnablePrefetch(context->EnablePrefetch()) .SetPrefetchBatchCount(context->GetPrefetchBatchCount()) diff --git a/src/paimon/core/table/system/read_optimized_system_table.cpp b/src/paimon/core/table/system/read_optimized_system_table.cpp index 0d44b5989..d7193b5cb 100644 --- a/src/paimon/core/table/system/read_optimized_system_table.cpp +++ b/src/paimon/core/table/system/read_optimized_system_table.cpp @@ -108,6 +108,14 @@ Result> ReadOptimizedSystemTable::CreateDataReadCon branch = branch_iter->second; } ReadContextBuilder builder(table_path_); + // The data table behind this view is served through the file system the view itself was + // opened with, but only when the context names neither a file system of its own nor a + // scheme map to resolve through - a file system and a scheme map cannot both be named, and + // the context's own answer always wins. + std::shared_ptr data_table_fs = context->GetSpecificFileSystem(); + if (data_table_fs == nullptr && context->GetFileSystemSchemeToIdentifierMap().empty()) { + data_table_fs = GetFileSystem(); + } builder.SetOptions(options) .WithBranch(branch) .SetPredicate(context->GetPredicate()) @@ -119,12 +127,7 @@ Result> ReadOptimizedSystemTable::CreateDataReadCon .SetRowToBatchThreadNumber(context->GetRowToBatchThreadNumber()) .WithMemoryPool(context->GetMemoryPool()) .WithExecutor(context->GetExecutor()) - // The data table behind this view is served through the file system the view itself was - // opened with, so a context without one of its own falls back to it rather than the - // default backend. - .WithFileSystem(context->GetSpecificFileSystem() != nullptr - ? context->GetSpecificFileSystem() - : GetFileSystem()) + .WithFileSystem(data_table_fs) .WithFileSystemSchemeToIdentifierMap(context->GetFileSystemSchemeToIdentifierMap()) .SetReadAheadCacheEnabled(context->ReadAheadCacheEnabled()) .WithCacheConfig(context->GetCacheConfig()) diff --git a/src/paimon/rest/rest_catalog.cpp b/src/paimon/rest/rest_catalog.cpp index 26a976d94..b570aa307 100644 --- a/src/paimon/rest/rest_catalog.cpp +++ b/src/paimon/rest/rest_catalog.cpp @@ -426,12 +426,14 @@ Result> RestCatalog::LoadSystemTable( if (branch) { dynamic_options[Options::BRANCH] = branch.value(); } - // The file system is only used to build the system table, whose schema does not - // read any file, so the catalog wide one is enough here. The credentials of the - // table are applied when its rows are read, through the file system of the read or - // scan context. - return SystemTableLoader::Load(system_table_name.value(), fs_, data_table_path, latest_schema, - dynamic_options); + // The system table reads the data table's own files - snapshots, manifests, data - so it is + // served by the file system that table's credentials open, which with data tokens is the + // only one that can: `GetTableFileSystem` folds the system identifier to the data table it + // belongs to and hands the catalog-wide file system out when no tokens are issued. + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table_file_system, + GetTableFileSystem(identifier)); + return SystemTableLoader::Load(system_table_name.value(), table_file_system, data_table_path, + latest_schema, dynamic_options); } Result> RestCatalog::GetTable(const Identifier& identifier) const { From f7b63384eb29f7ffb744460f8668c18c9b19cdb0 Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Fri, 23 Oct 2026 14:42:48 +0800 Subject: [PATCH 08/30] fix --- src/paimon/core/table/file_store_table.cpp | 7 +++- src/paimon/core/table/table_test.cpp | 45 ++++++++++++++++++++++ 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/src/paimon/core/table/file_store_table.cpp b/src/paimon/core/table/file_store_table.cpp index cbf2c41d7..a9da52d11 100644 --- a/src/paimon/core/table/file_store_table.cpp +++ b/src/paimon/core/table/file_store_table.cpp @@ -533,9 +533,14 @@ Result> FileStoreTable::Copy( return Status::Invalid(fmt::format("not found latest schema of {}", FullName())); } // `type` is structural and read from the schema alone, the promise `Copy` makes: an option - // given at the call must not restyle the table. + // given at the call must not restyle the table. `branch` is the same kind of promise: the + // copy is the table it was copied from, on the branch that table serves, so a branch option + // given at the call moves neither - it would only claim a branch the copy does not serve - + // and a caller that means to move it has `SwitchToBranch`. Both are dropped before the + // merge, so the copy's options never speak of a branch its operators do not. std::map overridable = dynamic_options; overridable.erase(Options::TYPE); + overridable.erase(Options::BRANCH); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merged_schema, loaded_schema->MergeOptions(overridable)); // The managers are handed over, not rebuilt, when the copy resolves the same file system diff --git a/src/paimon/core/table/table_test.cpp b/src/paimon/core/table/table_test.cpp index 7fde349d9..b4478918b 100644 --- a/src/paimon/core/table/table_test.cpp +++ b/src/paimon/core/table/table_test.cpp @@ -153,6 +153,51 @@ TEST(TableTest, TestTableCarriesManagersOfItsBranch) { EXPECT_EQ(file_store_copy->GetSnapshotManager(), file_store_table->GetSnapshotManager()); } +TEST(TableTest, TestCopyKeepsTheBranchTheTableServes) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
table, + CreateManagedTable(dir->GetFileSystem(), dir->Str(), identifier, /*branch=*/"main", + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + + // A copy handed a `branch` option is the table it was copied from: the branch is the + // table's own - `SwitchToBranch` is the way to change it - so the option moves neither the + // copy nor its options, and an operator asked for the branch the copy serves - `main` - is + // served rather than refused by options claiming a `dev` the copy does not serve. + ASSERT_OK_AND_ASSIGN(std::shared_ptr
copy, table->Copy({{Options::BRANCH, "dev"}})); + auto file_store_copy = std::dynamic_pointer_cast(copy); + ASSERT_TRUE(file_store_copy != nullptr); + EXPECT_EQ(file_store_copy->CurrentBranch(), "main"); + { + ScanContextBuilder scan_builder(dir->Str()); + scan_builder.AddOption(Options::BRANCH, "main"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, scan_builder.Finish()); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr scan, + file_store_copy->NewScan(std::move(ctx))); + } + + // The same the other way round: a copy of a branch table serves that branch still, so + // naming the branch it serves - `dev` - is agreed with rather than refused. + ASSERT_OK_AND_ASSIGN(std::shared_ptr
branch_table, + CreateManagedTable(dir->GetFileSystem(), dir->Str() + "/branch_tbl", + identifier, /*branch=*/"dev", + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
branch_copy, + branch_table->Copy({{Options::BRANCH, "main"}})); + auto branch_file_store_copy = std::dynamic_pointer_cast(branch_copy); + ASSERT_TRUE(branch_file_store_copy != nullptr); + EXPECT_EQ(branch_file_store_copy->CurrentBranch(), "dev"); + { + ScanContextBuilder scan_builder(dir->Str() + "/branch_tbl"); + scan_builder.AddOption(Options::BRANCH, "dev"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, scan_builder.Finish()); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr scan, + branch_file_store_copy->NewScan(std::move(ctx))); + } +} + TEST(TableTest, TestSwitchToBranch) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); From 0de6f757e5c538273e0f4f273de69d2b5a237539 Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Fri, 23 Oct 2026 15:03:31 +0800 Subject: [PATCH 09/30] fix --- src/paimon/core/operation/internal_read_context.cpp | 2 +- src/paimon/core/table/file_store_table.cpp | 2 +- src/paimon/core/table/format/format_table.cpp | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/paimon/core/operation/internal_read_context.cpp b/src/paimon/core/operation/internal_read_context.cpp index 784123828..3eb169ac7 100644 --- a/src/paimon/core/operation/internal_read_context.cpp +++ b/src/paimon/core/operation/internal_read_context.cpp @@ -413,7 +413,7 @@ Result> InternalReadContext::CreateFromCore context, table_schema, read_schema, core_options, arrow_pool, path != nullptr ? std::optional(*path) : std::nullopt)); PAIMON_RETURN_NOT_OK(internal_context->BindIndexScores()); - return internal_context + return internal_context; } Status InternalReadContext::BindIndexScores() { diff --git a/src/paimon/core/table/file_store_table.cpp b/src/paimon/core/table/file_store_table.cpp index a9da52d11..3e355e75a 100644 --- a/src/paimon/core/table/file_store_table.cpp +++ b/src/paimon/core/table/file_store_table.cpp @@ -330,7 +330,7 @@ Result> TableOf(const ReadContext& ctx) { PAIMON_ASSIGN_OR_RAISE(std::optional system_table_path, SystemTableLoader::TryParsePath(ctx.GetPath())); if (system_table_path) { - if (PredicateUtils::ContainsFullTextMatch(context->GetPredicate())) { + if (PredicateUtils::ContainsFullTextMatch(ctx.GetPredicate())) { return Status::NotImplemented( "FullTextMatch predicate execution is not implemented yet"); } diff --git a/src/paimon/core/table/format/format_table.cpp b/src/paimon/core/table/format/format_table.cpp index 6f2ac16a4..cb4141dde 100644 --- a/src/paimon/core/table/format/format_table.cpp +++ b/src/paimon/core/table/format/format_table.cpp @@ -43,6 +43,7 @@ #include "paimon/file_store_commit.h" #include "paimon/file_store_write.h" #include "paimon/fs/file_system.h" +#include "paimon/predicate/predicate_utils.h" #include "paimon/read_context.h" #include "paimon/scan_context.h" #include "paimon/status.h" From ebf6025da7044c5b121e4459b104170e387bbab8 Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Fri, 23 Oct 2026 15:30:38 +0800 Subject: [PATCH 10/30] fix --- .../core/table/source/table_read_test.cpp | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/paimon/core/table/source/table_read_test.cpp b/src/paimon/core/table/source/table_read_test.cpp index b6aef3992..072ba3e1d 100644 --- a/src/paimon/core/table/source/table_read_test.cpp +++ b/src/paimon/core/table/source/table_read_test.cpp @@ -36,6 +36,7 @@ #include "paimon/core/operation/split_read.h" #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/file_store_table.h" #include "paimon/core/table/source/append_only_table_read.h" #include "paimon/core/table/source/data_split_impl.h" #include "paimon/core/table/source/key_value_table_read.h" @@ -134,20 +135,22 @@ TEST(TableReadTest, TestFullTextMatchExecutionIsNotImplemented) { auto fs = std::make_shared(); SchemaManager schema_manager(fs, table_path); ASSERT_OK_AND_ASSIGN(std::shared_ptr schema, schema_manager.ReadSchema(0)); - Table table(fs, table_path, schema, Identifier("append_09.db", "append_09")); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr table, + FileStoreTable::Create(fs, table_path, schema, Identifier("append_09.db", "append_09"))); for (bool table_entry : {false, true}) { ReadContextBuilder builder(table_path); builder.SetReadFieldNames({"f0", "f1"}).SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); auto result = - table_entry ? table.NewRead(std::move(context)) : TableRead::Create(std::move(context)); + table_entry ? table->NewRead(std::move(context)) : TableRead::Create(std::move(context)); ASSERT_NOK_WITH_MSG(result, "FullTextMatch predicate execution is not implemented yet"); ASSERT_TRUE(result.status().IsNotImplemented()); builder.SetReadFieldNames({"f0", "f1", "_f0_INDEX_SCORE"}).SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(context, builder.Finish()); result = - table_entry ? table.NewRead(std::move(context)) : TableRead::Create(std::move(context)); + table_entry ? table->NewRead(std::move(context)) : TableRead::Create(std::move(context)); ASSERT_NOK_WITH_MSG(result, "Score-producing FullTextMatch cannot appear under OR"); } } @@ -159,13 +162,15 @@ TEST(TableReadTest, TestPerFieldScoreExecutionIsNotImplemented) { auto fs = std::make_shared(); SchemaManager schema_manager(fs, path); ASSERT_OK_AND_ASSIGN(std::shared_ptr schema, schema_manager.ReadSchema(0)); - Table table(fs, path, schema, Identifier("append_09.db", "append_09")); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr table, + FileStoreTable::Create(fs, path, schema, Identifier("append_09.db", "append_09"))); for (bool table_entry : {false, true}) { ReadContextBuilder builder(path); builder.SetFullTextSearch(search).SetReadFieldNames({"f1", "_f0_INDEX_SCORE"}); ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); auto result = - table_entry ? table.NewRead(std::move(context)) : TableRead::Create(std::move(context)); + table_entry ? table->NewRead(std::move(context)) : TableRead::Create(std::move(context)); ASSERT_NOK_WITH_MSG(result, "Per-field index score output is not implemented yet"); ASSERT_TRUE(result.status().IsNotImplemented()); } @@ -323,7 +328,10 @@ TEST(TableReadTest, TestDependencyReadWithoutCatalogUsesReadOptions) { expected_options[read_via_option] = explicit_header ? "read-view" : "table-view"; ASSERT_EQ(dependency_context->root_path, "warehouse"); ASSERT_EQ(dependency_context->options, expected_options); - ASSERT_EQ(internal_context->GetTableSchema()->Options(), table_options); + // The load merged the read options into the table it built - a table is the product of + // the load - so the schema the read serves carries the explicit header over the one the + // schema was written with, the same value the dependency context sees. + ASSERT_EQ(internal_context->GetTableSchema()->Options(), expected_options); } } From 281df79a0e098815fbd095bc1abf8dc1f1007ac9 Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Fri, 23 Oct 2026 15:43:57 +0800 Subject: [PATCH 11/30] fix --- src/paimon/core/table/source/table_read_test.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/paimon/core/table/source/table_read_test.cpp b/src/paimon/core/table/source/table_read_test.cpp index 072ba3e1d..3cd3b22de 100644 --- a/src/paimon/core/table/source/table_read_test.cpp +++ b/src/paimon/core/table/source/table_read_test.cpp @@ -142,15 +142,15 @@ TEST(TableReadTest, TestFullTextMatchExecutionIsNotImplemented) { ReadContextBuilder builder(table_path); builder.SetReadFieldNames({"f0", "f1"}).SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); - auto result = - table_entry ? table->NewRead(std::move(context)) : TableRead::Create(std::move(context)); + auto result = table_entry ? table->NewRead(std::move(context)) + : TableRead::Create(std::move(context)); ASSERT_NOK_WITH_MSG(result, "FullTextMatch predicate execution is not implemented yet"); ASSERT_TRUE(result.status().IsNotImplemented()); builder.SetReadFieldNames({"f0", "f1", "_f0_INDEX_SCORE"}).SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(context, builder.Finish()); - result = - table_entry ? table->NewRead(std::move(context)) : TableRead::Create(std::move(context)); + result = table_entry ? table->NewRead(std::move(context)) + : TableRead::Create(std::move(context)); ASSERT_NOK_WITH_MSG(result, "Score-producing FullTextMatch cannot appear under OR"); } } @@ -169,8 +169,8 @@ TEST(TableReadTest, TestPerFieldScoreExecutionIsNotImplemented) { ReadContextBuilder builder(path); builder.SetFullTextSearch(search).SetReadFieldNames({"f1", "_f0_INDEX_SCORE"}); ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); - auto result = - table_entry ? table->NewRead(std::move(context)) : TableRead::Create(std::move(context)); + auto result = table_entry ? table->NewRead(std::move(context)) + : TableRead::Create(std::move(context)); ASSERT_NOK_WITH_MSG(result, "Per-field index score output is not implemented yet"); ASSERT_TRUE(result.status().IsNotImplemented()); } From 08900f3aa5aaa1ef354d296b6d0496d79b0255d5 Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Fri, 23 Oct 2026 17:32:08 +0800 Subject: [PATCH 12/30] fix --- src/paimon/core/table/file_store_table.cpp | 98 ++++++++++++++++++++-- src/paimon/core/table/table_test.cpp | 52 ++++++++++++ 2 files changed, 143 insertions(+), 7 deletions(-) diff --git a/src/paimon/core/table/file_store_table.cpp b/src/paimon/core/table/file_store_table.cpp index 3e355e75a..1a5c15a7d 100644 --- a/src/paimon/core/table/file_store_table.cpp +++ b/src/paimon/core/table/file_store_table.cpp @@ -19,6 +19,7 @@ #include "paimon/core/table/file_store_table.h" +#include #include #include #include @@ -104,6 +105,26 @@ class MemoryPool; namespace { +/// The identifier a table path names when no catalog name goes with it, the C++ reading of +/// the Java `SchemaManagerUtils#identifierFromPath` over `ignoreIfUnknownDatabase = true`: the +/// directory above the table's names the database when it carries the `.db` suffix, the way a +/// catalog lays its databases out, and a path that names no database - a table sitting directly +/// under the root, say - leaves the database unknown, which is all a caller that only knows the +/// directory can say. +Identifier IdentifierFromPath(const std::string& table_path, const std::string& branch) { + const std::vector paths = StringUtils::Split(table_path, "/"); + if (paths.size() < 2) { + return Identifier(Identifier::kUnknownDatabase, + paths.empty() ? std::string() : paths.front()); + } + const std::string& database = paths[paths.size() - 2]; + const std::string::size_type index = database.rfind(Catalog::DB_SUFFIX); + if (index == std::string::npos) { + return Identifier(Identifier::kUnknownDatabase, paths.back(), branch); + } + return Identifier(database.substr(0, index), paths.back(), branch); +} + /// Recognises a format table behind `table_path`, or returns null when it is not one. /// /// The scan, read, write and commit entry points below all take a path and have to answer the @@ -135,9 +156,10 @@ Result> TryLoadFormatTable( fmt::format("a format table has no branches, so {} cannot be read on branch '{}'", table_path, branch)); } - // A context names a path, never a catalog identifier, so the table is known by the directory - // it sits in. - Identifier identifier(Identifier::kUnknownDatabase, PathUtil::GetName(table_path)); + // A context names a path, never a catalog identifier, so the table is known by the + // directory it sits in - which names the database too, when it carries the `.db` suffix a + // catalog lays its databases out with. + Identifier identifier = IdentifierFromPath(table_path, branch); return FormatTable::Create(file_system, table_path, identifier, checked_pointer_cast(table_schema), /*location_carries_paimon_metadata=*/true, options); @@ -163,6 +185,63 @@ bool FileSystemResolutionMayChange(const TableSchema& schema, return false; } +/// The options a copy is refused to change, mirroring the Java `SchemaManagerUtils# +/// checkAlterTableOption` over the immutable options the C++ side has constants for. These name +/// the physical layout and the file set the data on the table's path was written by: a +/// `bucket-key` the options rename, say, re-hashes the bucket pruning while the files stay +/// hashed by the old key, so the scans a copy serves would filter out the buckets its rows are +/// actually in. The check runs over every option the call gives, before the merge, so it holds +/// on every path that merges options into a table - `Copy` itself and the load a context +/// drives alike. The differences from the Java set: `merge-engine`, `bucket`, +/// `row-tracking.enabled` and `data-evolution.enabled` stay servable as dynamic options here - +/// the C++ tests load tables through context options that move them - and `type`/`branch` are +/// dropped by `Copy` before this point. +const char* const kImmutableOptions[] = { + Options::BUCKET_KEY, + Options::BUCKET_FUNCTION_TYPE, + Options::AGGREGATION_REMOVE_RECORD_ON_DELETE, + Options::PARTIAL_UPDATE_REMOVE_RECORD_ON_DELETE, + Options::PARTIAL_UPDATE_REMOVE_RECORD_ON_SEQUENCE_GROUP, + Options::FORCE_LOOKUP, + Options::INDEX_FILE_IN_DATA_FILE_DIR, + Options::BLOB_FIELD, + Options::FALLBACK_BLOB_DESCRIPTOR_FIELD, + Options::BLOB_VIEW_FIELD, + Options::PK_CLUSTERING_OVERRIDE, +}; + +/// Whether the dynamic options of a copy stay within what the data on the table's path was +/// laid out with - the C++ reading of the Java `AbstractFileStoreTable#checkImmutability`: +/// every option that says what the schema already says moves nothing, an immutable one that +/// says anything else is refused, and an `ignore-delete` carries the directional rule of its +/// own the Java check carries too: it may only ever be turned on, never back off. +Status CheckImmutableOptions(const TableSchema& schema, + const std::map& dynamic_options) { + const std::map& schema_options = schema.Options(); + for (const auto& [key, value] : dynamic_options) { + auto schema_option = schema_options.find(key); + if (schema_option != schema_options.end() && schema_option->second == value) { + continue; + } + if (std::find(std::begin(kImmutableOptions), std::end(kImmutableOptions), key) != + std::end(kImmutableOptions)) { + return Status::Invalid(fmt::format("Change '{}' is not supported yet.", key)); + } + if (key == Options::IGNORE_DELETE) { + // An unset option is one that never ignored deletes, the same default the options + // parse to. + PAIMON_ASSIGN_OR_RAISE(bool old_ignore_delete, + OptionsUtils::GetValueFromMap(schema_options, key, false)); + PAIMON_ASSIGN_OR_RAISE(bool new_ignore_delete, + OptionsUtils::GetValueFromMap(dynamic_options, key)); + if (old_ignore_delete && !new_ignore_delete) { + return Status::Invalid("Cannot change ignore-delete from true to false."); + } + } + } + return Status::OK(); +} + /// The options a context carries, merged into the table the load hands back: a table is the /// product of the load, so the options speak there - the way `Copy` merges them, over the ones /// the schema carries - and `NewXxx` on the product serves them without merging again. A @@ -211,10 +290,11 @@ Result> TableOfLoadedSchema( if (format_table != nullptr) { return std::shared_ptr
(std::move(format_table)); } - // A context names a path, never a catalog identifier, so the table is known by the directory - // it sits in - the same convention `TryLoadFormatTable` follows above. - Identifier table_identifier = - identifier.value_or(Identifier(Identifier::kUnknownDatabase, PathUtil::GetName(path))); + // A context names a path, never a catalog identifier, so the table is known by the + // directory it sits in - which names the database too, when it carries the `.db` suffix a + // catalog lays its databases out with. The branch the schema was read on travels the same + // way, folded into the object name. + Identifier table_identifier = identifier.value_or(IdentifierFromPath(path, branch)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, FileStoreTable::Create(explicit_file_system, path, latest_schema, table_identifier, uuid, branch, schema_manager, @@ -541,6 +621,10 @@ Result> FileStoreTable::Copy( std::map overridable = dynamic_options; overridable.erase(Options::TYPE); overridable.erase(Options::BRANCH); + // The physical layout the data on the path was written by is as immovable as the style or + // the branch: the copy serves the same data, so it is refused an option that would lay new + // data out differently than the data it serves was laid out. + PAIMON_RETURN_NOT_OK(CheckImmutableOptions(*loaded_schema, overridable)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merged_schema, loaded_schema->MergeOptions(overridable)); // The managers are handed over, not rebuilt, when the copy resolves the same file system diff --git a/src/paimon/core/table/table_test.cpp b/src/paimon/core/table/table_test.cpp index b4478918b..3ea6cf54f 100644 --- a/src/paimon/core/table/table_test.cpp +++ b/src/paimon/core/table/table_test.cpp @@ -973,6 +973,58 @@ TEST(TableTest, TestCopyServesTheTableWithTheGivenOptionsOnTop) { ASSERT_FALSE(plan->Splits().empty()); } +TEST(TableTest, TestCopyRefusesThePhysicalOptionsTheDataWasLaidOutWith) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + CreateManagedTable(dir->GetFileSystem(), dir->Str(), identifier, + /*branch=*/"main", + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + + // The data on the path was hashed by the `bucket-key` it was written with: a copy that + // renamed the key would prune by a hash its rows never matched, dropping them silently, so + // the change is refused - at `Copy` and at the load a context drives alike, which merges + // through `Copy` - the way the Java `checkImmutability` refuses it. The same goes for the + // other options that name the physical layout, which a key the schema does not carry + // changes as much as one it carries differently. `merge-engine`, `bucket`, + // `row-tracking.enabled` and `data-evolution.enabled` are the options the Java immutable + // set holds that stay servable here. + ASSERT_NOK_WITH_MSG(table->Copy({{Options::BUCKET_KEY, "v"}}), + "Change 'bucket-key' is not supported yet."); + ASSERT_NOK_WITH_MSG(table->Copy({{Options::BUCKET_FUNCTION_TYPE, "hash"}}), + "Change 'bucket-function.type' is not supported yet."); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::shared_ptr
restyled, + table->Copy({{Options::MERGE_ENGINE, "deduplicate"}})); + { + ScanContextBuilder scan_builder(dir->Str()); + scan_builder.AddOption(Options::BUCKET_KEY, "v"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, scan_builder.Finish()); + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(ctx)), + "Change 'bucket-key' is not supported yet."); + } + + // An option that says what the schema already says moves nothing, and a bucket count is + // the physical option a rescale legitimately rewrites, so a copy may move it. An + // `ignore-delete` may only be turned on, never back off. + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::shared_ptr
unchanged, + table->Copy({{Options::BUCKET_KEY, "id"}, {Options::BUCKET, "1"}})); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::shared_ptr
rescaled, + table->Copy({{Options::BUCKET, "2"}})); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::shared_ptr
ignoring_deletes, + table->Copy({{Options::IGNORE_DELETE, "true"}})); + { + ASSERT_OK_AND_ASSIGN(std::shared_ptr
ignore_delete_table, + CreateManagedTable(dir->GetFileSystem(), dir->Str() + "/ignore_delete", + Identifier("db", "igd"), /*branch=*/"main", + {{Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "id"}, + {Options::IGNORE_DELETE, "true"}})); + ASSERT_NOK_WITH_MSG(ignore_delete_table->Copy({{Options::IGNORE_DELETE, "false"}}), + "Cannot change ignore-delete from true to false."); + } +} + TEST(TableTest, TestCreateBuildsTheOperator) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); From f44cc4b51e29370f5a8d4cd52ea71b3b9a365a6a Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Fri, 23 Oct 2026 17:46:07 +0800 Subject: [PATCH 13/30] update --- src/paimon/core/table/table_test.cpp | 34 ++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/src/paimon/core/table/table_test.cpp b/src/paimon/core/table/table_test.cpp index 3ea6cf54f..b6cc22859 100644 --- a/src/paimon/core/table/table_test.cpp +++ b/src/paimon/core/table/table_test.cpp @@ -1025,6 +1025,40 @@ TEST(TableTest, TestCopyRefusesThePhysicalOptionsTheDataWasLaidOutWith) { } } +TEST(TableTest, TestALoadNamesTheDatabaseThePathLaysOut) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::shared_ptr fs = dir->GetFileSystem(); + + // A context that names no catalog knows the table by its directory: the directory above + // the table's names the database when it carries the `.db` suffix, the way the Java + // `identifierFromPath` reads it, so the table the load serves is named for both - which + // the load of a directory without a schema says in the error it fails with. The option + // the context carries is what drives the load to that error: it hands the table the merge + // would build, which is where the name is spoken. + const std::string table_dir = dir->Str() + "/my_db.db/my_table"; + ASSERT_OK(fs->Mkdirs(table_dir)); + { + ScanContextBuilder builder(table_dir); + builder.AddOption("read.batch-size", "2"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, builder.Finish()); + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(ctx)), + "not found latest schema of my_db.my_table"); + } + + // A table sitting directly under the root, with no `.db` directory above it, names no + // database: the table is known by its directory alone, the way it was before. + const std::string loose_table_dir = dir->Str() + "/loose_table"; + ASSERT_OK(fs->Mkdirs(loose_table_dir)); + { + ScanContextBuilder builder(loose_table_dir); + builder.AddOption("read.batch-size", "2"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, builder.Finish()); + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(ctx)), + "not found latest schema of loose_table"); + } +} + TEST(TableTest, TestCreateBuildsTheOperator) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); From 559e8c1f676d06770b164d9d44b54ed1652d845f Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Mon, 26 Oct 2026 18:31:39 +0800 Subject: [PATCH 14/30] fix --- include/paimon/catalog/catalog.h | 14 +- include/paimon/catalog/table.h | 48 +- include/paimon/commit_context.h | 10 +- include/paimon/read_context.h | 10 +- include/paimon/scan_context.h | 13 +- include/paimon/table/format/format_table.h | 46 +- include/paimon/write_context.h | 10 +- src/paimon/CMakeLists.txt | 3 + src/paimon/core/catalog/catalog_utils.cpp | 27 + src/paimon/core/catalog/catalog_utils.h | 17 +- .../core/catalog/file_system_catalog.cpp | 15 +- src/paimon/core/catalog/snapshot_commit.h | 25 + ...eted_primary_key_index_maintainer_test.cpp | 26 +- .../append_only_file_store_write_test.cpp | 22 +- .../core/operation/internal_read_context.cpp | 48 + .../core/operation/internal_read_context.h | 31 +- .../core/operation/orphan_files_cleaner.cpp | 11 +- src/paimon/core/schema/schema_validation.cpp | 49 ++ src/paimon/core/schema/schema_validation.h | 9 + .../table/append_only_file_store_table.cpp | 16 +- .../core/table/append_only_file_store_table.h | 6 - src/paimon/core/table/file_store.cpp | 49 ++ src/paimon/core/table/file_store.h | 154 ++-- src/paimon/core/table/file_store_table.cpp | 818 +++--------------- src/paimon/core/table/file_store_table.h | 123 ++- src/paimon/core/table/format/format_table.cpp | 41 +- .../table/primary_key_file_store_table.cpp | 17 +- .../core/table/primary_key_file_store_table.h | 6 - .../core/table/source/realtime_table_scan.cpp | 47 + .../core/table/source/realtime_table_scan.h | 9 + .../table/system/audit_log_system_table.cpp | 14 +- .../core/table/system/binlog_system_table.cpp | 3 + .../core/table/system/global_system_tables.h | 9 +- .../table/system/in_memory_system_table.cpp | 5 + .../system/read_optimized_system_table.cpp | 14 +- src/paimon/core/table/system/system_table.cpp | 9 + src/paimon/core/table/system/system_table.h | 28 +- .../core/table/system/system_table_test.cpp | 59 ++ src/paimon/core/table/table.cpp | 33 + src/paimon/core/table/table_test.cpp | 315 ++++++- src/paimon/rest/rest_catalog.cpp | 31 +- src/paimon/rest/rest_catalog.h | 4 +- src/paimon/rest/rest_catalog_test.cpp | 41 + test/inte/realtime_write_inte_test.cpp | 57 +- test/inte/scan_and_read_inte_test.cpp | 79 +- test/inte/scan_inte_test.cpp | 27 +- .../write_and_read_inte_with_index_test.cpp | 3 +- 47 files changed, 1143 insertions(+), 1308 deletions(-) create mode 100644 src/paimon/core/table/table.cpp diff --git a/include/paimon/catalog/catalog.h b/include/paimon/catalog/catalog.h index 0bb578383..033e7a822 100644 --- a/include/paimon/catalog/catalog.h +++ b/include/paimon/catalog/catalog.h @@ -147,12 +147,14 @@ class PAIMON_EXPORT Catalog { /// Gets a table. /// - /// The table's `type` option decides what is handed back: a `format-table` comes back as - /// the `FormatTable` it is - a directory of data files, with no snapshot and no manifest - - /// and every other implemented type as the managed table it is. A `T$system` identifier - /// names a read-only view instead, which comes back as the `SystemTable` it is. All go - /// through this one `Table` interface: `NewScan()` and `NewRead()` serve any of them, - /// `NewWrite()` and `NewCommit()` the writable ones. + /// The table's `type` option decides what is handed back: a `format-table` comes back as the + /// `FormatTable` it is - a directory of data files, with no snapshot and no manifest - and + /// every other implemented type as the managed table it is, while a `T$system` identifier names + /// a read-only `SystemTable`. All go through this one `Table` interface: `NewScan()` and + /// `NewRead()` serve any of them, `NewWrite()` and `NewCommit()` the writable ones. + /// + /// A global system table of the "sys" database reads its rows lazily through the catalog it + /// came from, so a catalog that serves one must be owned by a `std::shared_ptr`. /// /// @param identifier Identifier of the table to get. /// @return A result containing the table, or an error status. diff --git a/include/paimon/catalog/table.h b/include/paimon/catalog/table.h index a31d3100d..3364f6eff 100644 --- a/include/paimon/catalog/table.h +++ b/include/paimon/catalog/table.h @@ -44,36 +44,33 @@ class WriteContext; /// A table provides basic abstraction for table type. /// -/// The state a table of every kind carries - file system, root path, schema and identifier - lives -/// here, and each kind of table fills the operations in: the operators below create this table's -/// scan, read, write or commit, from a context the caller finishes through its builder. What the -/// context asks for that this table cannot honour is refused by name by the implementation rather +/// The state every kind of table carries - file system, root path, schema and identifier - lives +/// here, and each kind fills the operations in. The `NewXxx` operators below are the table-first +/// entry points: `TableScan::Create()` and friends load the table a context names and hand the +/// context to them. What a context asks for that the table cannot honour is refused by name rather /// than silently dropped. class PAIMON_EXPORT Table { public: virtual ~Table() = default; - /// Returns the file system the table was opened with, which serves the table's - /// files. A catalog that hands out per-table file systems opens a table with the - /// one for that table; it is null when the opener had none. The `NewXxx` - /// operations read the table through it. + /// Returns the file system the table was opened with, null when the opener had none. A + /// catalog that hands out per-table file systems opens a table with the one for that table. std::shared_ptr GetFileSystem() const { return file_system_; } - /// Returns the root path of the table in its file system. It is empty when the - /// opener had no location for the table, which a mock in a test is one case of. - /// The `NewXxx` operations serve the table from it. + /// Returns the root path of the table in its file system. It is empty when the opener had no + /// location for the table, which a mock in a test is one case of. const std::string& GetRootPath() const { return root_path_; } - /// A name to identify this table, which each kind of table answers for itself: a managed or - /// format table is named by its identifier, a system table by its own kind. + /// A name to identify this table: a managed or format table answers its identifier's object + /// name, a system table its own kind. virtual std::string Name() const = 0; - /// Full name of the table, which each kind of table answers for itself. A managed or format - /// table answers database.tableName, a system table the same name as `Name()`. + /// Full name of the table: `database.tableName` for a managed or format table, the same name + /// as `Name()` for a system table. virtual std::string FullName() const = 0; /// Returns the metastore UUID, falling back to `FullName()`. @@ -93,41 +90,34 @@ class PAIMON_EXPORT Table { return schema_; } - /// Copies this table with `dynamic_options` on top of the options it carries, which is - /// the precedence every context builder promises for a table it was handed rather than loaded - /// itself: an option given at the call wins over the one the schema stored. - /// - /// `type` is not overridable: it is structural and is read from the schema alone. + /// Copies this table with `dynamic_options` on top of the options it carries: an option given + /// at the call wins over the one the schema stored, the precedence every context builder + /// promises. `type` is not overridable - it is structural and read from the schema alone. /// /// @param dynamic_options Options given at the call. /// @return A result containing the copied table, or the error copying it reports. virtual Result> Copy( const std::map& dynamic_options) const = 0; - /// Creates a scan of this table from a finished scan context, the table-first entry point - /// that `TableScan::Create()` loads the table the context names and hands the context to. + /// Creates a scan of this table from a finished scan context. /// @param context The finished scan context. /// @return The scan, or the error building it reports. virtual Result> NewScan( std::unique_ptr context) const = 0; - /// Creates a read of this table from a finished read context, the table-first entry point - /// that `TableRead::Create()` loads the table the context names and hands the context to. + /// Creates a read of this table from a finished read context. /// @param context The finished read context. /// @return The read, or the error building it reports. virtual Result> NewRead( std::unique_ptr context) const = 0; - /// Creates a write of this table from a finished write context, the table-first entry point - /// that `FileStoreWrite::Create()` loads the table the context names and hands the context to. + /// Creates a write of this table from a finished write context. /// @param context The finished write context. /// @return The write, or the error building it reports. virtual Result> NewWrite( std::unique_ptr context) const = 0; - /// Creates a commit of this table from a finished commit context, the table-first entry point - /// that `FileStoreCommit::Create()` loads the table the context names and hands the context - /// to. + /// Creates a commit of this table from a finished commit context. /// @param context The finished commit context. /// @return The commit, or the error building it reports. virtual Result> NewCommit( diff --git a/include/paimon/commit_context.h b/include/paimon/commit_context.h index a70cacb7a..556db7bd2 100644 --- a/include/paimon/commit_context.h +++ b/include/paimon/commit_context.h @@ -156,12 +156,10 @@ class PAIMON_EXPORT CommitContextBuilder { ~CommitContextBuilder(); - /// Set a configuration options map. The options take part in loading the table a commit - /// serves: the load merges them over the ones the table's schema carries and the table it - /// hands back serves the merged ones. A caller holding a table already loaded - a load - /// that is over - changes it through `Table::Copy()`; an option contradicting a table held - /// that way is refused when a commit is asked of it. A format table takes the options as - /// its own. + /// Set a configuration options map to set some option entries which are not defined in the + /// table schema or whose values you want to overwrite. A load merges them into the table it + /// hands back; an option contradicting a table you loaded already is refused, so change that + /// one through `Table::Copy()`. /// @note The options map will clear the options added by `AddOption()` before. /// @param options The configuration options map. /// @return Reference to this builder for method chaining. diff --git a/include/paimon/read_context.h b/include/paimon/read_context.h index 8cf565996..d01e28df5 100644 --- a/include/paimon/read_context.h +++ b/include/paimon/read_context.h @@ -324,12 +324,10 @@ class PAIMON_EXPORT ReadContextBuilder { /// When set, read_field_ids and read_field_names are ignored. ReadContextBuilder& SetReadSchema(std::unique_ptr read_schema); - /// Set a configuration options map. The options take part in loading the table a read - /// serves: the load merges them over the ones the table's schema carries and the table it - /// hands back serves the merged ones. A caller holding a table already loaded - a load - /// that is over - changes it through `Table::Copy()`; an option contradicting a table held - /// that way is refused when a read is asked of it. A format table takes the options as its - /// own. + /// Set a configuration options map to set some option entries which are not defined in the + /// table schema or whose values you want to overwrite. A load merges them into the table it + /// hands back; an option contradicting a table you loaded already is refused, so change that + /// one through `Table::Copy()`. /// @note The options map will clear the options added by `AddOption()` before. /// @param options The configuration options map. /// @return Reference to this builder for method chaining. diff --git a/include/paimon/scan_context.h b/include/paimon/scan_context.h index fe57afb58..2e3f8d9a8 100644 --- a/include/paimon/scan_context.h +++ b/include/paimon/scan_context.h @@ -208,15 +208,12 @@ class PAIMON_EXPORT ScanContextBuilder { ScanContextBuilder& WithRealtimeContext( const std::shared_ptr& realtime_context); - /// Add a single configuration option. See `SetOptions()` for how the options a context - /// carries are served. + /// Add a single configuration option, served the way `SetOptions()` describes. ScanContextBuilder& AddOption(const std::string& key, const std::string& value); - /// Set a configuration options map. The options take part in loading the table a scan - /// serves: the load merges them over the ones the table's schema carries and the table it - /// hands back serves the merged ones. A caller holding a table already loaded - a load - /// that is over - changes it through `Table::Copy()`; an option contradicting a table held - /// that way is refused when a scan is asked of it. A format table takes the options as its - /// own. + /// Set a configuration options map to set some option entries which are not defined in the + /// table schema or whose values you want to overwrite. A load merges them into the table it + /// hands back; an option contradicting a table you loaded already is refused, so change that + /// one through `Table::Copy()`. /// @note The options map will clear the options added by `AddOption()` before. /// @param options The configuration options map. /// @return Reference to this builder for method chaining. diff --git a/include/paimon/table/format/format_table.h b/include/paimon/table/format/format_table.h index 88444d360..c5bfe60d3 100644 --- a/include/paimon/table/format/format_table.h +++ b/include/paimon/table/format/format_table.h @@ -48,12 +48,11 @@ class FileSystem; /// Writes only insert - there is nowhere to record an update or a delete - so a read fills /// `_VALUE_KIND` with inserts throughout. /// -/// A format table is a `Table`, so the operators it inherits create its scan, read, write and -/// commit from this table, with the context's options applied on top of the ones it carries. The -/// entry points every other table uses remain open too - `TableScan::Create()` and friends - each -/// of which recognises a format table from the schema under the table path, or takes one already -/// loaded through the `FormatTable` constructor every context builder has, which is how a table -/// whose schema lives in a metastore is reached. +/// A format table is a `Table`, so it is reached both ways: through the `NewXxx` operators below, +/// and through the entry points every other table uses - `TableScan::Create()` and friends - which +/// recognise one from the schema under the table path, or take one already loaded through the +/// `FormatTable` constructor every context builder has, how a table whose schema lives in a +/// metastore is reached. /// /// Every instance lives in a `shared_ptr`: a copy that adds nothing hands the same one back. /// @@ -117,8 +116,7 @@ class PAIMON_EXPORT FormatTable : public Table, public std::enable_shared_from_t return identifier_.GetFullName(); } - /// Copies the table with `dynamic_options` on top of the options it carries, the precedence - /// every context builder promises. The same table comes back when there is nothing to add. + /// The same table comes back when there is nothing to add. Result> Copy( const std::map& dynamic_options) const override; @@ -160,9 +158,8 @@ class PAIMON_EXPORT FormatTable : public Table, public std::enable_shared_from_t return options_; } - /// Schema of the table, including its partition fields. It hides `Table::LatestSchema()`, - /// which returns the same schema as the base every table sees; this one hands it back as the - /// `DataSchema` a format table is always created with. + /// Schema of the table, including its partition fields. Hides `Table::LatestSchema()` to hand + /// back the same schema as the `DataSchema` a format table is always created with. std::shared_ptr LatestSchema() const { return std::static_pointer_cast(schema_); } @@ -179,25 +176,16 @@ class PAIMON_EXPORT FormatTable : public Table, public std::enable_shared_from_t return location_carries_paimon_metadata_; } - /// Creates a scan of this table from a finished scan context, refusing by name what a format - /// table cannot honour. The context's options are applied on top of the ones this table - /// carries, the precedence every context builder promises. + /// The four operators below create this table's scan, read, write and commit, applying the + /// context's options on top of the ones this table carries and refusing by name what a + /// format table cannot honour. Result> NewScan(std::unique_ptr context) const override; - /// Creates a read of this table from a finished read context, refusing by name what a format - /// table cannot honour. The context's options are applied on top of the ones this table - /// carries, the precedence every context builder promises. Result> NewRead(std::unique_ptr context) const override; - /// Creates a write of this table from a finished write context, refusing by name what a format - /// table cannot honour. The context's options are applied on top of the ones this table - /// carries, the precedence every context builder promises. Result> NewWrite( std::unique_ptr context) const override; - /// Creates a commit of this table from a finished commit context, refusing by name what a - /// format table cannot honour. The context's options are applied on top of the ones this - /// table carries, the precedence every context builder promises. Result> NewCommit( std::unique_ptr context) const override; @@ -208,18 +196,14 @@ class PAIMON_EXPORT FormatTable : public Table, public std::enable_shared_from_t const std::string& file_compression, const std::string& partition_default_name, bool partition_only_value_in_path, bool location_carries_paimon_metadata); - /// This table as a `shared_ptr`, which a copy that adds nothing hands back. - /// - /// `shared_from_this()` on a const method hands back a `shared_ptr`, while - /// the copy below needs a mutable one; the cast is safe because the table is never written to. + /// This table as a `shared_ptr`. `shared_from_this()` on a const method hands back a + /// `shared_ptr`, while the copy below needs a mutable one; the cast is + /// safe because the table is never written to. std::shared_ptr SharedSelf() const { return std::const_pointer_cast(shared_from_this()); } - /// Copies this table with `dynamic_options` on top of the options it carries, as a format - /// table rather than the `Table` `Copy()` hands back, which is what its operators need. The - /// same table comes back when there is nothing to add; `type` is not overridable, and - /// `Create()` keeps the schema's own. + /// `Copy()` as a `FormatTable` rather than a `Table`, which is what its operators need. Result> CopySelf( const std::map& dynamic_options) const; diff --git a/include/paimon/write_context.h b/include/paimon/write_context.h index 23195793f..f1d3ac06e 100644 --- a/include/paimon/write_context.h +++ b/include/paimon/write_context.h @@ -183,12 +183,10 @@ class PAIMON_EXPORT WriteContextBuilder { ~WriteContextBuilder(); - /// Set a configuration options map. The options take part in loading the table a write - /// serves: the load merges them over the ones the table's schema carries and the table it - /// hands back serves the merged ones. A caller holding a table already loaded - a load - /// that is over - changes it through `Table::Copy()`; an option contradicting a table held - /// that way is refused when a write is asked of it. A format table takes the options as its - /// own. + /// Set a configuration options map to set some option entries which are not defined in the + /// table schema or whose values you want to overwrite. A load merges them into the table it + /// hands back; an option contradicting a table you loaded already is refused, so change that + /// one through `Table::Copy()`. /// @note The options map will clear the options added by `AddOption()` before. /// @param options The configuration options map. /// @return Reference to this builder for method chaining. diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index 86ca0f444..d2da868ea 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -267,6 +267,7 @@ set(PAIMON_CORE_SRCS core/catalog/catalog_utils.cpp core/catalog/file_system_catalog.cpp core/catalog/identifier.cpp + core/catalog/snapshot_commit.cpp core/core_options.cpp core/deletionvectors/deletion_vector.cpp core/deletionvectors/deletion_file_writer.cpp @@ -447,6 +448,7 @@ set(PAIMON_CORE_SRCS core/table/bucket_mode.cpp core/table/file_store.cpp core/table/file_store_table.cpp + core/table/file_store_table_factory.cpp core/table/format/format_file_listing.cpp core/table/format/format_file_naming.cpp core/table/format/format_path_validation.cpp @@ -492,6 +494,7 @@ set(PAIMON_CORE_SRCS core/table/system/system_table.cpp core/table/system/system_table_scan.cpp core/table/system/system_table_schema.cpp + core/table/table.cpp core/tag/tag.cpp core/utils/branch_manager.cpp core/utils/blob_view_lookup.cpp diff --git a/src/paimon/core/catalog/catalog_utils.cpp b/src/paimon/core/catalog/catalog_utils.cpp index 721942a88..4cf128a94 100644 --- a/src/paimon/core/catalog/catalog_utils.cpp +++ b/src/paimon/core/catalog/catalog_utils.cpp @@ -22,10 +22,12 @@ #include #include #include +#include #include "fmt/format.h" #include "paimon/catalog/catalog.h" #include "paimon/common/utils/path_util.h" +#include "paimon/common/utils/string_utils.h" #include "paimon/core/options/table_type.h" #include "paimon/core/utils/branch_manager.h" #include "paimon/defs.h" @@ -43,6 +45,21 @@ Status SystemTableError(const Identifier& identifier, const std::string& action) } // namespace +Identifier CatalogUtils::IdentifierFromPath(const std::string& table_path, + const std::string& branch) { + const std::vector paths = StringUtils::Split(table_path, "/"); + if (paths.size() < 2) { + return Identifier(Identifier::kUnknownDatabase, + paths.empty() ? std::string() : paths.front()); + } + const std::string& database = paths[paths.size() - 2]; + const std::string::size_type index = database.rfind(Catalog::DB_SUFFIX); + if (index == std::string::npos) { + return Identifier(Identifier::kUnknownDatabase, paths.back(), branch); + } + return Identifier(database.substr(0, index), paths.back(), branch); +} + bool CatalogUtils::IsSystemDatabase(const std::string& db_name) { return db_name == Catalog::SYSTEM_DATABASE_NAME; } @@ -74,6 +91,16 @@ Result CatalogUtils::BranchIdentifier(const Identifier& identifier, return Identifier(identifier.GetDatabaseName(), data_table_name, branch); } +Result> CatalogUtils::BranchIdentifier( + const std::optional& identifier, const std::string& branch) { + if (!identifier) { + return std::optional(); + } + PAIMON_ASSIGN_OR_RAISE(Identifier branch_identifier, + BranchIdentifier(identifier.value(), branch)); + return std::optional(std::move(branch_identifier)); +} + Status CatalogUtils::CheckNotBranch(const Identifier& identifier, const std::string& action) { PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); if (branch) { diff --git a/src/paimon/core/catalog/catalog_utils.h b/src/paimon/core/catalog/catalog_utils.h index 23cb77203..58775d4df 100644 --- a/src/paimon/core/catalog/catalog_utils.h +++ b/src/paimon/core/catalog/catalog_utils.h @@ -19,6 +19,7 @@ #pragma once #include +#include #include #include "paimon/catalog/identifier.h" @@ -31,13 +32,19 @@ class Catalog; class FormatTable; class Schema; -/// Checks shared by the catalog implementations. Every check takes an `action` naming -/// the rejected operation in the error message, e.g. "dropTable". +/// The identifier tools and the checks shared by the catalog implementations. Every check takes an +/// `action` naming the rejected operation in the error message, e.g. "dropTable". class CatalogUtils { public: CatalogUtils() = delete; ~CatalogUtils() = delete; + /// The identifier a table path names when no catalog name goes with it, the C++ reading of the + /// Java `SchemaManagerUtils#identifierFromPath` over `ignoreIfUnknownDatabase = true`: the + /// directory above the table's names the database when it carries the `.db` suffix a catalog + /// lays its databases out with, and a path that names no database leaves it unknown. + static Identifier IdentifierFromPath(const std::string& table_path, const std::string& branch); + /// Returns whether `db_name` is the reserved system database "sys". static bool IsSystemDatabase(const std::string& db_name); @@ -58,6 +65,12 @@ class CatalogUtils { static Result BranchIdentifier(const Identifier& identifier, const std::string& branch); + /// The identifier a catalog knows the table by, on `branch`, when there is one to fold a + /// branch into: a caller without an identifier has none to ask the catalog about, and gets + /// none back rather than an error. + static Result> BranchIdentifier( + const std::optional& identifier, const std::string& branch); + /// Fails when `db_name` cannot be used as a single path component, which is required to /// keep the database path under the warehouse. static Status CheckValidDatabaseName(const std::string& db_name); diff --git a/src/paimon/core/catalog/file_system_catalog.cpp b/src/paimon/core/catalog/file_system_catalog.cpp index d3c6bd92e..fdcddf20e 100644 --- a/src/paimon/core/catalog/file_system_catalog.cpp +++ b/src/paimon/core/catalog/file_system_catalog.cpp @@ -287,6 +287,9 @@ Result> FileSystemCatalog::LoadTableSchema( if (!supported) { return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); } + // The table built here is transient - its schema is read within this call and nothing + // outlives it - so its context keeps no shared ownership of the catalog. A catalog that + // hands a global system table back out (`RestCatalog::GetTable`) retains it instead. GlobalSystemTableContext context; context.catalog = this; context.fs = fs_; @@ -316,8 +319,7 @@ Result> FileSystemCatalog::LoadTableSchema( Result> FileSystemCatalog::GetTable(const Identifier& identifier) const { PAIMON_ASSIGN_OR_RAISE(bool is_system_table, identifier.IsSystemTable()); if (is_system_table) { - // A system table reads the files of the data table it belongs to, through the file - // system of that table's catalog, and the schema it exposes is its own read-only one. + // A system table reads the data table's files through this catalog's file system. return LoadSystemTable(identifier); } // The schema is read on the table's own branch: `root/branch/branch-` below the @@ -330,8 +332,7 @@ Result> FileSystemCatalog::GetTable(const Identifier& ide } PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(identifier)); // The schema's `type` says what the table is. A format table is the same directory read as - // plain data, so the schema read above is all it takes to serve one: no manifest is looked - // for, and the checks a format table adds run in `FormatTable::Create`. + // plain data, so the schema read above is all it takes to serve one; no manifest is read. std::shared_ptr data_schema = checked_pointer_cast(*latest_schema); PAIMON_ASSIGN_OR_RAISE(bool is_format_table, TableTypeDefine::IsFormatTable(data_schema->Options())); @@ -347,8 +348,7 @@ Result> FileSystemCatalog::GetTable(const Identifier& ide // table rather than at the first read. PAIMON_RETURN_NOT_OK(CatalogUtils::CheckManagedTableType(identifier, *latest_schema, "FileSystemCatalog::GetTable")); - // The branch the schema above was read on, which the table carries so its managers read - // the same branch: `GetBranchNameOrDefault` folds any spelling of `main`. + // The branch the schema above was read on, so the table's managers read the same one. PAIMON_ASSIGN_OR_RAISE(std::string branch, identifier.GetBranchNameOrDefault()); return FileStoreTable::Create(fs_, table_path, checked_pointer_cast(*latest_schema), identifier, /*uuid=*/"", branch); @@ -374,8 +374,7 @@ Result> FileSystemCatalog::LoadSystemTable( if (branch) { dynamic_options[Options::BRANCH] = branch.value(); } - // A system table reads the files of the data table it belongs to, so its root path is - // that table's. + // A system table reads the files of the data table it belongs to, so its root path is that one. PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(data_identifier)); return SystemTableLoader::Load(system_table_name.value(), fs_, table_path, latest_schema.value(), dynamic_options); diff --git a/src/paimon/core/catalog/snapshot_commit.h b/src/paimon/core/catalog/snapshot_commit.h index 41bd1712a..58b8bbe9d 100644 --- a/src/paimon/core/catalog/snapshot_commit.h +++ b/src/paimon/core/catalog/snapshot_commit.h @@ -23,12 +23,17 @@ #include #include +#include "paimon/catalog/identifier.h" #include "paimon/core/catalog/commit_lock.h" #include "paimon/core/partition/partition_statistics.h" #include "paimon/core/snapshot.h" namespace paimon { +class Catalog; +class FileSystem; +class SnapshotManager; + /// Interface to commit snapshot atomically. class SnapshotCommit { public: @@ -41,6 +46,26 @@ class SnapshotCommit { virtual Result> AcquireLock() { return std::unique_ptr(); } + /// The commit a table publishes its snapshots through, the C++ reading of the Java + /// `CatalogEnvironment#snapshotCommit`: a catalog that version-manages the table commits + /// through it, a caller that only wants the request built gets one with no catalog attached, + /// and everything else publishes by renaming the snapshot file into place. + /// + /// @param catalog The catalog the commit was handed, or null for one that has none. + /// @param catalog_identifier The identifier the catalog knows the table by, on the branch the + /// commit commits to. A version-managing catalog needs one and is refused without it. + /// @param table_id The catalog's UUID for the table, captured before the changes were + /// prepared; null when there is none. + /// @param use_rest_catalog_commit Whether the caller asks for the request rather than for the + /// catalog to send it. + /// @param file_system The file system the snapshot file is renamed through. + /// @param snapshot_manager The manager that names the snapshot file renamed. + static Result> Create( + const std::shared_ptr& catalog, + const std::optional& catalog_identifier, + const std::optional& table_id, bool use_rest_catalog_commit, + const std::shared_ptr& file_system, + const std::shared_ptr& snapshot_manager); /// @param base_snapshot_uuid Base snapshot UUID; null for an absent or legacy snapshot. /// @param snapshot Snapshot to be committed. diff --git a/src/paimon/core/index/pk/bucketed_primary_key_index_maintainer_test.cpp b/src/paimon/core/index/pk/bucketed_primary_key_index_maintainer_test.cpp index 769bbf326..81803ed90 100644 --- a/src/paimon/core/index/pk/bucketed_primary_key_index_maintainer_test.cpp +++ b/src/paimon/core/index/pk/bucketed_primary_key_index_maintainer_test.cpp @@ -33,7 +33,6 @@ #include "gtest/gtest.h" #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" -#include "paimon/catalog/table.h" #include "paimon/commit_context.h" #include "paimon/common/utils/path_util.h" #include "paimon/core/global_index/indexed_split_impl.h" @@ -474,21 +473,15 @@ TEST_P(BucketedPrimaryKeyIndexMaintainerTest, BuildsRestoresAndReplacesCompacted ASSERT_EQ(0, data_split_count); ReadContextBuilder read_builder(table_path_); + // The source build above keeps lazy decoding enabled. Disable it only for the final data-row + // assertion, whose predicate reader is outside the maintenance path covered by this test. read_builder.SetReadFieldNames({"id", "value"}) .SetPredicate(value_predicate) + .AddOption("orc.read.enable-lazy-decoding", "false") .EnablePredicateFilter(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); - // The source build above keeps lazy decoding enabled. Disable it only for the final data-row - // assertion, whose predicate reader is outside the maintenance path covered by this test. - // The table is copied with it off because a read through the loaded table refuses an option - // that contradicts the one it carries. - ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, Catalog::Create(directory_->Str(), {})); - ASSERT_OK_AND_ASSIGN(std::shared_ptr
loaded_table, - catalog->GetTable(Identifier("db", "table"))); - ASSERT_OK_AND_ASSIGN(std::shared_ptr
copied_table, - loaded_table->Copy({{"orc.read.enable-lazy-decoding", "false"}})); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - copied_table->NewRead(std::move(read_context))); + TableRead::Create(std::move(read_context))); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, @@ -714,18 +707,11 @@ TEST_P(BucketedPrimaryKeyIndexMaintainerTest, ContinuesCompactionWhenIndexBuildF ReadContextBuilder read_builder(table_path_); read_builder.SetReadFieldNames({"id", "value"}) .SetPredicate(value_predicate) + .AddOption("orc.read.enable-lazy-decoding", "false") .EnablePredicateFilter(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); - // Like the test above, the assertion reads with lazy decoding disabled, so it reads through - // a copy of the table carrying it off: a read through the loaded table refuses an option that - // contradicts the one it carries. - ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, Catalog::Create(directory_->Str(), {})); - ASSERT_OK_AND_ASSIGN(std::shared_ptr
loaded_table, - catalog->GetTable(Identifier("db", "table"))); - ASSERT_OK_AND_ASSIGN(std::shared_ptr
copied_table, - loaded_table->Copy({{"orc.read.enable-lazy-decoding", "false"}})); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - copied_table->NewRead(std::move(read_context))); + TableRead::Create(std::move(read_context))); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, diff --git a/src/paimon/core/operation/append_only_file_store_write_test.cpp b/src/paimon/core/operation/append_only_file_store_write_test.cpp index 3ce48ee2c..bfeaefced 100644 --- a/src/paimon/core/operation/append_only_file_store_write_test.cpp +++ b/src/paimon/core/operation/append_only_file_store_write_test.cpp @@ -36,7 +36,6 @@ #include "gtest/gtest.h" #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" -#include "paimon/catalog/table.h" #include "paimon/commit_context.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/data/binary_row_writer.h" @@ -514,25 +513,10 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestSharedShreddingNewWriterIgnoresExisting /*commit_identifier=*/0); Commit(table_path, avro_options, avro_commit_msgs); - // The second batch is written through a copy of the table carrying the shredding options: - // an option on the context would contradict the table the catalog loaded, which a write - // through that table refuses. - ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(dir->Str(), avro_options)); - ASSERT_OK_AND_ASSIGN(std::shared_ptr
avro_table, - catalog->GetTable(Identifier("foo", "bar"))); - ASSERT_OK_AND_ASSIGN(std::shared_ptr
shredded_table, - avro_table->Copy(shredding_options)); - WriteContextBuilder shredded_builder(table_path, commit_user_); - ASSERT_OK_AND_ASSIGN(std::unique_ptr shredded_context, shredded_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto shredded_write, - shredded_table->NewWrite(std::move(shredded_context))); - ASSERT_OK(shredded_write->Write(MakeBatch(logical_schema, R"([ + auto second_commit_msgs = WriteAndPrepare(table_path, logical_schema, shredding_options, R"([ [2, [["c", 3], ["d", 4], ["e", 5]]] - ])"))); - ASSERT_OK_AND_ASSIGN(auto second_commit_msgs, - shredded_write->PrepareCommit(/*wait_compaction=*/false, - /*commit_identifier=*/1)); - ASSERT_OK(shredded_write->Close()); + ])", + /*commit_identifier=*/1); auto second_file_schema = ReadDataFileSchema(table_path, OnlyNewFile(second_commit_msgs), shredding_options); auto second_meta = ShreddingMeta(second_file_schema, /*field_index=*/1); diff --git a/src/paimon/core/operation/internal_read_context.cpp b/src/paimon/core/operation/internal_read_context.cpp index 3eb169ac7..8866eae9f 100644 --- a/src/paimon/core/operation/internal_read_context.cpp +++ b/src/paimon/core/operation/internal_read_context.cpp @@ -19,6 +19,7 @@ #include "paimon/core/operation/internal_read_context.h" #include +#include #include #include #include @@ -44,7 +45,10 @@ #include "paimon/common/utils/url_utils.h" #include "paimon/core/options/map_storage_layout.h" #include "paimon/core/schema/arrow_schema_validator.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/utils/branch_manager.h" #include "paimon/core/utils/nested_projection_utils.h" +#include "paimon/defs.h" #include "paimon/predicate/compound_predicate.h" #include "paimon/predicate/full_text_search.h" #include "paimon/predicate/predicate_utils.h" @@ -296,6 +300,50 @@ Result> InternalReadContext::Create( return CreateFromCoreOptions(context, table_schema, std::move(core_options), path); } +Result> InternalReadContext::CreateForFallbackBranch( + const std::shared_ptr& context, const std::string& branch, + const std::shared_ptr& file_system, const std::string& root_path, + const std::map& table_dynamic_options) { + std::shared_ptr table_schema; + const auto& specific_table_schema = context->GetSpecificTableSchema(); + if (branch == BranchManager::DEFAULT_MAIN_BRANCH && specific_table_schema) { + PAIMON_ASSIGN_OR_RAISE(table_schema, + TableSchema::CreateFromJson(specific_table_schema.value())); + } else { + auto schema_manager = std::make_shared(file_system, root_path, branch); + PAIMON_ASSIGN_OR_RAISE(std::optional> latest_schema, + schema_manager->Latest()); + if (!latest_schema) { + return Status::Invalid( + fmt::format("schema file not found in path {}, branch {}", root_path, branch)); + } + table_schema = latest_schema.value(); + } + assert(table_schema); + auto options = table_schema->Options(); + // The dynamic options the load merged into this table - and the ones its copies accumulated - + // speak for the read, and the fallback branch serves the same read: they are merged over the + // fallback's own schema options the way the load merged them over the table's, so the fallback + // reader recognises what the main one does. `type` stays the branch's own: neither a copy nor + // a fallback restyles the table. + for (const auto& [key, value] : table_dynamic_options) { + if (key != Options::TYPE) { + options[key] = value; + } + } + for (const auto& [key, value] : context->GetOptions()) { + if (key != Options::TYPE) { + options[key] = value; + } + } + if (branch != BranchManager::DEFAULT_MAIN_BRANCH) { + options[Options::BRANCH] = branch; + } else { + options.erase(Options::BRANCH); + } + return Create(context, table_schema, options, file_system, &root_path); +} + Result> InternalReadContext::CreateFromCoreOptions( const std::shared_ptr& context, const std::shared_ptr& table_schema, CoreOptions core_options, const std::string* path) { diff --git a/src/paimon/core/operation/internal_read_context.h b/src/paimon/core/operation/internal_read_context.h index b7218a686..43783e950 100644 --- a/src/paimon/core/operation/internal_read_context.h +++ b/src/paimon/core/operation/internal_read_context.h @@ -59,20 +59,17 @@ class InternalReadContext { }; /// Builds the context of a read a table serves: the options are the table's own, already - /// parsed - the store's `options()` - and the cache the context carries is armed on them - /// here. `path` names the root path the table was opened with, which the read reads from in - /// place of the one named by the context. + /// parsed, and the cache the context carries is armed on them here. `path` names the root path + /// the table was opened with, which the read reads from in place of the context's own. static Result> Create( const std::shared_ptr& read_context, const std::shared_ptr& table_schema, CoreOptions core_options, const std::string* path = nullptr); - /// Builds the context of a read whose options are composed on the spot rather than the - /// table's own - the fallback branch of a read, whose schema belongs to another branch, and - /// the reads a write or a lookup serve inside themselves. The options are parsed from - /// `options` here: the read's own file system is `file_system`, or the context's when null - - /// the scheme map the context carries does not take part. A cache the context carries is - /// armed on the options this parses. + /// Builds the context of a read whose options are composed on the spot rather than the table's + /// own - the fallback branch of a read, whose schema belongs to another branch, and the reads a + /// write or a lookup serve inside themselves. The read's own file system is `file_system`, or + /// the context's when null; the scheme map the context carries does not take part. static Result> Create( const std::shared_ptr& read_context, const std::shared_ptr& table_schema, @@ -80,6 +77,22 @@ class InternalReadContext { const std::shared_ptr& file_system = nullptr, const std::string* path = nullptr); + /// The read context of a fallback branch, whose schema is not the table's own: it is read + /// through a manager of that branch, and the read serves the options that schema was loaded + /// with, the branch it was read from among them. A branch the context carries is not honoured + /// here. The C++ reading of the Java `FileStoreTableFactory#createOtherBranchTable`, which + /// loads the other branch's table the same way for the fallback reader to serve. + /// @param file_system The file system the table was opened with, in place of one resolved from + /// the context. + /// @param root_path The root path the table was opened with, in place of the one the context + /// names. + /// @param table_dynamic_options The dynamic options the load merged into the table - and the + /// ones its copies accumulated. + static Result> CreateForFallbackBranch( + const std::shared_ptr& context, const std::string& branch, + const std::shared_ptr& file_system, const std::string& root_path, + const std::map& table_dynamic_options); + /// Maps score output positions in the read schema to their logical search sources. const std::map& GetIndexScoreBindings() const { return index_score_bindings_; diff --git a/src/paimon/core/operation/orphan_files_cleaner.cpp b/src/paimon/core/operation/orphan_files_cleaner.cpp index 397ba68ea..8fae06e2c 100644 --- a/src/paimon/core/operation/orphan_files_cleaner.cpp +++ b/src/paimon/core/operation/orphan_files_cleaner.cpp @@ -33,6 +33,7 @@ #include "paimon/core/operation/orphan_files_cleaner_impl.h" #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/utils/context_utils.h" #include "paimon/core/utils/field_mapping.h" #include "paimon/core/utils/file_store_path_factory.h" #include "paimon/core/utils/snapshot_manager.h" @@ -150,15 +151,7 @@ Result> CleanContextBuilder::Finish() { Result> OrphanFilesCleaner::Create( std::unique_ptr&& ctx) { - if (ctx == nullptr) { - return Status::Invalid("clean context is null pointer"); - } - if (ctx->GetMemoryPool() == nullptr) { - return Status::Invalid("memory pool is null pointer"); - } - if (ctx->GetExecutor() == nullptr) { - return Status::Invalid("executor is null pointer"); - } + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "clean")); if (ctx->GetOlderThanMs() < 0) { return Status::Invalid("older than time needs to be greater than or equal to 0."); } diff --git a/src/paimon/core/schema/schema_validation.cpp b/src/paimon/core/schema/schema_validation.cpp index 3a73838d9..5d112417f 100644 --- a/src/paimon/core/schema/schema_validation.cpp +++ b/src/paimon/core/schema/schema_validation.cpp @@ -76,6 +76,29 @@ namespace { constexpr char kDeletionVectorsMergeOnRead[] = "deletion-vectors.merge-on-read"; +/// The options a copy is refused to change, mirroring the Java `SchemaManagerUtils# +/// checkAlterTableOption` over the immutable options the C++ side has constants for. They name +/// the physical layout the data on the table's path was written by: a `bucket-key` the options +/// rename, say, re-hashes the bucket pruning while the files stay hashed by the old key, so the +/// scans a copy serves would filter out the buckets its rows are actually in. The check runs over +/// every option the call gives, before the merge, so it holds on every path that merges options +/// into a table. Unlike the Java set, `merge-engine`, `bucket`, `row-tracking.enabled` and +/// `data-evolution.enabled` stay servable as dynamic options here, and `type`/`branch` are dropped +/// by `Copy` before this point. +const char* const kImmutableOptions[] = { + Options::BUCKET_KEY, + Options::BUCKET_FUNCTION_TYPE, + Options::AGGREGATION_REMOVE_RECORD_ON_DELETE, + Options::PARTIAL_UPDATE_REMOVE_RECORD_ON_DELETE, + Options::PARTIAL_UPDATE_REMOVE_RECORD_ON_SEQUENCE_GROUP, + Options::FORCE_LOOKUP, + Options::INDEX_FILE_IN_DATA_FILE_DIR, + Options::BLOB_FIELD, + Options::FALLBACK_BLOB_DESCRIPTOR_FIELD, + Options::BLOB_VIEW_FIELD, + Options::PK_CLUSTERING_OVERRIDE, +}; + bool ContainsBlobField(const std::shared_ptr& field) { if (BlobUtils::IsBlobField(field)) { return true; @@ -402,6 +425,32 @@ Status SchemaValidation::ValidateTableSchema(const TableSchema& schema) { return Status::OK(); } +Status SchemaValidation::CheckImmutableOptions( + const TableSchema& schema, const std::map& dynamic_options) { + const std::map& schema_options = schema.Options(); + for (const auto& [key, value] : dynamic_options) { + auto schema_option = schema_options.find(key); + if (schema_option != schema_options.end() && schema_option->second == value) { + continue; + } + if (std::find(std::begin(kImmutableOptions), std::end(kImmutableOptions), key) != + std::end(kImmutableOptions)) { + return Status::Invalid(fmt::format("Change '{}' is not supported yet.", key)); + } + if (key == Options::IGNORE_DELETE) { + // An unset option is one that never ignored deletes, the same default they parse to. + PAIMON_ASSIGN_OR_RAISE(bool old_ignore_delete, + OptionsUtils::GetValueFromMap(schema_options, key, false)); + PAIMON_ASSIGN_OR_RAISE(bool new_ignore_delete, + OptionsUtils::GetValueFromMap(dynamic_options, key)); + if (old_ignore_delete && !new_ignore_delete) { + return Status::Invalid("Cannot change ignore-delete from true to false."); + } + } + } + return Status::OK(); +} + Status SchemaValidation::ValidateNoDuplicateField(const std::vector& field_names, const std::string& error_message_intro) { auto duplicate_field_names = ObjectUtils::DuplicateItems(field_names); diff --git a/src/paimon/core/schema/schema_validation.h b/src/paimon/core/schema/schema_validation.h index 813aaebb4..ba169c057 100644 --- a/src/paimon/core/schema/schema_validation.h +++ b/src/paimon/core/schema/schema_validation.h @@ -51,6 +51,15 @@ class SchemaValidation { static Status ValidateTableSchema(const TableSchema& schema); + /// Whether a copy's dynamic options stay within what the data on the table's path was laid out + /// with - the C++ reading of the Java `AbstractFileStoreTable#checkImmutability`. An option + /// that says what the schema already says moves nothing, an immutable one that says anything + /// else is refused, and `ignore-delete` carries the Java check's directional rule of its own: + /// it may only ever be turned on, never back off. The immutable options themselves are the + /// Java `CoreOptions.IMMUTABLE_OPTIONS`, as far as the C++ side has constants for them. + static Status CheckImmutableOptions(const TableSchema& schema, + const std::map& dynamic_options); + /// Validates real-time options after operation-specific options have been merged with the /// table schema options. static Status ValidateRealtime(const TableSchema& schema, const CoreOptions& options); diff --git a/src/paimon/core/table/append_only_file_store_table.cpp b/src/paimon/core/table/append_only_file_store_table.cpp index cead79566..fb504ee4c 100644 --- a/src/paimon/core/table/append_only_file_store_table.cpp +++ b/src/paimon/core/table/append_only_file_store_table.cpp @@ -38,7 +38,6 @@ #include "paimon/core/table/bucket_mode.h" #include "paimon/core/table/file_store.h" #include "paimon/core/table/source/append_only_split_generator.h" -#include "paimon/core/table/source/append_only_table_read.h" #include "paimon/core/table/source/data_evolution_split_generator.h" #include "paimon/core/table/source/split_generator.h" #include "paimon/core/utils/snapshot_manager.h" @@ -60,7 +59,7 @@ Result> AppendOnlyFileStoreTable::CreateWrite( const std::shared_ptr& snapshot_manager, const std::shared_ptr& io_manager, bool ignore_previous_files) const { // append table - const std::shared_ptr& schema = store_->schema(); + const std::shared_ptr& schema = store_->GetSchema(); const bool realtime_deduplicate = !options.GetRealtimeDeduplicateKeyFields().empty(); bool need_dv_maintainer_factory = options.DeletionVectorsEnabled(); if (options.GetBucket() == -1) { @@ -139,7 +138,7 @@ Result> AppendOnlyFileStoreTable::NewFileStoreSca const std::shared_ptr& executor, const std::shared_ptr& memory_pool, const ScanContext* context) const { // TODO(liancheng.lsz): support fallback branch in scan - const std::shared_ptr& table_schema = store_->schema(); + const std::shared_ptr& table_schema = store_->GetSchema(); std::unique_ptr scan; if (core_options.DataEvolutionEnabled()) { PAIMON_ASSIGN_OR_RAISE( @@ -172,15 +171,6 @@ Result> AppendOnlyFileStoreTable::NewSplitGenera source_split_open_file_cost, bucket_mode); } -Result> AppendOnlyFileStoreTable::NewTableRead( - const std::shared_ptr& path_factory, - const std::shared_ptr& internal_context, - const std::shared_ptr& memory_pool, - const std::shared_ptr& executor) const { - return std::make_unique(path_factory, internal_context, memory_pool, - executor); -} - CommitScanner::ScanSupplier AppendOnlyFileStoreTable::NewCommitScanSupplier( const std::shared_ptr& snapshot_manager, const std::shared_ptr& manifest_list, @@ -190,7 +180,7 @@ CommitScanner::ScanSupplier AppendOnlyFileStoreTable::NewCommitScanSupplier( // The schema manager the table carries, which the scans this supplier hands out read the // schema through. const std::shared_ptr& schema_manager = GetSchemaManager(); - const std::shared_ptr& table_schema = store_->schema(); + const std::shared_ptr& table_schema = store_->GetSchema(); return [snapshot_manager, schema_manager, manifest_list, manifest_file, table_schema, arrow_schema, options, executor, pool](const std::shared_ptr& scan_filter) -> Result> { diff --git a/src/paimon/core/table/append_only_file_store_table.h b/src/paimon/core/table/append_only_file_store_table.h index 499431eb0..9b4af46ae 100644 --- a/src/paimon/core/table/append_only_file_store_table.h +++ b/src/paimon/core/table/append_only_file_store_table.h @@ -63,12 +63,6 @@ class AppendOnlyFileStoreTable : public FileStoreTable { Result> NewSplitGenerator( const CoreOptions& core_options) const override; - Result> NewTableRead( - const std::shared_ptr& path_factory, - const std::shared_ptr& internal_context, - const std::shared_ptr& memory_pool, - const std::shared_ptr& executor) const override; - CommitScanner::ScanSupplier NewCommitScanSupplier( const std::shared_ptr& snapshot_manager, const std::shared_ptr& manifest_list, diff --git a/src/paimon/core/table/file_store.cpp b/src/paimon/core/table/file_store.cpp index 472626a01..fe8b092c7 100644 --- a/src/paimon/core/table/file_store.cpp +++ b/src/paimon/core/table/file_store.cpp @@ -47,6 +47,19 @@ namespace paimon { +namespace { + +/// Whether an option key takes part in file-system resolution: the backend `file-system` picks, +/// an `fs.` access key carries, or a native S3 key names - `s3.` and its `s3a.` spelling, the +/// prefixes the S3 file system consumes alongside `fs.s3.`/`fs.s3a.`. The scheme-map keys a +/// context carries are the same set. +bool IsFileSystemResolutionKey(const std::string& key) { + return key == Options::FILE_SYSTEM || key.rfind("fs.", 0) == 0 || key.rfind("s3.", 0) == 0 || + key.rfind("s3a.", 0) == 0; +} + +} // namespace + Result> FileStore::Create( const std::shared_ptr& file_system, const std::string& root_path, const std::shared_ptr& schema, const std::string& branch, @@ -97,6 +110,42 @@ Result> FileStore::Create( return store; } +bool FileStore::FileSystemResolutionMayChange( + const TableSchema& schema, const std::map& dynamic_options) { + const std::map& schema_options = schema.Options(); + for (const auto& [key, value] : dynamic_options) { + if (!IsFileSystemResolutionKey(key)) { + continue; + } + auto schema_option = schema_options.find(key); + if (schema_option == schema_options.end() || schema_option->second != value) { + return true; + } + } + return false; +} + +bool FileStore::FileSystemResolutionAgrees(const std::map& left, + const std::map& right) { + for (const auto& [key, value] : left) { + if (IsFileSystemResolutionKey(key)) { + auto right_option = right.find(key); + if (right_option == right.end() || right_option->second != value) { + return false; + } + } + } + for (const auto& [key, value] : right) { + if (IsFileSystemResolutionKey(key)) { + auto left_option = left.find(key); + if (left_option == left.end() || left_option->second != value) { + return false; + } + } + } + return true; +} + FileStore::FileStore(const std::shared_ptr& file_system, const std::string& root_path, const std::shared_ptr& schema, const std::string& branch, const std::shared_ptr& schema_manager, diff --git a/src/paimon/core/table/file_store.h b/src/paimon/core/table/file_store.h index 66ccc09c2..e343ec2c4 100644 --- a/src/paimon/core/table/file_store.h +++ b/src/paimon/core/table/file_store.h @@ -47,46 +47,34 @@ class ManifestList; class TableSchema; /// The storage a managed table is served through: the file system, root path, schema, branch, -/// options and the schema and snapshot managers it was loaded with - the state Java Paimon's -/// `AbstractFileStore` carries, parsed once here rather than per operator. `FileStoreTable` hands -/// the state it was loaded with to one of these and asks it back for the pieces every operator -/// opens with. +/// options and managers it was loaded with - the state Java Paimon's `AbstractFileStore` carries, +/// parsed once here rather than per operator. `FileStoreTable` hands the state it was loaded with +/// to one of these and asks it back for the pieces every operator opens with. /// -/// The options are the ones the table was loaded with on this branch - the schema's own, or -/// the ones a load merged over them - the base every operator serves: `NewXxx` takes no -/// options of its own beyond them. A cache is not part of them - `WithCache()` arms a copy an -/// operator makes - so the factories below take the options of the call, not the store's own. -/// -/// The factory methods build a fresh instance on every call, never a cached one: a path factory -/// names each file it builds, and a snapshot manager a write or a commit arms with a catalog -/// loader is not one a scan should be served through afterwards. +/// Every operator serves the options the table was loaded with, so the factories below take the +/// options of the call: a cache is not part of the table's, `WithCache()` arms a copy an operator +/// makes. They build a fresh instance on every call, never a cached one - a snapshot manager a +/// write arms with a catalog loader is not one a scan should be served through afterwards. class FileStore { public: /// @param file_system The file system the caller opened the table through, which wins over - /// the access configuration the schema's options may carry when it is not null. A - /// null one hands the resolution to that configuration: the table is then served by - /// the file system its options name - the way a load that only had a path and a - /// schema to read names it - and by the default local one when they name none. + /// the access configuration the schema's options may carry when it is not null. A null + /// one hands the resolution to that configuration, and to the default local file system + /// when the options name none. /// @param root_path Root path of the table in the file system. - /// @param schema The schema the table was loaded with, whose partition keys name the - /// partitions the paths below are built for and whose options `options()` parses. It - /// is null when the caller had no schema; the operators then report that none was - /// found. + /// @param schema The schema the table was loaded with, whose partition keys name the partitions + /// the paths below are built for and whose options `GetOptions()` parses. Null when the + /// caller had none, which the operators then report. /// @param branch The branch the store serves, normalized here. - /// @param schema_manager A schema manager to carry instead of building one - the one a - /// context dispatch read the schema through, whose cache the operators then serve. - /// @param snapshot_manager A snapshot manager to carry instead of building one. - /// A null manager is built over `root_path` on `branch`, so the operators always read - /// through the managers the store carries. + /// @param schema_manager A schema manager to carry instead of building one - the one a context + /// dispatch read the schema through, whose cache the operators then serve. + /// @param snapshot_manager A snapshot manager to carry instead of building one; a null one is + /// built over `root_path` on `branch`. /// @param fs_scheme_map The scheme-to-identifier map the caller's contexts resolve custom - /// schemes through. It takes part in building the store's options the way it took - /// part in reading the schema: without it a schema naming a custom scheme would - /// resolve to the default backend once its operators run. It is kept for a copy to - /// hand over again, so the copy resolves as this store does. + /// schemes through, kept so a copy resolves as this store does. /// - /// Building the store parses the schema's options, so it fails when they do not parse: the - /// error is the caller's to see, which is why a store is built through this factory rather - /// than the constructor. + /// Building the store parses the schema's options, so it fails when they do not parse - which + /// is why a store is built through this factory rather than the constructor. static Result> Create( const std::shared_ptr& file_system, const std::string& root_path, const std::shared_ptr& schema, const std::string& branch, @@ -94,69 +82,74 @@ class FileStore { const std::shared_ptr& snapshot_manager = nullptr, const std::map& fs_scheme_map = {}); - /// The branch this store serves, as resolved when it was built. - const std::string& branch() const { + /// Whether a copy's dynamic options change the file system its store resolves to. If they do, + /// the managers of the table it was copied from are not its to inherit: built over that + /// table's resolution, they would keep reading metadata through the backend the copy just + /// left. + static bool FileSystemResolutionMayChange( + const TableSchema& schema, const std::map& dynamic_options); + + /// Whether two options maps resolve to the same file system: every resolution key has to + /// agree, a missing one on both sides included. + static bool FileSystemResolutionAgrees(const std::map& left, + const std::map& right); + + const std::string& CurrentBranch() const { return branch_; } - /// The schema manager the operators read through. It carries the schema cache of `root_path` - /// on `branch()`. - const std::shared_ptr& schema_manager() const { + /// The schema manager the operators read through, carrying the schema cache of `root_path` on + /// `CurrentBranch()`. + const std::shared_ptr& GetSchemaManager() const { return schema_manager_; } /// The snapshot manager a scan reads through. A write or a commit builds its own: see /// `NewSnapshotManager`. - const std::shared_ptr& snapshot_manager() const { + const std::shared_ptr& GetSnapshotManager() const { return snapshot_manager_; } - /// The schema the table was loaded with, which every operator serves rather than reading the - /// schema again. It is null when the caller had no schema. - const std::shared_ptr& schema() const { + /// The schema the table was loaded with, null when the caller had none. + const std::shared_ptr& GetSchema() const { return schema_; } - /// The file system the table's data is accessed through, as the table was opened with it: - /// the one the options resolve - a schema carrying access configuration names its own, - /// which the file system the caller opened the table through wins over - rather than the - /// one that call handed over, which `specified_file_system()` hands back untouched. - const std::shared_ptr& file_system() const { + /// The file system the table's data is accessed through: the one the options resolve, which + /// the file system the caller opened the table through wins over. + const std::shared_ptr& GetFileSystem() const { return file_system_; } - /// The file system the caller opened the table through, as it was handed to `Create`: null - /// when the call had none, so a copy of the table resolves from its own options again and - /// the runtime ones it merges in - a file-system option among them - can speak. - const std::shared_ptr& specified_file_system() const { + /// The file system the caller opened the table through, as handed to `Create`: null when the + /// call had none, so a copy resolves from its own options again. + const std::shared_ptr& GetSpecifiedFileSystem() const { return specified_file_system_; } /// The scheme-to-identifier map the options were resolved with, kept for a copy to resolve - /// through again: see `Create`'s `fs_scheme_map`. - const std::map& fs_scheme_map() const { + /// through again. + const std::map& GetFsSchemeMap() const { return fs_scheme_map_; } - /// The root path of the table in its file system, as the table was opened with it. - const std::string& root_path() const { + const std::string& GetRootPath() const { return root_path_; } - /// The options the schema was loaded with on `branch()`, which every operator serves. They - /// exist only over a schema: an operator reads them after reporting that no schema was - /// found when there is none. - const CoreOptions& options() const { + /// The options the schema was loaded with on `CurrentBranch()`. They exist only over a schema: + /// an operator reads them after reporting that none was found when there is not. + const CoreOptions& GetOptions() const { assert(core_options_.has_value()); return *core_options_; } /// The path factory an operator builds its metadata paths through: over this store's root - /// path and the partition keys of its schema - which no schema change can alter, so a read - /// serving a fallback branch's schema still names the same partitions. The file format - /// identifier is the caller's to name: a write names the format its level-0 files are - /// written in, which the options may set per level, while a scan, a read and a commit name - /// the table's format. + /// path and the partition keys of its schema, which no schema change can alter, so an + /// operator serving a schema read after the table was loaded still names the same partitions. + /// The file format identifier is the caller's to name: a write names the format its level-0 + /// files are written in, which the options may set per level, while a scan, a read and a + /// commit name the table's format. Result> NewPathFactory( const std::shared_ptr& arrow_schema, const CoreOptions& options, const std::string& file_format_identifier, @@ -164,46 +157,39 @@ class FileStore { /// The snapshot manager a write or a commit reads snapshots through, over this store's root /// path and branch. The file system is the caller's to name, from the options its operator - /// serves: a table opened without one has its options build the resolving file system. A - /// catalog that version-manages the table hands its snapshots out itself, so its loader is - /// attached, the catalog captured to keep the versioned interface - and the loader with it - /// - alive. Such a catalog names the table by an identifier, which the context must have - /// supplied. + /// serves. A catalog that version-manages the table hands its snapshots out itself, so its + /// loader is attached and the catalog captured to keep it alive; such a catalog names the + /// table by an identifier, which the context must have supplied. Result> NewSnapshotManager( const CoreOptions& options, const std::shared_ptr& catalog, const std::optional& catalog_identifier) const; - /// The manifest list a scan or a commit reads the table's manifest entries through, over - /// the caller's path factory: an operator's manifests sit in the paths its path factory - /// names. + /// The manifest list a scan or a commit reads the table's manifest entries through. Result> NewManifestList( const CoreOptions& options, const std::shared_ptr& path_factory, const std::shared_ptr& memory_pool) const; - /// The manifest file a scan or a commit reads a single manifest's entries through, over - /// the caller's path factory and the partition schema its entries are pruned by. + /// The manifest file a scan or a commit reads a single manifest's entries through, pruned by + /// `partition_schema`. Result> NewManifestFile( const CoreOptions& options, const std::shared_ptr& path_factory, const std::shared_ptr& partition_schema, const std::shared_ptr& memory_pool) const; /// The index manifest file a write, a scan or a commit reads the table's index manifest - /// through, over the caller's path factory. + /// through. Result> NewIndexManifestFile( const CoreOptions& options, const std::shared_ptr& path_factory, const std::shared_ptr& memory_pool) const; - /// The index file handler a write's deletion vectors and indexes are maintained through, or - /// a scan reads them through: the index manifest file above and the path factories over the - /// caller's path factory, in one handler. + /// The index file handler a write maintains its deletion vectors and indexes through, or a + /// scan reads them through. Result> NewIndexFileHandler( const CoreOptions& options, const std::shared_ptr& path_factory, const std::shared_ptr& memory_pool) const; /// The partition computer a commit names the partitions it writes with, over the partition - /// keys of this store's schema - which no schema change can alter, so a commit serving a - /// schema read after the table was loaded still names the same partitions - and the - /// caller's arrow schema and options. + /// keys of this store's schema and the caller's arrow schema. Result> NewPartitionComputer( const std::shared_ptr& arrow_schema, const CoreOptions& options, const std::shared_ptr& memory_pool) const; @@ -216,20 +202,14 @@ class FileStore { const std::map& fs_scheme_map); std::shared_ptr file_system_; - /// The file system the caller opened the table through, as handed to `Create` - null when - /// the call had none - kept for a copy to hand over again: what `file_system_` holds is - /// resolved, and would pin the copy to this table's resolution. std::shared_ptr specified_file_system_; - /// The scheme-to-identifier map the options were resolved with, kept for a copy to resolve - /// through again. std::map fs_scheme_map_; std::string root_path_; std::shared_ptr schema_; std::string branch_; std::shared_ptr schema_manager_; std::shared_ptr snapshot_manager_; - /// The options of `schema_` on `branch_`, or unset when there is no schema. `Create` parses - /// them: the constructor cannot report a parse failure. + /// The options of `schema_` on `branch_`, unset when there is no schema. std::optional core_options_; }; diff --git a/src/paimon/core/table/file_store_table.cpp b/src/paimon/core/table/file_store_table.cpp index 1a5c15a7d..c7bc8261b 100644 --- a/src/paimon/core/table/file_store_table.cpp +++ b/src/paimon/core/table/file_store_table.cpp @@ -19,13 +19,11 @@ #include "paimon/core/table/file_store_table.h" -#include #include #include #include #include #include -#include #include "fmt/format.h" #include "paimon/catalog/catalog.h" @@ -34,12 +32,9 @@ #include "paimon/common/predicate/predicate_validator.h" #include "paimon/common/types/data_field.h" #include "paimon/common/utils/binary_row_partition_computer.h" -#include "paimon/common/utils/checked_cast.h" #include "paimon/common/utils/options_utils.h" -#include "paimon/common/utils/path_util.h" #include "paimon/common/utils/sensitive_config_utils.h" #include "paimon/common/utils/string_utils.h" -#include "paimon/core/catalog/catalog_snapshot_commit.h" #include "paimon/core/catalog/catalog_utils.h" #include "paimon/core/catalog/commit_lock.h" #include "paimon/core/catalog/renaming_snapshot_commit.h" @@ -55,7 +50,6 @@ #include "paimon/core/operation/file_store_commit_impl.h" #include "paimon/core/operation/file_store_scan.h" #include "paimon/core/operation/internal_read_context.h" -#include "paimon/core/options/table_type.h" #include "paimon/core/realtime/realtime_context_impl.h" #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/schema_validation.h" @@ -63,20 +57,22 @@ #include "paimon/core/snapshot.h" #include "paimon/core/table/append_only_file_store_table.h" #include "paimon/core/table/file_store.h" +#include "paimon/core/table/file_store_table_factory.h" #include "paimon/core/table/primary_key_file_store_table.h" +#include "paimon/core/table/source/append_only_table_read.h" #include "paimon/core/table/source/data_evolution_batch_scan.h" #include "paimon/core/table/source/data_table_batch_scan.h" #include "paimon/core/table/source/data_table_stream_scan.h" #include "paimon/core/table/source/fallback_table_read.h" +#include "paimon/core/table/source/key_value_table_read.h" #include "paimon/core/table/source/primary_key_index_batch_scan.h" #include "paimon/core/table/source/read_optimized_scan_options.h" #include "paimon/core/table/source/realtime_table_scan.h" #include "paimon/core/table/source/snapshot/snapshot_reader.h" -#include "paimon/core/table/system/system_table.h" #include "paimon/core/utils/branch_manager.h" +#include "paimon/core/utils/context_utils.h" #include "paimon/core/utils/field_mapping.h" #include "paimon/core/utils/file_store_path_factory.h" -#include "paimon/core/utils/primary_key_table_utils.h" #include "paimon/core/utils/snapshot_manager.h" #include "paimon/file_store_commit.h" #include "paimon/file_store_write.h" @@ -89,7 +85,6 @@ #include "paimon/scan_context.h" #include "paimon/schema/schema.h" #include "paimon/status.h" -#include "paimon/table/format/format_table.h" #include "paimon/table/source/table_read.h" #include "paimon/table/source/table_scan.h" #include "paimon/write_context.h" @@ -103,477 +98,9 @@ namespace paimon { class Executor; class MemoryPool; -namespace { - -/// The identifier a table path names when no catalog name goes with it, the C++ reading of -/// the Java `SchemaManagerUtils#identifierFromPath` over `ignoreIfUnknownDatabase = true`: the -/// directory above the table's names the database when it carries the `.db` suffix, the way a -/// catalog lays its databases out, and a path that names no database - a table sitting directly -/// under the root, say - leaves the database unknown, which is all a caller that only knows the -/// directory can say. -Identifier IdentifierFromPath(const std::string& table_path, const std::string& branch) { - const std::vector paths = StringUtils::Split(table_path, "/"); - if (paths.size() < 2) { - return Identifier(Identifier::kUnknownDatabase, - paths.empty() ? std::string() : paths.front()); - } - const std::string& database = paths[paths.size() - 2]; - const std::string::size_type index = database.rfind(Catalog::DB_SUFFIX); - if (index == std::string::npos) { - return Identifier(Identifier::kUnknownDatabase, paths.back(), branch); - } - return Identifier(database.substr(0, index), paths.back(), branch); -} - -/// Recognises a format table behind `table_path`, or returns null when it is not one. -/// -/// The scan, read, write and commit entry points below all take a path and have to answer the -/// same question before they can dispatch, so one place answers it and they cannot disagree. -/// Null is also the answer when no schema was found under the path at all: what to say about a -/// table that is not there is the managed path's to decide, and it says it in more detail. -/// -/// The table type is read from the schema, never from `options`: `type` is structural, and one -/// call's options must not decide what kind of table this is. `branch` names the branch the -/// schema was read from: anything but the main branch is refused once the schema turns out to be -/// a format table's, which has no branches to read. -Result> TryLoadFormatTable( - const std::shared_ptr& file_system, const std::string& table_path, - const std::string& branch, const std::map& options, - const std::shared_ptr& table_schema) { - if (table_schema == nullptr) { - return std::shared_ptr(); - } - // From the schema alone: see `FormatTable::Create()`. - PAIMON_ASSIGN_OR_RAISE(bool is_format_table, - TableTypeDefine::IsFormatTable(table_schema->Options())); - if (!is_format_table) { - return std::shared_ptr(); - } - // A format table has no branches, so it is refused rather than served from the table path - // under another branch's name. See `SchemaValidation::ValidateFormatTableSchema()`. - if (BranchManager::NormalizeBranch(branch) != BranchManager::DEFAULT_MAIN_BRANCH) { - return Status::Invalid( - fmt::format("a format table has no branches, so {} cannot be read on branch '{}'", - table_path, branch)); - } - // A context names a path, never a catalog identifier, so the table is known by the - // directory it sits in - which names the database too, when it carries the `.db` suffix a - // catalog lays its databases out with. - Identifier identifier = IdentifierFromPath(table_path, branch); - return FormatTable::Create(file_system, table_path, identifier, - checked_pointer_cast(table_schema), - /*location_carries_paimon_metadata=*/true, options); -} - -/// Whether an option of a copy names the file system its store resolves: the backend -/// `file-system` picks, or the access configuration a `fs.` key carries. A copy whose dynamic -/// options change one of these resolves to another file system, so the managers of the table it -/// was copied from - built over that table's resolution - are not its to inherit: they would -/// keep reading metadata through the backend the copy just left. -bool FileSystemResolutionMayChange(const TableSchema& schema, - const std::map& dynamic_options) { - const std::map& schema_options = schema.Options(); - for (const auto& [key, value] : dynamic_options) { - if (key != Options::FILE_SYSTEM && key.rfind("fs.", 0) != 0) { - continue; - } - auto schema_option = schema_options.find(key); - if (schema_option == schema_options.end() || schema_option->second != value) { - return true; - } - } - return false; -} - -/// The options a copy is refused to change, mirroring the Java `SchemaManagerUtils# -/// checkAlterTableOption` over the immutable options the C++ side has constants for. These name -/// the physical layout and the file set the data on the table's path was written by: a -/// `bucket-key` the options rename, say, re-hashes the bucket pruning while the files stay -/// hashed by the old key, so the scans a copy serves would filter out the buckets its rows are -/// actually in. The check runs over every option the call gives, before the merge, so it holds -/// on every path that merges options into a table - `Copy` itself and the load a context -/// drives alike. The differences from the Java set: `merge-engine`, `bucket`, -/// `row-tracking.enabled` and `data-evolution.enabled` stay servable as dynamic options here - -/// the C++ tests load tables through context options that move them - and `type`/`branch` are -/// dropped by `Copy` before this point. -const char* const kImmutableOptions[] = { - Options::BUCKET_KEY, - Options::BUCKET_FUNCTION_TYPE, - Options::AGGREGATION_REMOVE_RECORD_ON_DELETE, - Options::PARTIAL_UPDATE_REMOVE_RECORD_ON_DELETE, - Options::PARTIAL_UPDATE_REMOVE_RECORD_ON_SEQUENCE_GROUP, - Options::FORCE_LOOKUP, - Options::INDEX_FILE_IN_DATA_FILE_DIR, - Options::BLOB_FIELD, - Options::FALLBACK_BLOB_DESCRIPTOR_FIELD, - Options::BLOB_VIEW_FIELD, - Options::PK_CLUSTERING_OVERRIDE, -}; - -/// Whether the dynamic options of a copy stay within what the data on the table's path was -/// laid out with - the C++ reading of the Java `AbstractFileStoreTable#checkImmutability`: -/// every option that says what the schema already says moves nothing, an immutable one that -/// says anything else is refused, and an `ignore-delete` carries the directional rule of its -/// own the Java check carries too: it may only ever be turned on, never back off. -Status CheckImmutableOptions(const TableSchema& schema, - const std::map& dynamic_options) { - const std::map& schema_options = schema.Options(); - for (const auto& [key, value] : dynamic_options) { - auto schema_option = schema_options.find(key); - if (schema_option != schema_options.end() && schema_option->second == value) { - continue; - } - if (std::find(std::begin(kImmutableOptions), std::end(kImmutableOptions), key) != - std::end(kImmutableOptions)) { - return Status::Invalid(fmt::format("Change '{}' is not supported yet.", key)); - } - if (key == Options::IGNORE_DELETE) { - // An unset option is one that never ignored deletes, the same default the options - // parse to. - PAIMON_ASSIGN_OR_RAISE(bool old_ignore_delete, - OptionsUtils::GetValueFromMap(schema_options, key, false)); - PAIMON_ASSIGN_OR_RAISE(bool new_ignore_delete, - OptionsUtils::GetValueFromMap(dynamic_options, key)); - if (old_ignore_delete && !new_ignore_delete) { - return Status::Invalid("Cannot change ignore-delete from true to false."); - } - } - } - return Status::OK(); -} - -/// The options a context carries, merged into the table the load hands back: a table is the -/// product of the load, so the options speak there - the way `Copy` merges them, over the ones -/// the schema carries - and `NewXxx` on the product serves them without merging again. A -/// caller holding a table already - a load that is over - changes it through `Copy` itself, -/// and an option that contradicts the table that way is refused. -Result> WithContextOptions( - const std::shared_ptr
& table, const std::shared_ptr& schema, - const std::map& options) { - if (table == nullptr || schema == nullptr || options.empty()) { - return table; - } - const std::map& schema_options = schema->Options(); - for (const auto& [key, value] : options) { - auto schema_option = schema_options.find(key); - // A key the schema does not carry, or carries differently: the merge has something to - // do. Every key agreeing with the schema leaves the table the load built as it is. - if (schema_option == schema_options.end() || schema_option->second != value) { - return table->Copy(options); - } - } - return table; -} - -/// The tail every `TableOf` ends with: the schema one of them read says what the path names. A -/// format table is handed back as the `FormatTable` it is, everything else as the -/// `FileStoreTable` a managed table is, carrying the schema for its `NewXxx` to build from. A -/// null schema is handed over as it is: what to say about a table that is not there is the -/// managed path's to decide, and it says it in more detail. -/// -/// @param file_system The file system the schema above was read through, which a format table -/// is served by - it has no schema options of its own to name another one. -/// @param explicit_file_system The file system the context supplied, which a managed table is -/// opened through when there is one. A null one leaves the table to resolve from the -/// options its schema carries: access configuration the schema was stored with then -/// names the files it serves, rather than being pinned to the one this load happened to -/// read the schema through. -Result> TableOfLoadedSchema( - const std::shared_ptr& file_system, - const std::shared_ptr& explicit_file_system, const std::string& path, - const std::string& branch, const std::map& options, - const std::shared_ptr& schema_manager, - const std::shared_ptr& latest_schema, const std::optional& identifier, - const std::string& uuid = "", const std::map& fs_scheme_map = {}) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, - TryLoadFormatTable(file_system, path, branch, options, latest_schema)); - if (format_table != nullptr) { - return std::shared_ptr
(std::move(format_table)); - } - // A context names a path, never a catalog identifier, so the table is known by the - // directory it sits in - which names the database too, when it carries the `.db` suffix a - // catalog lays its databases out with. The branch the schema was read on travels the same - // way, folded into the object name. - Identifier table_identifier = identifier.value_or(IdentifierFromPath(path, branch)); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, - FileStoreTable::Create(explicit_file_system, path, latest_schema, - table_identifier, uuid, branch, schema_manager, - /*snapshot_manager=*/nullptr, fs_scheme_map)); - return WithContextOptions(table, latest_schema, options); -} - -/// The managed table a context names behind a path: the schema is read through a manager of the -/// context's branch - handed over instead when the caller already read it - and what the schema -/// says is there decides the table `TableOfLoadedSchema` hands back. -/// -/// @param schema_json The schema of the table, when the caller already read it, or null when it -/// has to be read here. It is honoured on the main branch only: a branch publishes its own -/// under `branch/branch-`, which is where the read below goes. -/// @param explicit_file_system The file system the context supplied, which the managed table the -/// schema describes is opened through - see `TableOfLoadedSchema`. -Result> ManagedTableOf( - const std::shared_ptr& file_system, - const std::shared_ptr& explicit_file_system, const std::string& path, - const std::string& branch, const std::map& options, - const std::optional& schema_json, const std::optional& identifier, - const std::string& uuid = "", const std::map& fs_scheme_map = {}) { - auto schema_manager = std::make_shared(file_system, path, branch); - std::shared_ptr latest_schema; - if (branch == BranchManager::DEFAULT_MAIN_BRANCH && schema_json) { - // Handing the schema over only saves reading it. A context always names a table path and - // this library writes the metadata under it, so `schema` and `branch` stay metadata. - PAIMON_ASSIGN_OR_RAISE(latest_schema, TableSchema::CreateFromJson(schema_json.value())); - } else { - PAIMON_ASSIGN_OR_RAISE(std::optional> latest, - schema_manager->Latest()); - if (latest) { - latest_schema = *latest; - } - } - return TableOfLoadedSchema(file_system, explicit_file_system, path, branch, options, - schema_manager, latest_schema, identifier, uuid, fs_scheme_map); -} - -/// The identifier a catalog knows the table by, on `branch`: a non-main branch is folded into -/// the identifier, so the catalog is asked for the table on that branch rather than the main one. -Result> MakeBranchCatalogIdentifier( - const std::optional& identifier, const std::string& branch) { - if (!identifier) { - return std::optional(); - } - PAIMON_ASSIGN_OR_RAISE(Identifier branch_identifier, - CatalogUtils::BranchIdentifier(identifier.value(), branch)); - return std::optional(std::move(branch_identifier)); -} - -/// The schema a catalog may hold for the table, read from it on the main branch only and null -/// otherwise. A branch publishes its own schema under `branch/branch-`, which is where -/// `ManagedTableOf` reads instead, so a catalog answering for no branch schema does not fail the -/// `kind` operation to one. `kind` also names the operator the missing-identifier message speaks -/// of. -Result> CatalogMainBranchSchemaJson( - const std::shared_ptr& catalog, const std::optional& catalog_identifier, - const std::string& branch, const char* kind) { - if (catalog == nullptr) { - return std::optional(); - } - if (!catalog_identifier) { - return Status::Invalid(fmt::format("a catalog {} requires a table identifier", kind)); - } - if (BranchManager::IsMainBranch(branch)) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, - catalog->LoadTableSchema(catalog_identifier.value())); - PAIMON_ASSIGN_OR_RAISE(std::string schema_json, schema->GetJsonSchema()); - return std::optional(std::move(schema_json)); - } - return std::optional(); -} - -/// The table a scan context names, read from what the context carries: a format table the -/// builder was handed, a system table behind a `T$system` path, or the table - format or -/// managed - the schema under the context's path says is there. -Result> TableOf(const ScanContext& ctx) { - // A table the caller already loaded says what it is, so nothing is read to find out. - if (ctx.GetFormatTable() != nullptr) { - return std::shared_ptr
(ctx.GetFormatTable()); - } - PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, - CoreOptions::FromMap(ctx.GetOptions(), ctx.GetSpecificFileSystem(), {})); - PAIMON_ASSIGN_OR_RAISE(std::optional system_table_path, - SystemTableLoader::TryParsePath(ctx.GetPath())); - if (system_table_path) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, - SystemTableLoader::LoadFromPath(tmp_options.GetFileSystem(), - ctx.GetPath(), ctx.GetOptions())); - return std::shared_ptr
(std::move(system_table)); - } - // A format table is planned by listing directories, so it never reaches the manifest path - // below. One `TableScan` interface serves both, as Java Paimon serves both through one - // `ReadBuilder`. - const std::string branch = BranchManager::NormalizeBranch(tmp_options.GetBranch()); - return ManagedTableOf(tmp_options.GetFileSystem(), ctx.GetSpecificFileSystem(), ctx.GetPath(), - branch, ctx.GetOptions(), ctx.GetSpecificTableSchema(), - ctx.GetIdentifier()); -} - -/// The table a read context names, read from what the context carries: a format table the -/// builder was handed, a system table behind a `T$system` path, or the table - format or -/// managed - the schema under the context's path says is there. -Result> TableOf(const ReadContext& ctx) { - // A table the caller already loaded says what it is, so nothing is read to find out. - if (ctx.GetFormatTable() != nullptr) { - return std::shared_ptr
(ctx.GetFormatTable()); - } - PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_core_options, - CoreOptions::FromMap(ctx.GetOptions(), ctx.GetSpecificFileSystem(), - ctx.GetFileSystemSchemeToIdentifierMap())); - PAIMON_ASSIGN_OR_RAISE(std::optional system_table_path, - SystemTableLoader::TryParsePath(ctx.GetPath())); - if (system_table_path) { - if (PredicateUtils::ContainsFullTextMatch(ctx.GetPredicate())) { - return Status::NotImplemented( - "FullTextMatch predicate execution is not implemented yet"); - } - if (ctx.HasVectorOrFullTextSearch()) { - return Status::NotImplemented( - "VectorSearch or FullTextSearch is not supported for system tables"); - } - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, - SystemTableLoader::LoadFromPath(tmp_core_options.GetFileSystem(), - ctx.GetPath(), ctx.GetOptions())); - return std::shared_ptr
(std::move(system_table)); - } - - // A format table has no manifest and a file another engine wrote carries no field ids, so - // which files there are and how a row is put back together is its own. How a file is opened - // is not: both paths go through `DataFileReaderFactory`. - const std::string branch = ctx.GetBranch(); - // The scheme map the schema was read through, so the table it describes resolves custom - // schemes the same way - the data belongs to the metadata, both sit behind the map. - return ManagedTableOf(tmp_core_options.GetFileSystem(), ctx.GetSpecificFileSystem(), - ctx.GetPath(), branch, ctx.GetOptions(), ctx.GetSpecificTableSchema(), - ctx.GetIdentifier(), /*uuid=*/"", - ctx.GetFileSystemSchemeToIdentifierMap()); -} - -/// The table a write context names, read from what the context carries: a format table the -/// builder was handed, or the table - format or managed - the schema under the context's root -/// path says is there. -Result> TableOf(const WriteContext& ctx) { - // A table the caller already loaded says what it is, so nothing is read to find out. - if (ctx.GetFormatTable() != nullptr) { - return std::shared_ptr
(ctx.GetFormatTable()); - } - PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx.GetCatalog())); - std::shared_ptr specific_fs = ctx.GetSpecificFileSystem(); - if (specific_fs == nullptr && ctx.GetFileSystemSchemeToIdentifierMap().empty() && - ctx.GetCatalog() != nullptr) { - if (!ctx.GetIdentifier()) { - return Status::Invalid("a catalog write requires a table identifier"); - } - // A catalog issuing per-table temporary credentials only hands them out through - // GetTableFileSystem, so the write uses the credentials of this table. - PAIMON_ASSIGN_OR_RAISE(specific_fs, - ctx.GetCatalog()->GetTableFileSystem(ctx.GetIdentifier().value())); - } - PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, - CoreOptions::FromMap(ctx.GetOptions(), specific_fs, - ctx.GetFileSystemSchemeToIdentifierMap())); - std::string branch = ctx.GetBranch(); - PAIMON_ASSIGN_OR_RAISE(std::optional catalog_identifier, - MakeBranchCatalogIdentifier(ctx.GetIdentifier(), branch)); - // Only the main branch may have a current schema the catalog alone holds. A branch publishes - // its own under `branch/branch-`, which is where the load below and a read of the branch - // both read it, so a catalog answering for no branch schema does not fail a write to one. - PAIMON_ASSIGN_OR_RAISE( - std::optional catalog_table_schema, - CatalogMainBranchSchemaJson(ctx.GetCatalog(), catalog_identifier, branch, "write")); - // A format table writes plain data files into a directory, so it never reaches the manifest - // path below. - return ManagedTableOf(tmp_options.GetFileSystem(), specific_fs, ctx.GetRootPath(), branch, - ctx.GetOptions(), catalog_table_schema, ctx.GetIdentifier(), /*uuid=*/"", - ctx.GetFileSystemSchemeToIdentifierMap()); -} - -/// The table a commit context names, read from what the context carries: a format table the -/// builder was handed, or the table - format or managed - the schema under the context's root -/// path says is there. -Result> TableOf(const CommitContext& ctx) { - // A table the caller already loaded says what it is, so nothing is read to find out. - if (ctx.GetFormatTable() != nullptr) { - return std::shared_ptr
(ctx.GetFormatTable()); - } - PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx.GetCatalog())); - // Use catalog credentials unless the caller supplied a file system. - std::shared_ptr specific_fs = ctx.GetSpecificFileSystem(); - if (specific_fs == nullptr && ctx.GetCatalog() != nullptr) { - if (!ctx.GetIdentifier()) { - return Status::Invalid("a catalog commit requires a table identifier"); - } - // Use the credentials of this table, which a catalog issuing per-table temporary - // ones only hands out through GetTableFileSystem. - PAIMON_ASSIGN_OR_RAISE(specific_fs, - ctx.GetCatalog()->GetTableFileSystem(ctx.GetIdentifier().value())); - } - PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, - CoreOptions::FromMap(ctx.GetOptions(), specific_fs)); - const std::string& root_path = ctx.GetRootPath(); - // The branch the context was built for, already validated by `CommitContextBuilder`. - PAIMON_ASSIGN_OR_RAISE(std::string branch, BranchManager::ResolveBranch( - ctx.GetIdentifier(), ctx.GetOptions(), - /*explicit_branch=*/std::nullopt, "commit")); - PAIMON_ASSIGN_OR_RAISE(std::optional catalog_identifier, - MakeBranchCatalogIdentifier(ctx.GetIdentifier(), branch)); - // Catalog-managed schemas may be absent from the table directory, but only on the main branch: - // a branch publishes its own under `branch/branch-`, which is where the load below and a - // read of the branch both read it, so a catalog answering for no branch schema does not fail a - // commit to one. - PAIMON_ASSIGN_OR_RAISE( - std::optional catalog_table_schema, - CatalogMainBranchSchemaJson(ctx.GetCatalog(), catalog_identifier, branch, "commit")); - // A format table commits by renaming files into place, so it never reaches the snapshot path - // below. The managed path here reads the commit's branch, so this reads the same one: the two - // must not dispatch on different schemas. - return ManagedTableOf(tmp_options.GetFileSystem(), specific_fs, root_path, branch, - ctx.GetOptions(), catalog_table_schema, ctx.GetIdentifier(), - ctx.GetTableId().value_or("")); -} - -/// The options a context carries against the ones the table was loaded with. A table serves the -/// options it was loaded with, so an option the context contradicts is refused rather than -/// honoured - a caller that means to change them copies the table. A table a load hands back -/// carries the context's own options merged in, so it agrees with the context by construction; -/// the refusal belongs to a table the caller held already, whose load is over. A key the table -/// was not loaded with is no contradiction: it names a runtime setting - a scan mode, a batch -/// size - which is none of the table's business, and is served as given. -Status CheckOptionsAgreeWithTable(const TableSchema& table_schema, - const std::map& context_options) { - const std::map& table_options = table_schema.Options(); - for (const auto& [key, value] : context_options) { - auto table_option = table_options.find(key); - if (table_option != table_options.end() && table_option->second != value) { - // A value may be a credential, and the error travels into logs: it is redacted the - // way every surface that exposes options redacts them. - return Status::Invalid( - fmt::format("option '{}' is '{}' in the context but '{}' on the table", key, - SensitiveConfigUtils::RedactValue(key, value), - SensitiveConfigUtils::RedactValue(key, table_option->second))); - } - } - return Status::OK(); -} - -/// The checks every operator entry point opens with: a context, a memory pool and an executor -/// must all be there for the operator to be built at all. `kind` names the operator the first -/// message speaks of, which is all that sets the entry points' messages apart. -template -Status CheckOperatorContext(const ContextPointer& context, const char* kind) { - if (context == nullptr) { - return Status::Invalid(fmt::format("{} context is null pointer", kind)); - } - if (context->GetMemoryPool() == nullptr) { - return Status::Invalid("memory pool is null pointer"); - } - if (context->GetExecutor() == nullptr) { - return Status::Invalid("executor is null pointer"); - } - return Status::OK(); -} - -} // namespace - -Table::Table(const std::shared_ptr& file_system, const std::string& root_path, - const std::shared_ptr& schema, const Identifier& identifier, - const std::string& uuid) - : file_system_(file_system), - root_path_(root_path), - schema_(schema), - identifier_(identifier), - uuid_(uuid) {} - FileStoreTable::FileStoreTable(const std::shared_ptr& store, const Identifier& identifier, const std::string& uuid) - : Table(store->file_system(), store->root_path(), store->schema(), identifier, uuid), + : Table(store->GetFileSystem(), store->GetRootPath(), store->GetSchema(), identifier, uuid), store_(store) {} Result> FileStoreTable::Create( @@ -581,12 +108,12 @@ Result> FileStoreTable::Create( const std::shared_ptr& schema, const Identifier& identifier, const std::string& uuid, const std::string& branch, const std::shared_ptr& schema_manager, const std::shared_ptr& snapshot_manager, - const std::map& fs_scheme_map) { - // The kind the schema's primary keys name, which no schema change can add or drop, so the - // kind the table is built as is the kind every schema its root path ever held describes. A - // caller without a schema gets the append-only kind, whose operators then report that no - // schema was found. A schema that is not a table schema - the read-only one a system table - // exposes, say - is none to serve a managed table through, so it is refused rather than cast. + const std::map& fs_scheme_map, + const std::map& dynamic_options) { + // The kind the schema's primary keys name, which no schema change can add or drop, so the kind + // the table is built as is the kind every schema its root path ever held describes. A caller + // without a schema gets the append-only kind, whose operators report that none was found; a + // schema that is not a table schema - a system table's read-only one - is refused, not cast. std::shared_ptr table_schema; if (schema != nullptr) { table_schema = std::dynamic_pointer_cast(schema); @@ -595,15 +122,18 @@ Result> FileStoreTable::Create( fmt::format("{} is not a managed table schema", identifier.ToString())); } } - // The store parses the schema's options, so a schema whose options do not parse is refused - // here: the table it names cannot be served, not by one operator more than another. + // The store parses the schema's options, so one that does not parse is refused here. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr store, FileStore::Create(file_system, root_path, table_schema, branch, schema_manager, snapshot_manager, fs_scheme_map)); + std::shared_ptr table; if (table_schema != nullptr && !table_schema->PrimaryKeys().empty()) { - return std::make_shared(store, identifier, uuid); + table = std::make_shared(store, identifier, uuid); + } else { + table = std::make_shared(store, identifier, uuid); } - return std::make_shared(store, identifier, uuid); + table->dynamic_options_ = dynamic_options; + return table; } Result> FileStoreTable::Copy( @@ -613,69 +143,80 @@ Result> FileStoreTable::Copy( return Status::Invalid(fmt::format("not found latest schema of {}", FullName())); } // `type` is structural and read from the schema alone, the promise `Copy` makes: an option - // given at the call must not restyle the table. `branch` is the same kind of promise: the - // copy is the table it was copied from, on the branch that table serves, so a branch option - // given at the call moves neither - it would only claim a branch the copy does not serve - - // and a caller that means to move it has `SwitchToBranch`. Both are dropped before the - // merge, so the copy's options never speak of a branch its operators do not. + // given at the call must not restyle the table. `branch` is the same promise - the copy serves + // the branch this table does, and a caller that means to move has `SwitchToBranch`. Both are + // dropped before the merge, so the copy's options never speak of a branch it does not serve. std::map overridable = dynamic_options; overridable.erase(Options::TYPE); overridable.erase(Options::BRANCH); - // The physical layout the data on the path was written by is as immovable as the style or - // the branch: the copy serves the same data, so it is refused an option that would lay new - // data out differently than the data it serves was laid out. - PAIMON_RETURN_NOT_OK(CheckImmutableOptions(*loaded_schema, overridable)); + // The physical layout the data on the path was written by is as immovable as the style or the + // branch: the copy serves the same data. + PAIMON_RETURN_NOT_OK(SchemaValidation::CheckImmutableOptions(*loaded_schema, overridable)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merged_schema, loaded_schema->MergeOptions(overridable)); - // The managers are handed over, not rebuilt, when the copy resolves the same file system - // this table does: its operators then read the schema cache this table already warmed. The - // file system is the one the caller opened this table through, not the one it resolved to: - // the copy resolves from its own merged options again, so a file-system option among the - // ones it was given speaks rather than being pinned to this table's resolution - and when - // one does change the resolution, fresh managers are built over it instead, so the metadata - // is read through the same backend the data is. The scheme map travels the same way, so the - // copy resolves custom schemes as this table does. + // The managers are handed over, not rebuilt, when the copy resolves the same file system this + // table does: its operators then read the schema cache this table already warmed. The copy + // resolves from the file system the caller opened this table through, not the one it resolved + // to, so a file-system option it was given speaks; when one does change the resolution, fresh + // managers are built over it instead. The scheme map travels the same way. const bool fs_resolution_may_change = - FileSystemResolutionMayChange(*loaded_schema, overridable); - return FileStoreTable::Create(store_->specified_file_system(), root_path_, + FileStore::FileSystemResolutionMayChange(*loaded_schema, overridable); + // The dynamic options the copy was given join the ones this table was loaded with - which a + // copy of a copy carries the same way - so a read's fallback reader serves what the copy's own + // reader does however many copies deep. The branch was dropped above, so none accumulates, and + // an option given again replaces the one carried, as the merge above does. + std::map accumulated = dynamic_options_; + for (const auto& [key, value] : overridable) { + accumulated.insert_or_assign(key, value); + } + return FileStoreTable::Create(store_->GetSpecifiedFileSystem(), root_path_, std::shared_ptr(std::move(merged_schema)), identifier_, CatalogUuid().value_or(""), CurrentBranch(), fs_resolution_may_change ? nullptr : GetSchemaManager(), fs_resolution_may_change ? nullptr : GetSnapshotManager(), - store_->fs_scheme_map()); + store_->GetFsSchemeMap(), accumulated); } Result> FileStoreTable::SwitchToBranch(const std::string& branch) const { const std::string target_branch = BranchManager::NormalizeBranch(branch); if (target_branch == CurrentBranch()) { - // The table already serves this branch: a copy of it - schema and managers shared - is - // the same table to the caller. - return FileStoreTable::Create(store_->specified_file_system(), root_path_, schema_, + // The table already serves this branch, so a copy of it is the same table to the caller. + return FileStoreTable::Create(store_->GetSpecifiedFileSystem(), root_path_, schema_, identifier_, CatalogUuid().value_or(""), CurrentBranch(), GetSchemaManager(), GetSnapshotManager(), - store_->fs_scheme_map()); + store_->GetFsSchemeMap(), dynamic_options_); } - // The schema of the branch switched to, read from its own directory, which a branch without - // one does not have. + // The schema of the branch switched to, read from its own directory. auto schema_manager = std::make_shared(file_system_, root_path_, target_branch); PAIMON_ASSIGN_OR_RAISE(std::optional> latest, schema_manager->Latest()); if (!latest) { return Status::Invalid(fmt::format("branch {} does not exist", target_branch)); } - // A branch other than main is named in the options the schema carries - the way a read - // context names it - so what reads through them serves the branch the managers point at. + // A branch other than main is named in the options the schema carries - the way a read context + // names it - so what reads through them serves the branch the managers point at. std::map overridable; if (target_branch != BranchManager::DEFAULT_MAIN_BRANCH) { overridable[Options::BRANCH] = target_branch; } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr branch_schema, (*latest)->MergeOptions(overridable)); - return FileStoreTable::Create(store_->specified_file_system(), root_path_, + // The branch's schema may name another file system than this table's options resolved - a + // recreation that changed the access configuration - and a manager kept through that + // disagreement would read the branch's metadata through the backend this table left, so the + // branch builds its own. The schema is in hand already, so only the cache is lost. + const std::shared_ptr branch_schema_manager = + (store_->GetSpecifiedFileSystem() == nullptr && + (!store_->GetFsSchemeMap().empty() || + !FileStore::FileSystemResolutionAgrees( + branch_schema->Options(), std::static_pointer_cast(schema_)->Options()))) + ? nullptr + : schema_manager; + return FileStoreTable::Create(store_->GetSpecifiedFileSystem(), root_path_, std::shared_ptr(std::move(branch_schema)), identifier_, CatalogUuid().value_or(""), target_branch, - schema_manager, /*snapshot_manager=*/nullptr, - store_->fs_scheme_map()); + branch_schema_manager, /*snapshot_manager=*/nullptr, + store_->GetFsSchemeMap()); } Status FileStoreTable::RestoreRealtimeCommittedProgress( @@ -696,6 +237,22 @@ Status FileStoreTable::RestoreRealtimeCommittedProgress( return Status::OK(); } +Status FileStoreTable::CheckOptionsAgreeWithTable( + const TableSchema& table_schema, const std::map& context_options) { + const std::map& table_options = table_schema.Options(); + for (const auto& [key, value] : context_options) { + auto table_option = table_options.find(key); + if (table_option != table_options.end() && table_option->second != value) { + // A value may be a credential and the error travels into logs, so it is redacted. + return Status::Invalid( + fmt::format("option '{}' is '{}' in the context but '{}' on the table", key, + SensitiveConfigUtils::RedactValue(key, value), + SensitiveConfigUtils::RedactValue(key, table_option->second))); + } + } + return Status::OK(); +} + std::unique_ptr FileStoreTable::WithTablePath(std::unique_ptr&& scan, const std::string& root_path) { scan->WithTablePath(root_path); @@ -704,32 +261,24 @@ std::unique_ptr FileStoreTable::WithTablePath(std::unique_ptr> FileStoreTable::NewWrite( std::unique_ptr ctx) const { - PAIMON_RETURN_NOT_OK(CheckOperatorContext(ctx, "write")); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "write")); PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog())); std::shared_ptr specific_fs = ctx->GetSpecificFileSystem(); - // A catalog write names its table; the file system itself comes from the table, which its - // catalog's credentials already opened. + // A catalog write names its table; the file system comes from the table's own credentials. if (specific_fs == nullptr && ctx->GetFileSystemSchemeToIdentifierMap().empty() && ctx->GetCatalog() != nullptr && !ctx->GetIdentifier()) { return Status::Invalid("a catalog write requires a table identifier"); } - // The write goes to the branch the table serves: a table is the branch it was loaded for, - // so the context's own branch is not consulted. + // The write goes to the branch the table serves, not the context's own. PAIMON_ASSIGN_OR_RAISE(std::optional catalog_identifier, - MakeBranchCatalogIdentifier(ctx->GetIdentifier(), CurrentBranch())); - // The schema the table was constructed with, so a write serves the table as it was loaded - // rather than reading the schema again. - const std::shared_ptr& schema = store_->schema(); + CatalogUtils::BranchIdentifier(ctx->GetIdentifier(), CurrentBranch())); + const std::shared_ptr& schema = store_->GetSchema(); if (schema == nullptr) { return Status::Invalid( fmt::format("not found latest schema in branch {}", CurrentBranch())); } - // The options the store was built with are the ones this write serves: an option the - // context contradicts is refused rather than honoured, and a key the table was not loaded - // with names a runtime setting the write does not take - `Copy` derives the table that one - // belongs to. PAIMON_RETURN_NOT_OK(CheckOptionsAgreeWithTable(*schema, ctx->GetOptions())); - CoreOptions options = store_->options(); + CoreOptions options = store_->GetOptions(); auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(schema->Fields()); PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*schema, options)); PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateVectorStore(*schema, options)); @@ -768,64 +317,15 @@ Result> FileStoreTable::NewWrite( } Result> FileStoreWrite::Create(std::unique_ptr ctx) { - PAIMON_RETURN_NOT_OK(CheckOperatorContext(ctx, "write")); - // First the table the context names, then the context handed to that table's `NewWrite`. - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, TableOf(*ctx)); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "write")); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, FileStoreTableFactory::Create(*ctx)); return table->NewWrite(std::move(ctx)); } -namespace { - -Status ValidateRealtimeScan(const TableSchema& table_schema, const CoreOptions& core_options, - const ScanContext& context, bool read_optimized) { - if (!context.GetRealtimeContext()) { - return Status::OK(); - } - if (!core_options.RealtimeEnabled()) { - return Status::Invalid("real-time scan requires realtime.enabled=true"); - } - if (core_options.GetBucket() <= 0) { - return Status::Invalid("real-time union read requires fixed bucket mode"); - } - if (core_options.DataEvolutionEnabled()) { - return Status::Invalid("real-time union read does not support data evolution"); - } - if (!table_schema.PrimaryKeys().empty()) { - if (read_optimized) { - return Status::NotImplemented( - "PK real-time union read does not support read-optimized scans"); - } - PAIMON_RETURN_NOT_OK( - PrimaryKeyTableUtils::ValidateRealtimeOptions(core_options, table_schema)); - } - if (context.IsStreamingMode()) { - return Status::Invalid("real-time union read currently supports batch scans only"); - } - if (context.GetLimit()) { - return Status::Invalid("real-time union read does not support scan limit pushdown"); - } - if (context.GetGlobalIndexResult()) { - return Status::Invalid("real-time union read does not support global index splits"); - } - StartupMode startup_mode = core_options.GetStartupMode(); - const bool specified_snapshot = - startup_mode == StartupMode::FromSnapshot() && core_options.GetScanSnapshotId().has_value(); - if (!(startup_mode == StartupMode::LatestFull() || startup_mode == StartupMode::Latest() || - specified_snapshot)) { - return Status::Invalid( - "real-time union read requires the latest snapshot or an explicit snapshot id"); - } - return Status::OK(); -} - -} // namespace - Result> FileStoreTable::NewScan( std::unique_ptr context) const { - PAIMON_RETURN_NOT_OK(CheckOperatorContext(context, "scan")); - // The schema the table was constructed with, so a scan serves the table as it was loaded - // rather than reading the schema again. - const std::shared_ptr& table_schema = store_->schema(); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "scan")); + const std::shared_ptr& table_schema = store_->GetSchema(); if (table_schema == nullptr) { return Status::Invalid("not found latest schema"); } @@ -833,20 +333,15 @@ Result> FileStoreTable::NewScan( PAIMON_ASSIGN_OR_RAISE(bool read_optimized, OptionsUtils::GetValueFromMap(context->GetOptions(), kReadOptimizedScanOption, false)); - // The options the store was built with are the ones this scan serves: an option the context - // contradicts is refused above, and a key the table was not loaded with names a runtime - // setting the scan does not take - `Copy` derives the table that one belongs to. The branch - // in them is the table's, not the context's: a table is the branch it was loaded for, so a - // branch the context carries is not honoured. - CoreOptions core_options = store_->options(); - // The cache is a runtime setting of this scan, not of the table: the copy the scan serves - // carries it. + // The branch in the store's options is the table's, so one the context carries is not honoured. + CoreOptions core_options = store_->GetOptions(); + // The cache is a runtime setting of this scan, not of the table. core_options.WithCache(context->GetCache()); PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*table_schema, core_options)); PAIMON_RETURN_NOT_OK( - ValidateRealtimeScan(*table_schema, core_options, *context, read_optimized)); + RealtimeTableScan::ValidateScan(*table_schema, core_options, *context, read_optimized)); // validate options if (core_options.GetBucket() == -1) { if (!table_schema->PrimaryKeys().empty()) { @@ -881,7 +376,6 @@ Result> FileStoreTable::NewScan( store_->NewManifestFile(core_options, path_factory, partition_schema, context->GetMemoryPool())); - // The managers the table carries, whose schema cache the dispatch already warmed. PAIMON_ASSIGN_OR_RAISE( std::shared_ptr file_store_scan, NewFileStoreScan(manifest_list, manifest_file, arrow_schema, core_options, @@ -931,62 +425,11 @@ Result> FileStoreTable::NewScan( } Result> TableScan::Create(std::unique_ptr context) { - PAIMON_RETURN_NOT_OK(CheckOperatorContext(context, "scan")); - // First the table the context names, then the context handed to that table's `NewScan`. - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, TableOf(*context)); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "scan")); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, FileStoreTableFactory::Create(*context)); return table->NewScan(std::move(context)); } -namespace { - -/// The read context of a fallback branch, whose schema is not the table's own: it is read -/// through a manager of that branch, and the options the read serves are the ones that schema -/// was loaded with - the same way the table's own read serves the store's - with the branch it -/// was read from in them. A branch the context carries is not honoured here. -/// @param file_system The file system the table was opened with, which the fallback reads -/// through in place of one resolved from the context. -/// @param root_path The root path the table was opened with, which the fallback reads from in -/// place of the one named by the context. -Result> CreateFallbackReadContext( - const std::shared_ptr& context, const std::string& branch, - const std::shared_ptr& file_system, const std::string& root_path) { - std::shared_ptr table_schema; - const auto& specific_table_schema = context->GetSpecificTableSchema(); - if (branch == BranchManager::DEFAULT_MAIN_BRANCH && specific_table_schema) { - PAIMON_ASSIGN_OR_RAISE(table_schema, - TableSchema::CreateFromJson(specific_table_schema.value())); - } else { - auto schema_manager = std::make_shared(file_system, root_path, branch); - PAIMON_ASSIGN_OR_RAISE(std::optional> latest_schema, - schema_manager->Latest()); - if (!latest_schema) { - return Status::Invalid( - fmt::format("schema file not found in path {}, branch {}", root_path, branch)); - } - table_schema = latest_schema.value(); - } - assert(table_schema); - auto options = table_schema->Options(); - // The dynamic options the load merged into the table this read serves - a sequence-number - // flag, a batch size - speak for the read, and the fallback branch serves the same read: - // they are merged over the fallback branch's own schema options the way the load merged - // them over the table's, so the fallback reader recognises what the main one does. `type` - // stays the branch's own: a copy never restyles the table, and neither does a fallback. - for (const auto& [key, value] : context->GetOptions()) { - if (key != Options::TYPE) { - options[key] = value; - } - } - if (branch != BranchManager::DEFAULT_MAIN_BRANCH) { - options[Options::BRANCH] = branch; - } else { - options.erase(Options::BRANCH); - } - return InternalReadContext::Create(context, table_schema, options, file_system, &root_path); -} - -} // namespace - Result> FileStoreTable::CreateTableRead( const std::shared_ptr& internal_context, const std::shared_ptr& memory_pool, @@ -1008,29 +451,29 @@ Result> FileStoreTable::CreateTableRead( std::shared_ptr path_factory, store_->NewPathFactory(arrow_schema, core_options, core_options.GetFileFormat()->Identifier(), memory_pool)); - return NewTableRead(path_factory, internal_context, memory_pool, executor); + // The reader is selected by the schema the read serves, not by the table's kind: an + // append-only table can have a primary-key fallback branch and vice versa, so a reader typed + // after the main table would fail the fallback's splits. + if (internal_context->GetPrimaryKeys().empty()) { + return std::make_unique(path_factory, internal_context, memory_pool, + executor); + } + return KeyValueTableRead::Create(path_factory, internal_context, memory_pool, executor); } Result> FileStoreTable::NewRead( std::unique_ptr context) const { std::shared_ptr read_context = std::move(context); - PAIMON_RETURN_NOT_OK(CheckOperatorContext(read_context, "read")); - // The schema the table was constructed with, so a read serves the table as it was loaded - // rather than reading the schema again. Its options are the ones the read serves: an option - // the context would have changed is refused rather than honoured, and a key the table was - // not loaded with names a runtime setting the read does not take - `Copy` derives the table - // that one belongs to. - const std::shared_ptr& loaded_schema = store_->schema(); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(read_context, "read")); + const std::shared_ptr& loaded_schema = store_->GetSchema(); if (loaded_schema == nullptr) { return Status::Invalid("not found latest schema"); } PAIMON_RETURN_NOT_OK(CheckOptionsAgreeWithTable(*loaded_schema, read_context->GetOptions())); - // The options the store was built with, which the read reads through on the branch the - // table serves: a table is the branch it was loaded for, so the context's own branch is - // not consulted. - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr internal_context, - InternalReadContext::Create(read_context, loaded_schema, store_->options(), &root_path_)); + // The read reads through the branch the table serves, not the context's own. + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr internal_context, + InternalReadContext::Create(read_context, loaded_schema, + store_->GetOptions(), &root_path_)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, CreateTableRead(internal_context, read_context->GetMemoryPool(), @@ -1043,12 +486,11 @@ Result> FileStoreTable::NewRead( return std::move(table_read); } - // The fallback branch is not the table's, so its schema is read through a manager of its - // own rather than the table's. - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr fallback_context, - CreateFallbackReadContext(read_context, /*branch=*/scan_fallback_branch.value(), - file_system_, root_path_)); + // The fallback branch is not the table's, so its schema is read through its own manager. + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr fallback_context, + InternalReadContext::CreateForFallbackBranch( + read_context, /*branch=*/scan_fallback_branch.value(), file_system_, + root_path_, dynamic_options_)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr fallback_table_read, CreateTableRead(fallback_context, read_context->GetMemoryPool(), @@ -1059,9 +501,8 @@ Result> FileStoreTable::NewRead( } Result> TableRead::Create(std::unique_ptr ctx) { - PAIMON_RETURN_NOT_OK(CheckOperatorContext(ctx, "read")); - // First the table the context names, then the context handed to that table's `NewRead`. - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, TableOf(*ctx)); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "read")); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, FileStoreTableFactory::Create(*ctx)); return table->NewRead(std::move(ctx)); } @@ -1111,19 +552,16 @@ Result> NewSnapshotCommit( Result> FileStoreTable::NewCommit( std::unique_ptr ctx) const { - PAIMON_RETURN_NOT_OK(CheckOperatorContext(ctx, "commit")); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "commit")); PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog())); std::shared_ptr specific_fs = ctx->GetSpecificFileSystem(); - // A catalog commit names its table; the file system itself comes from the table, which its - // catalog's credentials already opened. + // A catalog commit names its table; the file system comes from the table's own credentials. if (specific_fs == nullptr && ctx->GetCatalog() != nullptr && !ctx->GetIdentifier()) { return Status::Invalid("a catalog commit requires a table identifier"); } - // A commit commits to the root path the table was opened with and the branch the table - // serves, in place of the ones named by the context: a table is the branch it was loaded - // for, so the context's own branch is not consulted. + // A commit commits to the root path and branch the table was loaded with, not the context's. PAIMON_ASSIGN_OR_RAISE(std::optional catalog_identifier, - MakeBranchCatalogIdentifier(ctx->GetIdentifier(), CurrentBranch())); + CatalogUtils::BranchIdentifier(ctx->GetIdentifier(), CurrentBranch())); // The id a snapshot records the catalog's current schema by, read through the loader below // rather than trusted from a schema this commit will not re-read. FileStoreCommitImpl::SchemaIdLoader schema_id_loader; @@ -1145,21 +583,15 @@ Result> FileStoreTable::NewCommit( return data_schema->Id(); }; } - // The schema the table was constructed with, so a commit serves the table as it was loaded - // rather than reading the schema again. - const std::shared_ptr& schema = store_->schema(); + const std::shared_ptr& schema = store_->GetSchema(); if (schema == nullptr) { return Status::Invalid( fmt::format("not found latest schema in branch {}", CurrentBranch())); } std::shared_ptr arrow_schema = DataField::ConvertDataFieldsToArrowSchema(schema->Fields()); - // The options the store was built with are the ones this commit serves: an option the - // context contradicts is refused rather than honoured, and a key the table was not loaded - // with names a runtime setting the commit does not take - `Copy` derives the table that one - // belongs to. PAIMON_RETURN_NOT_OK(CheckOptionsAgreeWithTable(*schema, ctx->GetOptions())); - CoreOptions options = store_->options(); + CoreOptions options = store_->GetOptions(); PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*schema, options)); assert(options.GetFileSystem()); assert(options.GetFileFormat()); @@ -1167,7 +599,8 @@ Result> FileStoreTable::NewCommit( PAIMON_ASSIGN_OR_RAISE(bool is_object_store, FileSystem::IsObjectStore(root_path_)); // Catalog commits do not require atomic rename on the object store. The inte-test marker is // not a table option, so it is read from the context the commit was handed. - if (is_object_store && !ctx->UseRESTCatalogCommit() && !IsCatalogCommit(*ctx) && + if (is_object_store && !ctx->UseRESTCatalogCommit() && + AsVersionManaged(ctx->GetCatalog()) == nullptr && ctx->GetOptions().find("enable-object-store-commit-in-inte-test") == ctx->GetOptions().end()) { return Status::NotImplemented( @@ -1206,7 +639,9 @@ Result> FileStoreTable::NewCommit( ctx->GetExecutor(), ctx->GetMemoryPool()); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr snapshot_commit, - NewSnapshotCommit(*ctx, catalog_identifier, options, snapshot_manager)); + SnapshotCommit::Create(ctx->GetCatalog(), catalog_identifier, options, + ctx->GetTableId(), ctx->UseRESTCatalogCommit(), + options.GetFileSystem(), snapshot_manager)); return std::make_unique( ctx->GetMemoryPool(), ctx->GetExecutor(), arrow_schema, root_path_, ctx->GetCommitUser(), @@ -1218,9 +653,8 @@ Result> FileStoreTable::NewCommit( Result> FileStoreCommit::Create( std::unique_ptr ctx) { - PAIMON_RETURN_NOT_OK(CheckOperatorContext(ctx, "commit")); - // First the table the context names, then the context handed to that table's `NewCommit`. - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, TableOf(*ctx)); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "commit")); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, FileStoreTableFactory::Create(*ctx)); return table->NewCommit(std::move(ctx)); } diff --git a/src/paimon/core/table/file_store_table.h b/src/paimon/core/table/file_store_table.h index df841d548..1fb307495 100644 --- a/src/paimon/core/table/file_store_table.h +++ b/src/paimon/core/table/file_store_table.h @@ -53,34 +53,25 @@ class SplitGenerator; /// decide the kind of managed table it is - `AppendOnlyFileStoreTable` or /// `PrimaryKeyFileStoreTable`, the `Create` factory below - and that kind builds the pieces the /// flows hand down: the write itself, the scans, the read and the split generator. -/// -/// Its `NewScan`/`NewRead`/`NewWrite`/`NewCommit` build the operators here, from the context plus -/// the state the table was constructed with, so they serve the table as it was loaded. The -/// context-only entry points - `TableScan::Create()`, `TableRead::Create()`, -/// `FileStoreWrite::Create()` and `FileStoreCommit::Create()` - load the table the context names -/// and hand the context to that table's `NewXxx`, which is how a context built without a table -/// reaches the same operators. class FileStoreTable : public Table { public: /// The managed table the schema describes, built as the kind its primary keys name: an /// `AppendOnlyFileStoreTable` when there are none, a `PrimaryKeyFileStoreTable` when there /// are, the way Java Paimon's `FileStoreTableFactory` does. Primary keys cannot be added or - /// dropped by a schema change, so the kind the table was built as is the kind every schema - /// its root path ever held describes. + /// dropped by a schema change, so the kind is the same for every schema the root path held. /// @param schema The schema the table was loaded with, whose primary keys name the kind and - /// whose options the operators serve. It is null when the caller had no schema, and - /// the table is built as the append-only kind - its operators then report that no - /// schema was found. + /// whose options the operators serve. Null when the caller had none, in which case the + /// table is built as the append-only kind and its operators report that none was found. /// @param branch The branch the table serves, which the managers below read and write on. - /// @param schema_manager A schema manager to carry instead of building one - the one a - /// context dispatch read the schema through, whose cache the operators then serve. - /// @param snapshot_manager A snapshot manager to carry instead of building one, the way - /// `Copy` hands its table's over so both share one. - /// A null manager is built over `root_path` on `branch`, so the operators always read - /// through the managers the table carries. + /// @param schema_manager A schema manager to carry instead of building one - see + /// `FileStore::Create`. + /// @param snapshot_manager A snapshot manager to carry instead of building one, the way `Copy` + /// hands its table's over so both share one - see `FileStore::Create`. /// @param fs_scheme_map The scheme-to-identifier map the caller's contexts resolve custom - /// schemes through, which the store's options resolve through as well - see - /// `FileStore::Create`. `Copy` hands its table's map over. + /// schemes through, which the store's options resolve through as well. `Copy` hands its + /// table's map over. + /// @param dynamic_options The options a copy was given over the ones its schema stored, which + /// `Copy` accumulates - see the member of the same name. static Result> Create( const std::shared_ptr& file_system, const std::string& root_path, const std::shared_ptr& schema, const Identifier& identifier, @@ -88,26 +79,19 @@ class FileStoreTable : public Table { const std::string& branch = BranchManager::DEFAULT_MAIN_BRANCH, const std::shared_ptr& schema_manager = nullptr, const std::shared_ptr& snapshot_manager = nullptr, - const std::map& fs_scheme_map = {}); + const std::map& fs_scheme_map = {}, + const std::map& dynamic_options = {}); - /// The branch this table serves, as resolved when the table was built: the schema the table - /// was loaded with came from it, and its managers point at it. const std::string& CurrentBranch() const { - return store_->branch(); + return store_->CurrentBranch(); } - /// The schema manager the operators read through. It carries the schema cache of `root_path` - /// on `CurrentBranch()`, which every operator serves rather than each reading the schema - /// files again. const std::shared_ptr& GetSchemaManager() const { - return store_->schema_manager(); + return store_->GetSchemaManager(); } - /// The snapshot manager a scan reads through. A write or a commit builds its own: it arms - /// the manager with a catalog snapshot loader, which a shared one would keep serving to - /// scans afterwards. const std::shared_ptr& GetSnapshotManager() const { - return store_->snapshot_manager(); + return store_->GetSnapshotManager(); } std::string Name() const override { @@ -118,17 +102,15 @@ class FileStoreTable : public Table { return identifier_.GetFullName(); } - /// Copies the table with `dynamic_options` on top of the ones its schema stored, the - /// precedence `Copy` promises. The file system, root path and identifier stay as they are: - /// they name this table, which an option does not rename. + /// The file system, root path and identifier stay as they are: they name this table, which an + /// option does not rename. Result> Copy( const std::map& dynamic_options) const override; /// Returns the table of `branch`, as Java Paimon's `switchToBranch` does. The branch switched - /// to is served as it is: its schema is read from its own directory, so the dynamic options - /// of this table are not carried over. Switching to the branch already served hands back a - /// copy of this table, schema and managers shared, and a branch with no schema does not - /// exist. + /// to is served as it is: its schema is read from its own directory, so this table's dynamic + /// options are not carried over. Switching to the branch already served hands back a copy of + /// this table, schema and managers shared, and a branch with no schema does not exist. Result> SwitchToBranch(const std::string& branch) const; Result> NewScan(std::unique_ptr context) const override; @@ -142,38 +124,43 @@ class FileStoreTable : public Table { std::unique_ptr context) const override; private: - /// The read `NewRead` hands back, from the internal context it read the schema into: the - /// path factory comes from the store - a fallback branch's schema carries the table's - /// partition keys, which no schema change can alter - and the read itself is the kind's, - /// `NewTableRead`. + /// The options a context carries against the ones the table was loaded with. A table serves + /// the options it was loaded with, so an option the context contradicts is refused rather than + /// honoured - a caller that means to change them copies the table. A table a load hands back + /// carries the context's own options merged in - `FileStoreTableFactory` does the merging - so + /// it agrees by construction; the refusal belongs to a table the caller held already, whose + /// load is over. A key the table was not loaded with is no contradiction: it names a runtime + /// setting - a scan mode, a batch size - which is none of the table's business, and is served + /// as given. + static Status CheckOptionsAgreeWithTable( + const TableSchema& table_schema, const std::map& context_options); + + /// The read `NewRead` hands back, from the internal context it read the schema into: the path + /// factory comes from the store, and the reader is selected by the schema the read serves - the + /// primary keys it carries - rather than by the table's own kind. Result> CreateTableRead( const std::shared_ptr& internal_context, const std::shared_ptr& memory_pool, const std::shared_ptr& executor) const; protected: - /// The base constructor the kinds of managed table build through, `Create` above: the store - /// the table was loaded with carries every state the kinds hand down - see `Create` for what - /// each piece of it is. + /// The base constructor the kinds of managed table build through: the store carries every state + /// the kinds hand down - see `Create` for what each piece of it is. FileStoreTable(const std::shared_ptr& store, const Identifier& identifier, const std::string& uuid); /// The committed progress a real-time write restores, from the latest snapshot the snapshot - /// manager reads: where the write picks up after, which both kinds of real-time write ask - /// for the same way. + /// manager reads: where the write picks up after, which both kinds ask for the same way. static Status RestoreRealtimeCommittedProgress( const std::shared_ptr& realtime_context, const std::shared_ptr& snapshot_manager, const CoreOptions& options); - /// The scan the table's root path names, which the kinds of file store scan read from: the - /// kinds hand their scans back through it. + /// The scan the table's root path names, which the kinds of file store scan read from. static std::unique_ptr WithTablePath(std::unique_ptr&& scan, const std::string& root_path); - /// The write of this kind of table, which `NewWrite` hands the shared flow's pieces to: the - /// kind's writer, built over the path factory, snapshot manager and options that flow - /// resolved. The schema is the store's own, the one the table was loaded with, and the - /// branch the write serves is the table's, `CurrentBranch()`. + /// The write of this kind of table, which `NewWrite` hands the shared flow's pieces to. The + /// schema and the branch the write serves are the table's own. /// @param io_manager The spill manager the context's temp directory names, or null when the /// context named none. /// @param ignore_previous_files Whether the write starts over rather than appending, which @@ -186,8 +173,7 @@ class FileStoreTable : public Table { const std::shared_ptr& io_manager, bool ignore_previous_files) const = 0; /// The scan of this kind of table, which `NewScan` hands the manifest list and file it built - /// to. The schema is the store's own and the snapshot and schema managers are the table's - /// own, which the scan serves. + /// to, over the table's own schema and managers. virtual Result> NewFileStoreScan( const std::shared_ptr& manifest_list, const std::shared_ptr& manifest_file, @@ -195,22 +181,12 @@ class FileStoreTable : public Table { const std::shared_ptr& executor, const std::shared_ptr& memory_pool, const ScanContext* context) const = 0; - /// The split generator of this kind of table, Java Paimon's `splitGenerator()`. The schema - /// it reads its kind's keys from is the store's own. + /// The split generator of this kind of table, Java Paimon's `splitGenerator()`. virtual Result> NewSplitGenerator( const CoreOptions& core_options) const = 0; - /// The read of this kind of table, which `NewRead` hands the internal context it read the - /// schema into and the path factory it built to. - virtual Result> NewTableRead( - const std::shared_ptr& path_factory, - const std::shared_ptr& internal_context, - const std::shared_ptr& memory_pool, - const std::shared_ptr& executor) const = 0; - - /// The manifest entry scan supplier of this kind of table, which `NewCommit` hands the - /// manifest list and file it built to. The schema is the store's own, and the schema manager - /// the scans it hands out read through is the table's own. + /// The manifest entry scan supplier of this kind of table, which `NewCommit` hands the manifest + /// list and file it built to. virtual CommitScanner::ScanSupplier NewCommitScanSupplier( const std::shared_ptr& snapshot_manager, const std::shared_ptr& manifest_list, @@ -219,9 +195,14 @@ class FileStoreTable : public Table { const std::shared_ptr& executor, const std::shared_ptr& pool) const = 0; - /// The storage this table serves: the file system, root path, schema, branch and managers it - /// was loaded with, and the factories that build the pieces every operator opens with. + /// The storage this table serves - see `FileStore`. std::shared_ptr store_; + + /// The dynamic options the table was loaded with over the ones its schema stored, which `Copy` + /// accumulates into the copy it derives. They stay separate from the schema they were merged + /// into because the fallback reader of a read needs them alone: it serves another branch's + /// schema, over which the same options are merged again. + std::map dynamic_options_; }; } // namespace paimon diff --git a/src/paimon/core/table/format/format_table.cpp b/src/paimon/core/table/format/format_table.cpp index cb4141dde..d4b611eac 100644 --- a/src/paimon/core/table/format/format_table.cpp +++ b/src/paimon/core/table/format/format_table.cpp @@ -39,6 +39,7 @@ #include "paimon/core/table/format/format_table_file_store_write.h" #include "paimon/core/table/format/format_table_read.h" #include "paimon/core/table/format/format_table_scan.h" +#include "paimon/core/utils/context_utils.h" #include "paimon/defs.h" #include "paimon/file_store_commit.h" #include "paimon/file_store_write.h" @@ -226,15 +227,7 @@ Result> FormatTable::GetArrowSchema() const { Result> FormatTable::NewScan( std::unique_ptr context) const { std::shared_ptr shared_context = std::move(context); - if (shared_context == nullptr) { - return Status::Invalid("scan context is null pointer"); - } - if (shared_context->GetMemoryPool() == nullptr) { - return Status::Invalid("memory pool is null pointer"); - } - if (shared_context->GetExecutor() == nullptr) { - return Status::Invalid("executor is null pointer"); - } + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(shared_context, "scan")); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, CopySelf(shared_context->GetOptions())); // Anything a `ScanContext` carries that a format table cannot honour is refused rather than @@ -280,15 +273,7 @@ Result> FormatTable::NewScan( Result> FormatTable::NewRead( std::unique_ptr context) const { std::shared_ptr read_context = std::move(context); - if (read_context == nullptr) { - return Status::Invalid("read context is null pointer"); - } - if (read_context->GetMemoryPool() == nullptr) { - return Status::Invalid("memory pool is null pointer"); - } - if (read_context->GetExecutor() == nullptr) { - return Status::Invalid("executor is null pointer"); - } + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(read_context, "read")); if (read_context->HasVectorOrFullTextSearch()) { return Status::NotImplemented( "VectorSearch or FullTextSearch is not supported for format tables"); @@ -305,15 +290,7 @@ Result> FormatTable::NewRead( Result> FormatTable::NewWrite( std::unique_ptr context) const { - if (context == nullptr) { - return Status::Invalid("write context is null pointer"); - } - if (context->GetMemoryPool() == nullptr) { - return Status::Invalid("memory pool is null pointer"); - } - if (context->GetExecutor() == nullptr) { - return Status::Invalid("executor is null pointer"); - } + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "write")); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, CopySelf(context->GetOptions())); if (context->GetCatalog() != nullptr) { return Status::Invalid( @@ -346,15 +323,7 @@ Result> FormatTable::NewWrite( Result> FormatTable::NewCommit( std::unique_ptr context) const { - if (context == nullptr) { - return Status::Invalid("commit context is null pointer"); - } - if (context->GetMemoryPool() == nullptr) { - return Status::Invalid("memory pool is null pointer"); - } - if (context->GetExecutor() == nullptr) { - return Status::Invalid("executor is null pointer"); - } + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "commit")); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr given_table, CopySelf(context->GetOptions())); if (!context->IgnoreEmptyCommit()) { diff --git a/src/paimon/core/table/primary_key_file_store_table.cpp b/src/paimon/core/table/primary_key_file_store_table.cpp index 06e27e0d1..9a5875c9f 100644 --- a/src/paimon/core/table/primary_key_file_store_table.cpp +++ b/src/paimon/core/table/primary_key_file_store_table.cpp @@ -52,7 +52,6 @@ #include "paimon/core/snapshot.h" #include "paimon/core/table/bucket_mode.h" #include "paimon/core/table/file_store.h" -#include "paimon/core/table/source/key_value_table_read.h" #include "paimon/core/table/source/merge_tree_split_generator.h" #include "paimon/core/table/source/split_generator.h" #include "paimon/core/utils/primary_key_table_utils.h" @@ -108,7 +107,7 @@ Result> PrimaryKeyFileStoreTable::CreateWrite( const std::shared_ptr& snapshot_manager, const std::shared_ptr& io_manager, bool ignore_previous_files) const { // pk table - const std::shared_ptr& schema = store_->schema(); + const std::shared_ptr& schema = store_->GetSchema(); if (ctx.GetRealtimeContext()) { PAIMON_RETURN_NOT_OK(PrimaryKeyTableUtils::ValidateRealtimeOptions(options, *schema)); if (ignore_previous_files) { @@ -228,7 +227,7 @@ Result> PrimaryKeyFileStoreTable::NewFileStoreSca const std::shared_ptr& executor, const std::shared_ptr& memory_pool, const ScanContext* context) const { // TODO(liancheng.lsz): support fallback branch in scan - const std::shared_ptr& table_schema = store_->schema(); + const std::shared_ptr& table_schema = store_->GetSchema(); std::unique_ptr scan; PAIMON_ASSIGN_OR_RAISE( scan, KeyValueFileStoreScan::Create(GetSnapshotManager(), GetSchemaManager(), manifest_list, @@ -243,7 +242,7 @@ Result> PrimaryKeyFileStoreTable::NewSplitGenera auto source_split_target_size = core_options.GetSourceSplitTargetSize(); auto source_split_open_file_cost = core_options.GetSourceSplitOpenFileCost(); // TODO(liancheng.lsz): support evolution - const std::shared_ptr& table_schema = store_->schema(); + const std::shared_ptr& table_schema = store_->GetSchema(); PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_keys, table_schema->TrimmedPrimaryKeys()); PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_pk_fields, @@ -256,14 +255,6 @@ Result> PrimaryKeyFileStoreTable::NewSplitGenera core_options.DeletionVectorsEnabled(), core_options.GetMergeEngine(), key_comparator); } -Result> PrimaryKeyFileStoreTable::NewTableRead( - const std::shared_ptr& path_factory, - const std::shared_ptr& internal_context, - const std::shared_ptr& memory_pool, - const std::shared_ptr& executor) const { - return KeyValueTableRead::Create(path_factory, internal_context, memory_pool, executor); -} - CommitScanner::ScanSupplier PrimaryKeyFileStoreTable::NewCommitScanSupplier( const std::shared_ptr& snapshot_manager, const std::shared_ptr& manifest_list, @@ -273,7 +264,7 @@ CommitScanner::ScanSupplier PrimaryKeyFileStoreTable::NewCommitScanSupplier( // The schema manager the table carries, which the scans this supplier hands out read the // schema through. const std::shared_ptr& schema_manager = GetSchemaManager(); - const std::shared_ptr& table_schema = store_->schema(); + const std::shared_ptr& table_schema = store_->GetSchema(); return [snapshot_manager, schema_manager, manifest_list, manifest_file, table_schema, arrow_schema, options, executor, pool](const std::shared_ptr& scan_filter) -> Result> { diff --git a/src/paimon/core/table/primary_key_file_store_table.h b/src/paimon/core/table/primary_key_file_store_table.h index 9b0ed5c9a..ce7547a71 100644 --- a/src/paimon/core/table/primary_key_file_store_table.h +++ b/src/paimon/core/table/primary_key_file_store_table.h @@ -63,12 +63,6 @@ class PrimaryKeyFileStoreTable : public FileStoreTable { Result> NewSplitGenerator( const CoreOptions& core_options) const override; - Result> NewTableRead( - const std::shared_ptr& path_factory, - const std::shared_ptr& internal_context, - const std::shared_ptr& memory_pool, - const std::shared_ptr& executor) const override; - CommitScanner::ScanSupplier NewCommitScanSupplier( const std::shared_ptr& snapshot_manager, const std::shared_ptr& manifest_list, diff --git a/src/paimon/core/table/source/realtime_table_scan.cpp b/src/paimon/core/table/source/realtime_table_scan.cpp index 42a7171b3..cfb61c17d 100644 --- a/src/paimon/core/table/source/realtime_table_scan.cpp +++ b/src/paimon/core/table/source/realtime_table_scan.cpp @@ -27,17 +27,21 @@ #include "fmt/format.h" #include "paimon/common/utils/scope_guard.h" +#include "paimon/core/core_options.h" #include "paimon/core/operation/commit/realtime_commit_properties.h" +#include "paimon/core/schema/table_schema.h" #include "paimon/core/snapshot.h" #include "paimon/core/table/source/data_split_impl.h" #include "paimon/core/table/source/plan_impl.h" #include "paimon/core/table/source/realtime_split.h" #include "paimon/core/utils/file_store_path_factory.h" +#include "paimon/core/utils/primary_key_table_utils.h" #include "paimon/core/utils/snapshot_manager.h" #include "paimon/realtime/realtime_context.h" #include "paimon/realtime/realtime_store.h" #include "paimon/scan_context.h" #include "paimon/status.h" +#include "paimon/table/source/startup_mode.h" namespace paimon { @@ -203,6 +207,49 @@ Result>> RealtimeTableScan::CreateRealtimeSpl return result; } +Status RealtimeTableScan::ValidateScan(const TableSchema& table_schema, + const CoreOptions& core_options, const ScanContext& context, + bool read_optimized) { + if (!context.GetRealtimeContext()) { + return Status::OK(); + } + if (!core_options.RealtimeEnabled()) { + return Status::Invalid("real-time scan requires realtime.enabled=true"); + } + if (core_options.GetBucket() <= 0) { + return Status::Invalid("real-time union read requires fixed bucket mode"); + } + if (core_options.DataEvolutionEnabled()) { + return Status::Invalid("real-time union read does not support data evolution"); + } + if (!table_schema.PrimaryKeys().empty()) { + if (read_optimized) { + return Status::NotImplemented( + "PK real-time union read does not support read-optimized scans"); + } + PAIMON_RETURN_NOT_OK( + PrimaryKeyTableUtils::ValidateRealtimeOptions(core_options, table_schema)); + } + if (context.IsStreamingMode()) { + return Status::Invalid("real-time union read currently supports batch scans only"); + } + if (context.GetLimit()) { + return Status::Invalid("real-time union read does not support scan limit pushdown"); + } + if (context.GetGlobalIndexResult()) { + return Status::Invalid("real-time union read does not support global index splits"); + } + StartupMode startup_mode = core_options.GetStartupMode(); + const bool specified_snapshot = + startup_mode == StartupMode::FromSnapshot() && core_options.GetScanSnapshotId().has_value(); + if (!(startup_mode == StartupMode::LatestFull() || startup_mode == StartupMode::Latest() || + specified_snapshot)) { + return Status::Invalid( + "real-time union read requires the latest snapshot or an explicit snapshot id"); + } + return Status::OK(); +} + RealtimeTableScan::RealtimeTableScan(std::unique_ptr&& disk_scan, bool pk_table, const std::shared_ptr& realtime_context, const std::shared_ptr& path_factory, diff --git a/src/paimon/core/table/source/realtime_table_scan.h b/src/paimon/core/table/source/realtime_table_scan.h index 821e67fd7..318319139 100644 --- a/src/paimon/core/table/source/realtime_table_scan.h +++ b/src/paimon/core/table/source/realtime_table_scan.h @@ -30,14 +30,23 @@ namespace paimon { +class CoreOptions; class FileStorePathFactory; class FileSystem; +class ScanContext; class ScanFilter; class SnapshotManager; +class TableSchema; /// Adds process-local memory splits to a normal data-table batch scan. class RealtimeTableScan : public TableScan { public: + /// The checks a scan that asks for a real-time union read opens with, run before one of these + /// is built: the table has to be able to serve it, and the options have to name a scan one + /// can be served over. A context that asks for none of it passes straight through. + static Status ValidateScan(const TableSchema& table_schema, const CoreOptions& core_options, + const ScanContext& context, bool read_optimized); + RealtimeTableScan(std::unique_ptr&& disk_scan, bool pk_table, const std::shared_ptr& realtime_context, const std::shared_ptr& path_factory, diff --git a/src/paimon/core/table/system/audit_log_system_table.cpp b/src/paimon/core/table/system/audit_log_system_table.cpp index e191b653a..09fd6328f 100644 --- a/src/paimon/core/table/system/audit_log_system_table.cpp +++ b/src/paimon/core/table/system/audit_log_system_table.cpp @@ -44,6 +44,7 @@ #include "paimon/core/schema/table_schema.h" #include "paimon/core/table/source/data_split_impl.h" #include "paimon/core/table/source/key_value_table_read.h" +#include "paimon/core/utils/context_utils.h" #include "paimon/defs.h" #include "paimon/read_context.h" #include "paimon/scan_context.h" @@ -386,6 +387,7 @@ Result> AuditLogSystemTable::ArrowSchema() const Result> AuditLogSystemTable::NewScan( std::unique_ptr ctx) const { const std::shared_ptr context = std::move(ctx); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "scan")); if (context->GetScanFilters() && context->GetScanFilters()->GetPredicate()) { return Status::NotImplemented("audit_log system table predicate pushdown is not supported"); } @@ -395,9 +397,7 @@ Result> AuditLogSystemTable::NewScan( .WithStreamingMode(context->IsStreamingMode()) .WithMemoryPool(context->GetMemoryPool()) .WithExecutor(context->GetExecutor()) - // The data table behind this view is served through the file system the view itself was - // opened with, so a context without one of its own falls back to it rather than the - // default backend. + // The view's own file system when the context names none, rather than the default backend. .WithFileSystem(context->GetSpecificFileSystem() != nullptr ? context->GetSpecificFileSystem() : GetFileSystem()) @@ -420,6 +420,8 @@ Result> AuditLogSystemTable::NewScan( Result> AuditLogSystemTable::NewRead( std::unique_ptr ctx) const { const std::shared_ptr context = std::move(ctx); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "read")); + PAIMON_RETURN_NOT_OK(CheckNoSearch(context)); return NewChangelogRead(context, std::make_shared()); } @@ -458,10 +460,8 @@ Result> AuditLogSystemTable::CreateDataReadContext( PAIMON_ASSIGN_OR_RAISE(std::shared_ptr base_read_schema, BaseReadSchema()); PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(read_options)); ReadContextBuilder builder(table_path_); - // The data table behind this view is served through the file system the view itself was - // opened with, but only when the context names neither a file system of its own nor a - // scheme map to resolve through - a file system and a scheme map cannot both be named, and - // the context's own answer always wins. + // The view's own file system, but only when the context names neither one of its own nor a + // scheme map to resolve through - a file system and a scheme map cannot both be named. std::shared_ptr data_table_fs = context->GetSpecificFileSystem(); if (data_table_fs == nullptr && context->GetFileSystemSchemeToIdentifierMap().empty()) { data_table_fs = GetFileSystem(); diff --git a/src/paimon/core/table/system/binlog_system_table.cpp b/src/paimon/core/table/system/binlog_system_table.cpp index 8a821d45f..a3c39078d 100644 --- a/src/paimon/core/table/system/binlog_system_table.cpp +++ b/src/paimon/core/table/system/binlog_system_table.cpp @@ -31,6 +31,7 @@ #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/core/core_options.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/utils/context_utils.h" #include "paimon/defs.h" #include "paimon/read_context.h" #include "paimon/table/source/table_read.h" @@ -100,6 +101,8 @@ Result> BinlogSystemTable::ArrowSchema() const { Result> BinlogSystemTable::NewRead( std::unique_ptr ctx) const { const std::shared_ptr context = std::move(ctx); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "read")); + PAIMON_RETURN_NOT_OK(CheckNoSearch(context)); return NewChangelogRead(context, CreateBinlogBatchConverter()); } diff --git a/src/paimon/core/table/system/global_system_tables.h b/src/paimon/core/table/system/global_system_tables.h index 3d6936262..eadc83af1 100644 --- a/src/paimon/core/table/system/global_system_tables.h +++ b/src/paimon/core/table/system/global_system_tables.h @@ -33,7 +33,14 @@ class FileSystem; /// Context passed to global system table constructors, providing catalog-level /// access for enumerating databases, tables, and reading metadata. struct GlobalSystemTableContext { - const Catalog* catalog = nullptr; // non-owning pointer + /// The catalog the table reads its rows through, non-owning: the rows are built lazily, + /// so a table that outlives the call that built it must be handed the shared ownership + /// below too, or this pointer dangles at the first `BuildRows`. + const Catalog* catalog = nullptr; + /// The shared ownership a returned table keeps of its catalog, so its lazy `BuildRows` + /// cannot read through one already destroyed. Empty where the table is transient - used + /// within the call, before the catalog can go away. + std::shared_ptr catalog_retainer; std::shared_ptr fs; std::string warehouse; std::map catalog_options; diff --git a/src/paimon/core/table/system/in_memory_system_table.cpp b/src/paimon/core/table/system/in_memory_system_table.cpp index 773f621b8..d21d6fb4d 100644 --- a/src/paimon/core/table/system/in_memory_system_table.cpp +++ b/src/paimon/core/table/system/in_memory_system_table.cpp @@ -29,6 +29,7 @@ #include "paimon/common/utils/checked_cast.h" #include "paimon/core/io/generic_row_to_arrow_array_converter.h" #include "paimon/core/table/system/system_table_scan.h" +#include "paimon/core/utils/context_utils.h" #include "paimon/memory/memory_pool.h" #include "paimon/read_context.h" #include "paimon/scan_context.h" @@ -117,6 +118,10 @@ Result> InMemorySystemTable::NewScan( Result> InMemorySystemTable::NewRead( std::unique_ptr ctx) const { const std::shared_ptr context = std::move(ctx); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "read")); + // The rows this view serves are its own, read straight out of memory: there is no search + // over them, so a search the caller asked for would be quietly answered with plain rows. + PAIMON_RETURN_NOT_OK(CheckNoSearch(context)); return std::make_unique( checked_pointer_cast(shared_from_this()), context->GetMemoryPool()); diff --git a/src/paimon/core/table/system/read_optimized_system_table.cpp b/src/paimon/core/table/system/read_optimized_system_table.cpp index d7193b5cb..65a48c048 100644 --- a/src/paimon/core/table/system/read_optimized_system_table.cpp +++ b/src/paimon/core/table/system/read_optimized_system_table.cpp @@ -27,6 +27,7 @@ #include "paimon/common/types/data_field.h" #include "paimon/core/schema/table_schema.h" #include "paimon/core/table/source/read_optimized_scan_options.h" +#include "paimon/core/utils/context_utils.h" #include "paimon/defs.h" #include "paimon/read_context.h" #include "paimon/scan_context.h" @@ -61,6 +62,7 @@ std::map ReadOptimizedSystemTable::ReadOptimizedOption Result> ReadOptimizedSystemTable::NewScan( std::unique_ptr ctx) const { const std::shared_ptr context = std::move(ctx); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "scan")); if (context->GetRealtimeContext() && !table_schema_->PrimaryKeys().empty()) { return Status::NotImplemented( "PK real-time union read does not support read-optimized scans"); @@ -71,9 +73,7 @@ Result> ReadOptimizedSystemTable::NewScan( .WithStreamingMode(context->IsStreamingMode()) .WithMemoryPool(context->GetMemoryPool()) .WithExecutor(context->GetExecutor()) - // The data table behind this view is served through the file system the view itself was - // opened with, so a context without one of its own falls back to it rather than the - // default backend. + // The view's own file system when the context names none, rather than the default backend. .WithFileSystem(context->GetSpecificFileSystem() != nullptr ? context->GetSpecificFileSystem() : GetFileSystem()) @@ -108,10 +108,8 @@ Result> ReadOptimizedSystemTable::CreateDataReadCon branch = branch_iter->second; } ReadContextBuilder builder(table_path_); - // The data table behind this view is served through the file system the view itself was - // opened with, but only when the context names neither a file system of its own nor a - // scheme map to resolve through - a file system and a scheme map cannot both be named, and - // the context's own answer always wins. + // The view's own file system, but only when the context names neither one of its own nor a + // scheme map to resolve through - a file system and a scheme map cannot both be named. std::shared_ptr data_table_fs = context->GetSpecificFileSystem(); if (data_table_fs == nullptr && context->GetFileSystemSchemeToIdentifierMap().empty()) { data_table_fs = GetFileSystem(); @@ -151,6 +149,8 @@ Result> ReadOptimizedSystemTable::CreateDataReadCon Result> ReadOptimizedSystemTable::NewRead( std::unique_ptr ctx) const { const std::shared_ptr context = std::move(ctx); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "read")); + PAIMON_RETURN_NOT_OK(CheckNoSearch(context)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr base_context, CreateDataReadContext(context)); return TableRead::Create(std::move(base_context)); diff --git a/src/paimon/core/table/system/system_table.cpp b/src/paimon/core/table/system/system_table.cpp index cf4864071..0161ad6b7 100644 --- a/src/paimon/core/table/system/system_table.cpp +++ b/src/paimon/core/table/system/system_table.cpp @@ -42,6 +42,7 @@ #include "paimon/core/utils/branch_manager.h" #include "paimon/file_store_commit.h" #include "paimon/file_store_write.h" +#include "paimon/read_context.h" #include "paimon/status.h" #include "paimon/write_context.h" @@ -185,6 +186,14 @@ SystemTable::SystemTable(const std::shared_ptr& fs, const std::strin : Table(fs, table_path, /*schema=*/nullptr, Identifier(Identifier::kUnknownDatabase, "system_table"), /*uuid=*/"") {} +Status SystemTable::CheckNoSearch(const std::shared_ptr& context) { + if (context->HasVectorOrFullTextSearch()) { + return Status::NotImplemented( + "VectorSearch or FullTextSearch does not support system-table reads yet"); + } + return Status::OK(); +} + Status SystemTable::AdoptOwnSchema() { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr arrow_schema, ArrowSchema()); schema_ = std::make_shared(std::move(arrow_schema)); diff --git a/src/paimon/core/table/system/system_table.h b/src/paimon/core/table/system/system_table.h index 96eead153..4c235d3cf 100644 --- a/src/paimon/core/table/system/system_table.h +++ b/src/paimon/core/table/system/system_table.h @@ -50,13 +50,10 @@ struct SystemTablePath { bool is_global = false; }; -/// Base class for table-scoped system tables such as `T$options` and `T$snapshots`. -/// -/// A system table is the read-only view a `T$system` identifier names: it exposes a schema of -/// its own and serves scans and reads from the data table it belongs to, through the same -/// `Table` interface as every other kind of table. `Name()` - which each kind of system table -/// answers for itself - is its full name as well: the identifier it is opened by belongs to the -/// data table underneath and would name that one. +/// Base class for table-scoped system tables such as `T$options` and `T$snapshots`: the read-only +/// view a `T$system` identifier names, which exposes a schema of its own and serves scans and +/// reads from the data table it belongs to. `Name()` - which each kind answers for itself - is +/// its full name as well: the identifier it is opened by belongs to the data table underneath. class SystemTable : public Table, public std::enable_shared_from_this { public: std::string FullName() const override { @@ -66,14 +63,11 @@ class SystemTable : public Table, public std::enable_shared_from_this> ArrowSchema() const = 0; - /// A system table has no options of its own to override: its schema is the read-only view it - /// exposes, which is not reconfigured by a copy. The data table underneath is copied through - /// its own `Table`. + /// A system table is read-only: copying it, writing it and committing for it are refused + /// rather than routed to the data table underneath, which its own `Table` serves. Result> Copy( const std::map& dynamic_options) const override; - /// A system table is read-only: writing it or committing for it is refused rather than - /// routed to the data table underneath. Result> NewWrite( std::unique_ptr context) const override; @@ -85,9 +79,13 @@ class SystemTable : public Table, public std::enable_shared_from_this& fs, const std::string& table_path); - /// Loaders call this once the object exists, because `ArrowSchema()` is virtual and cannot - /// run from a constructor: it fills the schema the base hands out with this table's own - /// read-only `SystemTableSchema`. + /// Refuses a read whose context asks for a search: each system table builds a context of its + /// own for the data table underneath, which carries no search, so answering one would return + /// plain rows and quietly drop the search and its top K. + static Status CheckNoSearch(const std::shared_ptr& context); + + /// Loaders call this once the object exists, because `ArrowSchema()` is virtual and cannot run + /// from a constructor: it fills in this table's own read-only `SystemTableSchema`. Status AdoptOwnSchema(); friend class SystemTableLoader; diff --git a/src/paimon/core/table/system/system_table_test.cpp b/src/paimon/core/table/system/system_table_test.cpp index 0a9f85a2e..f1f0cbfb1 100644 --- a/src/paimon/core/table/system/system_table_test.cpp +++ b/src/paimon/core/table/system/system_table_test.cpp @@ -45,10 +45,13 @@ #include "paimon/fs/file_system_factory.h" #include "paimon/memory/memory_pool.h" #include "paimon/metrics.h" +#include "paimon/predicate/vector_search.h" #include "paimon/read_context.h" #include "paimon/reader/batch_reader.h" #include "paimon/result.h" +#include "paimon/scan_context.h" #include "paimon/status.h" +#include "paimon/table/source/table_read.h" #include "paimon/testing/mock/mock_file_batch_reader.h" #include "paimon/testing/utils/read_result_collector.h" #include "paimon/testing/utils/testharness.h" @@ -301,6 +304,62 @@ TEST(SystemTableTest, TestGlobalSystemTableWithoutCatalogReturnsNotImplemented) "global system table requires catalog context: tables"); } +// A system table reached through `NewXxx` directly - one a catalog handed out - is checked the +// way the static factories check the contexts they take: a null context is refused rather than +// dereferenced. +TEST(SystemTableTest, TestOperatorEntriesRefuseANullContext) { + std::map options = {{Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "orc"}}; + ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, + CreateTableSchemaForTest(options)); + + AuditLogSystemTable audit_log(/*fs=*/nullptr, "/tmp/table", table_schema, options); + BinlogSystemTable binlog(/*fs=*/nullptr, "/tmp/table", table_schema, options); + ReadOptimizedSystemTable read_optimized(/*fs=*/nullptr, "/tmp/table", table_schema, options); + + ASSERT_NOK_WITH_MSG(audit_log.NewScan(nullptr), "scan context is null pointer"); + ASSERT_NOK_WITH_MSG(binlog.NewScan(nullptr), "scan context is null pointer"); + ASSERT_NOK_WITH_MSG(read_optimized.NewScan(nullptr), "scan context is null pointer"); + ASSERT_NOK_WITH_MSG(audit_log.NewRead(nullptr), "read context is null pointer"); + ASSERT_NOK_WITH_MSG(binlog.NewRead(nullptr), "read context is null pointer"); + ASSERT_NOK_WITH_MSG(read_optimized.NewRead(nullptr), "read context is null pointer"); +} + +// The reads these views serve build a context of their own for the data table underneath, which +// carries no search: a search the caller asked for is refused rather than quietly answered with +// plain rows. +TEST(SystemTableTest, TestNewReadRefusesASearch) { + std::map options = {{Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "orc"}}; + ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, + CreateTableSchemaForTest(options)); + + AuditLogSystemTable audit_log(/*fs=*/nullptr, "/tmp/table", table_schema, options); + BinlogSystemTable binlog(/*fs=*/nullptr, "/tmp/table", table_schema, options); + ReadOptimizedSystemTable read_optimized(/*fs=*/nullptr, "/tmp/table", table_schema, options); + + auto make_context = [] { + auto vector_search = std::make_shared( + "v", /*limit=*/3, std::vector{1.0f}, /*pre_filter=*/nullptr, + /*predicate=*/nullptr, VectorSearch::DistanceType::COSINE, + std::map{}); + ReadContextBuilder builder("/tmp/table"); + builder.SetVectorSearch(vector_search); + return builder.Finish(); + }; + + ASSERT_OK_AND_ASSIGN(std::unique_ptr audit_log_context, make_context()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr binlog_context, make_context()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_optimized_context, make_context()); + + ASSERT_NOK_WITH_MSG(audit_log.NewRead(std::move(audit_log_context)), + "VectorSearch or FullTextSearch does not support system-table reads yet"); + ASSERT_NOK_WITH_MSG(binlog.NewRead(std::move(binlog_context)), + "VectorSearch or FullTextSearch does not support system-table reads yet"); + ASSERT_NOK_WITH_MSG(read_optimized.NewRead(std::move(read_optimized_context)), + "VectorSearch or FullTextSearch does not support system-table reads yet"); +} + namespace { /// A catalog that serves the files of every table through `GetTableFileSystem`, like one diff --git a/src/paimon/core/table/table.cpp b/src/paimon/core/table/table.cpp new file mode 100644 index 000000000..3180a48fe --- /dev/null +++ b/src/paimon/core/table/table.cpp @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/catalog/table.h" + +namespace paimon { + +Table::Table(const std::shared_ptr& file_system, const std::string& root_path, + const std::shared_ptr& schema, const Identifier& identifier, + const std::string& uuid) + : file_system_(file_system), + root_path_(root_path), + schema_(schema), + identifier_(identifier), + uuid_(uuid) {} + +} // namespace paimon diff --git a/src/paimon/core/table/table_test.cpp b/src/paimon/core/table/table_test.cpp index b6cc22859..055fc5995 100644 --- a/src/paimon/core/table/table_test.cpp +++ b/src/paimon/core/table/table_test.cpp @@ -36,10 +36,13 @@ #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/table_schema.h" #include "paimon/core/table/file_store_table.h" +#include "paimon/core/table/source/data_split_impl.h" +#include "paimon/core/table/source/fallback_data_split.h" #include "paimon/defs.h" #include "paimon/file_store_commit.h" #include "paimon/file_store_write.h" #include "paimon/read_context.h" +#include "paimon/reader/batch_reader.h" #include "paimon/record_batch.h" #include "paimon/scan_context.h" #include "paimon/schema/schema.h" @@ -49,6 +52,8 @@ #include "paimon/table/source/table_scan.h" #include "paimon/testing/mock/mock_catalog.h" #include "paimon/testing/mock/mock_file_system.h" +#include "paimon/testing/utils/counting_cache_test_utils.h" +#include "paimon/testing/utils/read_result_collector.h" #include "paimon/testing/utils/testharness.h" #include "paimon/write_context.h" @@ -73,29 +78,6 @@ Result> CreateManagedTable( /*uuid=*/"", branch); } -/// Counts the lookups a scan makes, which is enough to tell that the cache a context carries -/// reached the scan's manifest reading rather than being dropped on the way. -class CountingCache : public Cache { - public: - Result> Get( - const std::shared_ptr& key, - std::function>(const std::shared_ptr&)> - supplier) override { - gets++; - return supplier(key); - } - Status Put(const std::shared_ptr&, const std::shared_ptr&) override { - return Status::OK(); - } - void Invalidate(const std::shared_ptr&) override {} - void InvalidateAll() override {} - size_t Size() const override { - return 0; - } - - int32_t gets = 0; -}; - /// Writes two rows through the table's own write and commit, so a scan below has data to find. void WriteAndCommitTwoRows(const std::shared_ptr
& table, const std::string& root_path) { WriteContextBuilder write_builder(root_path, "commit-user"); @@ -124,6 +106,42 @@ void WriteAndCommitTwoRows(const std::shared_ptr
& table, const std::strin ASSERT_OK(write->Close()); } +/// Writes one `id`/`v` row through the table's own write and commit, so a scan below has data +/// to find. +void WriteAndCommitRow(const std::shared_ptr
& table, const std::string& root_path, + int32_t id, int32_t v) { + WriteContextBuilder write_builder(root_path, "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write, + table->NewWrite(std::move(write_context))); + arrow::Int32Builder ids_builder; + arrow::Int32Builder vs_builder; + ASSERT_TRUE(ids_builder.AppendValues({id}).ok()); + ASSERT_TRUE(vs_builder.AppendValues({v}).ok()); + std::shared_ptr ids_array; + std::shared_ptr vs_array; + ASSERT_TRUE(ids_builder.Finish(&ids_array).ok()); + ASSERT_TRUE(vs_builder.Finish(&vs_array).ok()); + auto struct_array = + arrow::StructArray::Make({ids_array, vs_array}, + {arrow::field("id", arrow::int32(), /*nullable=*/false), + arrow::field("v", arrow::int32())}) + .ValueOrDie(); + ::ArrowArray arrow_array; + ASSERT_TRUE(arrow::ExportArray(*struct_array, &arrow_array).ok()); + RecordBatchBuilder batch_builder(&arrow_array); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, batch_builder.SetBucket(0).Finish()); + ASSERT_OK(write->Write(std::move(batch))); + ASSERT_OK_AND_ASSIGN(std::vector> messages, + write->PrepareCommit()); + CommitContextBuilder commit_builder(root_path, "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, + table->NewCommit(std::move(commit_context))); + ASSERT_OK(commit->Commit(messages)); + ASSERT_OK(write->Close()); +} + } // namespace TEST(TableTest, TestTableCarriesManagersOfItsBranch) { @@ -198,6 +216,219 @@ TEST(TableTest, TestCopyKeepsTheBranchTheTableServes) { } } +TEST(TableTest, TestACopyHandsItsDynamicOptionsToTheFallbackReader) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::shared_ptr fs = dir->GetFileSystem(); + const std::string path = dir->Str() + "/tbl"; + const Identifier identifier("db", "tbl"); + + // A primary-key table whose reads fall back to the `fb` branch, and that branch's schema. + SchemaManager schema_manager(fs, path, "main"); + auto schema = arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false)}); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr created, + schema_manager.CreateTable(schema, /*partition_keys=*/{}, /*primary_keys=*/{"id"}, + {{Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "id"}, + {Options::SCAN_FALLBACK_BRANCH, "fb"}})); + SchemaManager fallback_schema_manager(fs, path, "fb"); + ASSERT_OK_AND_ASSIGN( + [[maybe_unused]] std::unique_ptr fallback_schema, + fallback_schema_manager.CreateTable(schema, /*partition_keys=*/{}, + /*primary_keys=*/{"id"}, + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
table, + FileStoreTable::Create( + fs, path, + checked_pointer_cast(std::shared_ptr(std::move(created))), + identifier, /*uuid=*/"", /*branch=*/"main")); + + // The copy turns the sequence-number column on, and a read projects it through a context + // that carries no options of its own. The main reader serves the copy's merged options, so + // it knows the column; the fallback serves the `fb` branch's schema, whose options do not + // name it, so the copy's own dynamic options are what let the fallback reader know it too - + // without them the read as a whole fails to build. + ASSERT_OK_AND_ASSIGN(std::shared_ptr
copy, + table->Copy({{Options::KEY_VALUE_SEQUENCE_NUMBER_ENABLED, "true"}})); + ReadContextBuilder read_builder(path); + read_builder.SetReadFieldNames({"id", "_SEQUENCE_NUMBER"}); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr read, + copy->NewRead(std::move(read_context))); +} + +TEST(TableTest, TestACopyOfACopyServesTheLatestValueOfARepeatedDynamicOption) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::shared_ptr fs = dir->GetFileSystem(); + const std::string path = dir->Str() + "/tbl"; + const Identifier identifier("db", "tbl"); + + // A primary-key table whose reads fall back to the `fb` branch, and that branch's schema. + SchemaManager schema_manager(fs, path, "main"); + auto schema = arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false)}); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr created, + schema_manager.CreateTable(schema, /*partition_keys=*/{}, /*primary_keys=*/{"id"}, + {{Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "id"}, + {Options::SCAN_FALLBACK_BRANCH, "fb"}})); + SchemaManager fallback_schema_manager(fs, path, "fb"); + ASSERT_OK_AND_ASSIGN( + [[maybe_unused]] std::unique_ptr fallback_schema, + fallback_schema_manager.CreateTable(schema, /*partition_keys=*/{}, + /*primary_keys=*/{"id"}, + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
table, + FileStoreTable::Create( + fs, path, + checked_pointer_cast(std::shared_ptr(std::move(created))), + identifier, /*uuid=*/"", /*branch=*/"main")); + + // The option is turned off and then on again by the copy the read serves, so the copy's + // schema carries the latest value. The fallback branch's schema does not carry the option + // at all, so what lets its reader know the column is the copy's accumulated dynamic + // options - which must speak the same latest value the schema does, not the older one a + // first copy left, or the fallback reader fails to build while the main one succeeds. + ASSERT_OK_AND_ASSIGN(std::shared_ptr
off, + table->Copy({{Options::KEY_VALUE_SEQUENCE_NUMBER_ENABLED, "false"}})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
copy, + off->Copy({{Options::KEY_VALUE_SEQUENCE_NUMBER_ENABLED, "true"}})); + ReadContextBuilder read_builder(path); + read_builder.SetReadFieldNames({"id", "_SEQUENCE_NUMBER"}); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr read, + copy->NewRead(std::move(read_context))); +} + +TEST(TableTest, TestAFallbackReaderServesItsBranchsOwnSchema) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::shared_ptr fs = dir->GetFileSystem(); + const std::string path = dir->Str() + "/tbl"; + const Identifier identifier("db", "tbl"); + + // The main branch is an append-only table whose reads fall back to the `fb` branch, and + // that branch is laid out as a primary-key table: a second write of the key the first + // wrote is a merge, not a duplicate. + SchemaManager main_schema_manager(fs, path, "main"); + auto row_schema = arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false), + arrow::field("v", arrow::int32())}); + ASSERT_OK_AND_ASSIGN(std::unique_ptr main_created, + main_schema_manager.CreateTable(row_schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, + {{Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "id"}, + {Options::SCAN_FALLBACK_BRANCH, "fb"}})); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
main, + FileStoreTable::Create( + fs, path, + checked_pointer_cast(std::shared_ptr(std::move(main_created))), + identifier, /*uuid=*/"", /*branch=*/"main")); + + SchemaManager fallback_schema_manager(fs, path, "fb"); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr fallback_created, + fallback_schema_manager.CreateTable(row_schema, /*partition_keys=*/{}, + /*primary_keys=*/{"id"}, + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
fallback, + FileStoreTable::Create( + fs, path, + checked_pointer_cast(std::shared_ptr(std::move(fallback_created))), + identifier, /*uuid=*/"", /*branch=*/"fb")); + WriteAndCommitRow(fallback, path, /*id=*/1, /*v=*/10); + WriteAndCommitRow(fallback, path, /*id=*/1, /*v=*/20); + + // The read of the main table carries its fallback reader, which reads the `fb` branch's + // data through the `fb` branch's own schema - so it is the primary-key read that schema's + // layout asks for, not the append-only one the main table's kind would pick. Its splits + // are the `fb` branch's own, scanned through a table that serves `fb` and read as + // fallback ones, which is how a fallback reader is asked for its rows. + ReadContextBuilder read_builder(path); + read_builder.SetReadFieldNames({"id", "v"}); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, main->NewRead(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, + ScanContextBuilder(path).Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr fallback_scan, + fallback->NewScan(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr fallback_plan, fallback_scan->CreatePlan()); + ASSERT_FALSE(fallback_plan->Splits().empty()); + std::vector> fallback_splits; + for (const std::shared_ptr& split : fallback_plan->Splits()) { + auto data_split = std::dynamic_pointer_cast(split); + ASSERT_TRUE(data_split != nullptr); + fallback_splits.push_back( + std::make_shared(data_split, /*is_fallback=*/true)); + } + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, + read->CreateReader(fallback_splits)); + + // The one merged row the `fb` branch's primary-key layout leaves behind - at the value + // the second write carried - is what the fallback reader answers with; the append-only + // reader the main table's kind would have picked answers with both rows, or refuses the + // split outright. + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, batch_reader->NextBatch()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr array, + ReadResultCollector::GetArray(std::move(batch))); + ASSERT_TRUE(array); + ASSERT_EQ(array->length(), 1); + auto struct_array = std::dynamic_pointer_cast(array); + ASSERT_TRUE(struct_array != nullptr); + EXPECT_EQ(struct_array->num_fields(), 3); + auto value_kind_array = std::dynamic_pointer_cast(struct_array->field(0)); + auto id_array = std::dynamic_pointer_cast(struct_array->field(1)); + auto v_array = std::dynamic_pointer_cast(struct_array->field(2)); + ASSERT_TRUE(value_kind_array != nullptr); + ASSERT_TRUE(id_array != nullptr); + ASSERT_TRUE(v_array != nullptr); + EXPECT_EQ(value_kind_array->Value(0), 0); + EXPECT_EQ(id_array->Value(0), 1); + EXPECT_EQ(v_array->Value(0), 20); + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch eof_batch, batch_reader->NextBatch()); + EXPECT_TRUE(BatchReader::IsEofBatch(eof_batch)); +} + +TEST(TableTest, TestACopyThatMovesAnS3OptionRebuildsItsManagers) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::shared_ptr fs = dir->GetFileSystem(); + const std::string path = dir->Str() + "/tbl"; + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
table, + CreateManagedTable( + fs, path, identifier, /*branch=*/"main", + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}, {"s3.endpoint", "http://old"}})); + auto file_store_table = std::dynamic_pointer_cast(table); + ASSERT_TRUE(file_store_table != nullptr); + + // A native `s3.` option names the file system the S3 backend resolves from, the way an + // `fs.` one does: a copy that moves it re-resolves, so the managers are rebuilt rather + // than inherited - otherwise the copy would read its metadata through the old endpoint + // while its data goes through the new one. + ASSERT_OK_AND_ASSIGN(std::shared_ptr
moved, + table->Copy({{"s3.endpoint", "http://new"}})); + auto moved_file_store_table = std::dynamic_pointer_cast(moved); + ASSERT_TRUE(moved_file_store_table != nullptr); + EXPECT_NE(moved_file_store_table->GetSchemaManager(), file_store_table->GetSchemaManager()); + EXPECT_NE(moved_file_store_table->GetSnapshotManager(), file_store_table->GetSnapshotManager()); + + // A copy whose `s3.` option says what the schema already says resolves the same file + // system, so the managers are the ones this table already holds. + ASSERT_OK_AND_ASSIGN(std::shared_ptr
kept, table->Copy({{"s3.endpoint", "http://old"}})); + auto kept_file_store_table = std::dynamic_pointer_cast(kept); + ASSERT_TRUE(kept_file_store_table != nullptr); + EXPECT_EQ(kept_file_store_table->GetSchemaManager(), file_store_table->GetSchemaManager()); + EXPECT_EQ(kept_file_store_table->GetSnapshotManager(), file_store_table->GetSnapshotManager()); +} + TEST(TableTest, TestSwitchToBranch) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); @@ -896,15 +1127,20 @@ TEST(TableTest, TestTheCacheAScanContextCarriesReachesTheScan) { WriteAndCommitTwoRows(table, dir->Str()); // The cache is a runtime setting the scan serves from the context rather than from the - // table's own options, so a scan built with one reads its manifests through it. - auto cache = std::make_shared(); + // table's own options, so a scan built with one reads its manifests through it. A scan reads + // the manifest list of a snapshot on its executor's threads, so the counter the cache keeps + // is guarded rather than a plain field. + auto cache = std::make_shared( + std::map{{CacheKind::MANIFEST, 64 * 1024 * 1024}, + {CacheKind::SNAPSHOT_LIVE_MANIFEST, 64 * 1024 * 1024}}); ScanContextBuilder scan_builder(dir->Str()); scan_builder.WithCache(cache); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_ctx, scan_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(scan_ctx))); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); ASSERT_FALSE(plan->Splits().empty()); - ASSERT_GT(cache->gets, 0) << "the cache the scan context carries never reached the scan"; + ASSERT_GT(cache->GetCount(CacheKind::MANIFEST), 0) + << "the cache the scan context carries never reached the scan"; } TEST(TableTest, TestCopyServesTheTableWithTheGivenOptionsOnTop) { @@ -1029,33 +1265,44 @@ TEST(TableTest, TestALoadNamesTheDatabaseThePathLaysOut) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); const std::shared_ptr fs = dir->GetFileSystem(); + auto schema = arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false), + arrow::field("dt", arrow::utf8())}); // A context that names no catalog knows the table by its directory: the directory above // the table's names the database when it carries the `.db` suffix, the way the Java - // `identifierFromPath` reads it, so the table the load serves is named for both - which - // the load of a directory without a schema says in the error it fails with. The option - // the context carries is what drives the load to that error: it hands the table the merge - // would build, which is where the name is spoken. + // `identifierFromPath` reads it, so the table the load serves is named for both. The + // refusal a format table open speaks names the table - and the option the context + // carries is what drives the open to it, since the schema alone opens cleanly. const std::string table_dir = dir->Str() + "/my_db.db/my_table"; ASSERT_OK(fs->Mkdirs(table_dir)); + SchemaManager schema_manager(fs, table_dir); + ASSERT_OK_AND_ASSIGN( + [[maybe_unused]] std::unique_ptr format_schema, + schema_manager.CreateTable(schema, /*partition_keys=*/{"dt"}, /*primary_keys=*/{}, + {{"type", "format-table"}})); { ScanContextBuilder builder(table_dir); - builder.AddOption("read.batch-size", "2"); + builder.AddOption(Options::REALTIME_ENABLED, "true"); ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, builder.Finish()); ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(ctx)), - "not found latest schema of my_db.my_table"); + "cannot open format table my_db.my_table"); } // A table sitting directly under the root, with no `.db` directory above it, names no // database: the table is known by its directory alone, the way it was before. const std::string loose_table_dir = dir->Str() + "/loose_table"; ASSERT_OK(fs->Mkdirs(loose_table_dir)); + SchemaManager loose_schema_manager(fs, loose_table_dir); + ASSERT_OK_AND_ASSIGN( + [[maybe_unused]] std::unique_ptr loose_schema, + loose_schema_manager.CreateTable(schema, /*partition_keys=*/{"dt"}, /*primary_keys=*/{}, + {{"type", "format-table"}})); { ScanContextBuilder builder(loose_table_dir); - builder.AddOption("read.batch-size", "2"); + builder.AddOption(Options::REALTIME_ENABLED, "true"); ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, builder.Finish()); ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(ctx)), - "not found latest schema of loose_table"); + "cannot open format table loose_table"); } } diff --git a/src/paimon/rest/rest_catalog.cpp b/src/paimon/rest/rest_catalog.cpp index b570aa307..cae923d7f 100644 --- a/src/paimon/rest/rest_catalog.cpp +++ b/src/paimon/rest/rest_catalog.cpp @@ -426,10 +426,8 @@ Result> RestCatalog::LoadSystemTable( if (branch) { dynamic_options[Options::BRANCH] = branch.value(); } - // The system table reads the data table's own files - snapshots, manifests, data - so it is - // served by the file system that table's credentials open, which with data tokens is the - // only one that can: `GetTableFileSystem` folds the system identifier to the data table it - // belongs to and hands the catalog-wide file system out when no tokens are issued. + // The system table reads the data table's own files, so it is served by that table's own + // credentials - with data tokens the only file system that can open them. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table_file_system, GetTableFileSystem(identifier)); return SystemTableLoader::Load(system_table_name.value(), table_file_system, data_table_path, @@ -438,9 +436,8 @@ Result> RestCatalog::LoadSystemTable( Result> RestCatalog::GetTable(const Identifier& identifier) const { // The global system tables of the "sys" database expose the catalog itself rather than any - // table the server stands behind, so they are served locally - the way `LoadTableSchema` - // serves them their schema - and never reach the managed-table load below, whose schema - // cast a read-only system table schema would not survive. + // table the server stands behind, so they are served locally and never reach the managed-table + // load below, whose schema cast a read-only system table schema would not survive. if (CatalogUtils::IsSystemDatabase(identifier.GetDatabaseName())) { PAIMON_ASSIGN_OR_RAISE(bool supported, GlobalSystemTableLoader::IsSupported(identifier.GetObjectName(), @@ -450,13 +447,21 @@ Result> RestCatalog::GetTable(const Identifier& identifie } GlobalSystemTableContext context; context.catalog = this; + // A global system table builds its rows lazily through the catalog above, so the table + // handed back has to keep it alive or the pointer dangles at the first `BuildRows`. A + // catalog not owned by a `std::shared_ptr` cannot be kept, so it is refused by name. + std::shared_ptr self = weak_from_this().lock(); + if (self == nullptr) { + return Status::Invalid( + "a global system table retains its catalog, so the catalog must be owned by " + "a std::shared_ptr before it is asked for one"); + } + context.catalog_retainer = std::move(self); context.fs = fs_; context.warehouse = warehouse_; context.catalog_options = api_->GetMergedOptions(); return GlobalSystemTableLoader::Load(identifier.GetObjectName(), context); } - // A `T$system` identifier opens the read-only view it names, which reads the data table it - // belongs to and answers for itself through the same `Table` interface. PAIMON_ASSIGN_OR_RAISE(bool is_system_table, identifier.IsSystemTable()); if (is_system_table) { PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); @@ -468,9 +473,8 @@ Result> RestCatalog::GetTable(const Identifier& identifie std::string table_path; PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, LoadTableSchema(identifier, &uuid, &table_path)); - // The schema the same response carried says what the table is: a format table is handed - // back as the `FormatTable` it is, built from the location of the same response, rather - // than refused as the managed table it is not. + // The schema the same response carried says what the table is: a format table is handed back + // as the `FormatTable` it is rather than refused as the managed table it is not. std::shared_ptr data_schema = std::dynamic_pointer_cast(schema); if (data_schema != nullptr) { PAIMON_ASSIGN_OR_RAISE(bool is_format_table, @@ -489,8 +493,7 @@ Result> RestCatalog::GetTable(const Identifier& identifie // tokens binds to this table. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table_file_system, GetTableFileSystem(identifier)); - // The branch the identifier carries, which the schema and path above were read on, so the - // table and its managers serve it. + // The branch the schema and path above were read on, which the table's managers then serve. PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); return FileStoreTable::Create( table_file_system, table_path, schema, identifier, uuid, diff --git a/src/paimon/rest/rest_catalog.h b/src/paimon/rest/rest_catalog.h index 5c62ed472..1663b6d2e 100644 --- a/src/paimon/rest/rest_catalog.h +++ b/src/paimon/rest/rest_catalog.h @@ -42,7 +42,9 @@ class TableSchema; /// A catalog backed by a REST catalog server. Metadata operations are delegated to /// `RestApi`; table data is accessed through the file system configured by the /// server-merged options. -class RestCatalog : public Catalog, public VersionManagedCatalog { +class RestCatalog : public Catalog, + public VersionManagedCatalog, + public std::enable_shared_from_this { public: /// Creates the catalog: fetches and merges "/v1/config" from the server configured /// by `CatalogOptions::URI`, then builds the file system from the merged options. diff --git a/src/paimon/rest/rest_catalog_test.cpp b/src/paimon/rest/rest_catalog_test.cpp index 4acd6796d..54929fb48 100644 --- a/src/paimon/rest/rest_catalog_test.cpp +++ b/src/paimon/rest/rest_catalog_test.cpp @@ -56,10 +56,17 @@ #include "paimon/file_store_commit.h" #include "paimon/fs/file_system.h" #include "paimon/memory/memory_pool.h" +#include "paimon/read_context.h" +#include "paimon/reader/batch_reader.h" #include "paimon/rest/mock_rest_server.h" #include "paimon/rest/rest_api.h" +#include "paimon/scan_context.h" #include "paimon/schema/schema.h" #include "paimon/table/format/format_table.h" +#include "paimon/table/source/plan.h" +#include "paimon/table/source/table_read.h" +#include "paimon/table/source/table_scan.h" +#include "paimon/testing/utils/read_result_collector.h" #include "paimon/testing/utils/snapshot_test_helper.h" #include "paimon/testing/utils/testharness.h" @@ -1537,6 +1544,40 @@ TEST_F(RestCatalogTest, SystemDatabase) { ASSERT_TRUE(missing.IsNotExist()) << missing.ToString(); } +TEST_F(RestCatalogTest, GlobalSystemTableRetainsItsCatalog) { + // A global system table builds its rows lazily through the catalog that served it, so the + // table handed back keeps that catalog alive: reading it after the caller let its own + // reference go still answers, rather than reading through a catalog already destroyed. + ASSERT_OK_AND_ASSIGN(std::unique_ptr created, CreateRestCatalog()); + std::shared_ptr catalog = std::move(created); + ASSERT_OK(catalog->CreateDatabase("db1", {}, /*ignore_if_exists=*/false)); + ASSERT_OK(CreateSampleTable(catalog.get(), Identifier("db1", "t1"))); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + catalog->GetTable(Identifier("sys", "tables"))); + std::weak_ptr weak = catalog; + catalog.reset(); + ASSERT_FALSE(weak.expired()) << "the global system table does not retain its catalog"; + + // The rows are built through the retained catalog, so they still answer once the caller's + // own reference is gone: the one table the catalog stands behind shows up in them. + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, + ScanContextBuilder("sys/tables").Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_EQ(plan->Splits().size(), 1); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, + ReadContextBuilder("sys/tables").Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, table->NewRead(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, + read->CreateReader(plan->Splits())); + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, batch_reader->NextBatch()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr array, + ReadResultCollector::GetArray(std::move(batch))); + ASSERT_TRUE(array); + ASSERT_GT(array->length(), 0); +} + TEST_F(RestCatalogTest, ListSnapshots) { ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); ASSERT_OK(catalog->CreateDatabase("db1", {}, /*ignore_if_exists=*/false)); diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp index fd9b49650..5de6c9fa7 100644 --- a/test/inte/realtime_write_inte_test.cpp +++ b/test/inte/realtime_write_inte_test.cpp @@ -43,7 +43,6 @@ #include "gtest/gtest.h" #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" -#include "paimon/catalog/table.h" #include "paimon/commit_context.h" #include "paimon/common/data/variant/variant_type_utils.h" #include "paimon/common/table/special_fields.h" @@ -560,41 +559,6 @@ class RealtimeWriteInteTest : public ::testing::Test { return FileStoreWrite::Create(std::move(context)); } - // A writer that means to change an option the table was created with writes through a copy - // carrying it, which the context then agrees with. - Result> CopiedTable( - const std::map& overrides) const { - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr catalog, Catalog::Create(dir_->Str(), {})); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, - catalog->GetTable(Identifier("foo", "bar"))); - return table->Copy(overrides); - } - - // Evolves the table schema to the current options with its fields unchanged, the way an - // ALTER TABLE changing an option would: the table itself then serves the changed option. - // The fields are carried over from the latest schema so their attributes survive: rebuilt - // raw fields would lose the non-nullable primary keys the table schema enforces. - Status EvolveSchemaWithOptions() const { - SchemaManager schema_manager(dir_->GetFileSystem(), table_path_); - PAIMON_ASSIGN_OR_RAISE(std::optional> latest, - schema_manager.Latest()); - if (!latest.has_value()) { - return Status::Invalid("table schema does not exist"); - } - return TestHelper::WriteNextSchema(dir_->GetFileSystem(), table_path_, - latest.value()->Fields(), - latest.value()->HighestFieldId(), options_); - } - - Result> CreateRealtimeWriter( - const std::shared_ptr& realtime_context, - const std::shared_ptr
& table) const { - WriteContextBuilder builder(table_path_, commit_user_); - builder.SetOptions(options_).WithStreamingMode(true).WithRealtimeContext(realtime_context); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); - return table->NewWrite(std::move(context)); - } - Result> CreateRealtimeWriter() const { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr realtime_context, RealtimeContext::Create()); @@ -1285,36 +1249,32 @@ TEST_F(RealtimeWriteInteTest, TestRealtimeOperationsRequireEnabledOption) { disabled_options[Options::REALTIME_ENABLED] = "false"; ASSERT_OK_AND_ASSIGN(std::shared_ptr realtime_context, RealtimeContext::Create()); - // The table was created enabled, so each operation below goes through a copy carrying it - // disabled, whose contexts then agree: each reports that real-time needs the option itself. - ASSERT_OK_AND_ASSIGN(std::shared_ptr
disabled_table, - CopiedTable({{Options::REALTIME_ENABLED, "false"}})); WriteContextBuilder write_builder(table_path_, commit_user_); write_builder.SetOptions(disabled_options) .WithStreamingMode(true) .WithRealtimeContext(realtime_context); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); - ASSERT_NOK_WITH_MSG(disabled_table->NewWrite(std::move(write_context)), + ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_context)), "real-time write requires realtime.enabled=true"); ScanContextBuilder scan_builder(table_path_); scan_builder.SetOptions(disabled_options).WithRealtimeContext(realtime_context); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); - ASSERT_NOK_WITH_MSG(disabled_table->NewScan(std::move(scan_context)), + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), "real-time scan requires realtime.enabled=true"); ReadContextBuilder read_builder(table_path_); read_builder.SetOptions(disabled_options).WithRealtimeContext(realtime_context); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); - ASSERT_NOK_WITH_MSG(disabled_table->NewRead(std::move(read_context)), + ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), "real-time read requires realtime.enabled=true"); CommitContextBuilder commit_builder(table_path_, commit_user_); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.SetOptions(disabled_options).Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - disabled_table->NewCommit(std::move(commit_context))); + FileStoreCommit::Create(std::move(commit_context))); ASSERT_NOK_WITH_MSG(commit->CommitWithProgress(/*realtime_commits=*/{}, /*commit_identifier=*/0, /*watermark=*/std::nullopt), @@ -2124,9 +2084,6 @@ TEST_F(RealtimeWriteInteTest, TestRealtimeDeduplicateAcrossMultipleFileIndexStor ASSERT_OK(embedded_writer->Close()); options_[Options::FILE_INDEX_IN_MANIFEST_THRESHOLD] = "1B"; - // The tighter threshold is a change the table itself carries, the way an ALTER TABLE would: - // the schema is evolved to it, so every later write, commit, and read serves it. - ASSERT_OK(EvolveSchemaWithOptions()); ASSERT_OK_AND_ASSIGN(std::unique_ptr external_writer, CreateRealtimeWriter(realtime_context)); ASSERT_OK_AND_ASSIGN(std::unique_ptr external_batch, @@ -3809,9 +3766,6 @@ TEST_F(RealtimeWriteInteTest, TestAppendRealtimeReadUsesEmbeddedAndExternalBitma ASSERT_OK(embedded_writer->Close()); options_[Options::FILE_INDEX_IN_MANIFEST_THRESHOLD] = "1B"; - // The tighter threshold is a change the table itself carries, the way an ALTER TABLE would: - // the schema is evolved to it, so every later write, commit, and read serves it. - ASSERT_OK(EvolveSchemaWithOptions()); ASSERT_OK_AND_ASSIGN(std::unique_ptr external_writer, CreateRealtimeWriter(realtime_context)); ASSERT_OK_AND_ASSIGN( @@ -4129,9 +4083,6 @@ TEST_F(RealtimeWriteInteTest, TestPkDvRealtimeReadUsesEmbeddedAndExternalBitmapI ASSERT_OK(embedded_writer->Close()); options_[Options::FILE_INDEX_IN_MANIFEST_THRESHOLD] = "1B"; - // The tighter threshold is a change the table itself carries, the way an ALTER TABLE would: - // the schema is evolved to it, so every later write, commit, and read serves it. - ASSERT_OK(EvolveSchemaWithOptions()); ASSERT_OK_AND_ASSIGN(std::unique_ptr external_writer, CreateRealtimeWriter(realtime_context)); ASSERT_OK_AND_ASSIGN(std::unique_ptr external_batch, diff --git a/test/inte/scan_and_read_inte_test.cpp b/test/inte/scan_and_read_inte_test.cpp index 4fa4e1da1..b7cc73b96 100644 --- a/test/inte/scan_and_read_inte_test.cpp +++ b/test/inte/scan_and_read_inte_test.cpp @@ -36,9 +36,6 @@ #include "fmt/ranges.h" #include "gtest/gtest.h" #include "paimon/bucket/bucket_id_calculator.h" -#include "paimon/catalog/catalog.h" -#include "paimon/catalog/identifier.h" -#include "paimon/catalog/table.h" #include "paimon/common/factories/io_hook.h" #include "paimon/common/io/cache/lru_cache.h" #include "paimon/common/table/special_fields.h" @@ -799,36 +796,19 @@ TEST_P(ScanAndReadInteTest, TestWithAppendBucketKeyPointLookup) { } ASSERT_GE(other_index, 0); // These keys share a current two-bucket selection but need different historical buckets. - // The data was written with four buckets, so reading a rescaled layout reads through a - // copy of the table carrying the rescaled bucket count: an option on the context would - // contradict the table the catalog loaded. - ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, - Catalog::Create(test_dir->Str(), {})); - ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, - catalog->GetTable(Identifier("foo", "bar"))); for (int32_t lookup_index : {32, other_index}) { + // The data was written with four buckets. Read it with rescaled table options too. for (const std::string current_bucket_count : {"2", "4", "8", "17"}) { SCOPED_TRACE(current_bucket_count); const std::string lookup_key = fmt::format("key{:03}", lookup_index); auto lookup_predicate = PredicateBuilder::Equal( 0, "rowkey", field_type, Literal(field_type, lookup_key.data(), lookup_key.size())); - // The rescaled bucket count and, with it, the cache settings `FinishScanContext` - // also puts on the context: a table already held takes its runtime settings - // through the copy it is made of, and the context carries them only to agree - // with it. - std::map dynamic_options{ - {Options::BUCKET, current_bucket_count}}; - if (EnableSnapshotLiveManifestCache()) { - dynamic_options.emplace(Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, "3"); - } - ASSERT_OK_AND_ASSIGN(std::shared_ptr
rescaled_table, - table->Copy(dynamic_options)); ScanContextBuilder scan_builder(table_path); scan_builder.SetPredicate(lookup_predicate) .AddOption(Options::BUCKET, current_bucket_count); ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_builder)); - ASSERT_OK_AND_ASSIGN(auto scan, rescaled_table->NewScan(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto plan, scan->CreatePlan()); if (EnableSnapshotLiveManifestCache()) { ASSERT_OK_AND_ASSIGN( @@ -840,7 +820,7 @@ TEST_P(ScanAndReadInteTest, TestWithAppendBucketKeyPointLookup) { .AddOption(Options::BUCKET, current_bucket_count); ASSERT_OK_AND_ASSIGN(auto cached_context, FinishScanContext(cached_builder)); ASSERT_OK_AND_ASSIGN(auto cached_scan, - rescaled_table->NewScan(std::move(cached_context))); + TableScan::Create(std::move(cached_context))); ASSERT_OK_AND_ASSIGN(plan, cached_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(uint64_t hit, cached_scan->GetMetrics()->GetCounter( ScanMetrics::LAST_SNAPSHOT_CACHE_HIT)); @@ -3106,21 +3086,12 @@ TEST_P(ScanAndReadInteTest, TestScanAndReadWithDisableIndex) { PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, Literal(FieldType::STRING, "Alice0", 6)); - // The table was loaded with its index enabled, so the index-disabled pass below goes through - // a copy carrying it disabled: an option on the context would contradict the table. - ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, - Catalog::Create(paimon::test::GetDataDir() + file_format, {})); - ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, - catalog->GetTable(Identifier("append_with_bitmap", "append_with_bitmap"))); - ASSERT_OK_AND_ASSIGN(std::shared_ptr
disabled_table, - table->Copy({{"file-index.read.enabled", "false"}})); - // scan ScanContextBuilder scan_context_builder(table_path); ASSERT_OK_AND_ASSIGN( auto scan_context, scan_context_builder.AddOption("file-index.read.enabled", "false").Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, disabled_table->NewScan(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -3131,7 +3102,7 @@ TEST_P(ScanAndReadInteTest, TestScanAndReadWithDisableIndex) { read_context_builder.AddOption("file-index.read.enabled", "false"); AddReadOptionsForPrefetch(&read_context_builder); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, disabled_table->NewRead(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -3187,13 +3158,6 @@ TEST(ScanAndReadFileIndexInteTest, TestPkDvEmbeddedBitmapFiltersScanAndRead) { helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0, std::nullopt)); ASSERT_FALSE(commit_messages.empty()); - // The statistics-only baseline below disables the file index the table carries, so it scans - // through a copy with it disabled: an option on the context would contradict the table. - ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, Catalog::Create(dir->Str(), {})); - ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog->GetTable(Identifier("foo", "bar"))); - ASSERT_OK_AND_ASSIGN(std::shared_ptr
stats_only_table, - table->Copy({{Options::FILE_INDEX_READ_ENABLED, "false"}})); - // 15 falls inside the file's [10, 30] min/max range, so statistics alone must retain the // file. Disabling file-index reads verifies that baseline explicitly. const std::shared_ptr absent_predicate = PredicateBuilder::Equal( @@ -3205,7 +3169,7 @@ TEST(ScanAndReadFileIndexInteTest, TestPkDvEmbeddedBitmapFiltersScanAndRead) { ASSERT_OK_AND_ASSIGN(std::unique_ptr stats_only_scan_context, stats_only_scan_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr stats_only_scan, - stats_only_table->NewScan(std::move(stats_only_scan_context))); + TableScan::Create(std::move(stats_only_scan_context))); ASSERT_OK_AND_ASSIGN(std::shared_ptr stats_only_plan, stats_only_scan->CreatePlan()); ASSERT_FALSE(stats_only_plan->Splits().empty()); @@ -3298,13 +3262,6 @@ TEST(ScanAndReadFileIndexInteTest, TestPkDvReconstructsFileIndexPredicateAfterSc const std::shared_ptr predicate = PredicateBuilder::Equal( /*field_index=*/1, /*field_name=*/"renamed_value", FieldType::INT, Literal(15)); - // The statistics-only baseline below disables the file index the table carries, so it scans - // through a copy with it disabled: an option on the context would contradict the table. - ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, Catalog::Create(dir->Str(), {})); - ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog->GetTable(Identifier("foo", "bar"))); - ASSERT_OK_AND_ASSIGN(std::shared_ptr
stats_only_table, - table->Copy({{Options::FILE_INDEX_READ_ENABLED, "false"}})); - // The evolved min/max range is still [10, 30], so statistics alone retain the old file. ScanContextBuilder stats_only_scan_builder(table_path); stats_only_scan_builder.SetOptions(options) @@ -3313,7 +3270,7 @@ TEST(ScanAndReadFileIndexInteTest, TestPkDvReconstructsFileIndexPredicateAfterSc ASSERT_OK_AND_ASSIGN(std::unique_ptr stats_only_scan_context, stats_only_scan_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr stats_only_scan, - stats_only_table->NewScan(std::move(stats_only_scan_context))); + TableScan::Create(std::move(stats_only_scan_context))); ASSERT_OK_AND_ASSIGN(std::shared_ptr stats_only_plan, stats_only_scan->CreatePlan()); ASSERT_FALSE(stats_only_plan->Splits().empty()); @@ -4064,31 +4021,16 @@ TEST_P(ScanAndReadInteTest, TestWithPKBucketSelectByPredicate) { auto predicate = PredicateBuilder::Equal(/*field_index=*/2, /*field_name=*/"f2", FieldType::INT, Literal(static_cast(0))); - // Historical files use two buckets even when the current option has changed, so the scan - // goes through a copy of the table carrying the bucket count it selects by: an option on - // the context would contradict the table the catalog loaded. - ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, - Catalog::Create(paimon::test::GetDataDir() + file_format, {})); - ASSERT_OK_AND_ASSIGN( - std::shared_ptr
table, - catalog->GetTable(Identifier("pk_table_scan_and_read_dv", "pk_table_scan_and_read_dv"))); + // Historical files use two buckets even when the current option has changed. for (const std::string current_bucket_count : {"2", "4", "8", "17"}) { SCOPED_TRACE(current_bucket_count); - // The rescaled bucket count and, with it, the cache settings `FinishScanContext` also - // puts on the context: a table already held takes its runtime settings through the copy - // it is made of, and the context carries them only to agree with it. - std::map dynamic_options{{Options::BUCKET, current_bucket_count}}; - if (EnableSnapshotLiveManifestCache()) { - dynamic_options.emplace(Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, "3"); - } - ASSERT_OK_AND_ASSIGN(std::shared_ptr
rescaled_table, table->Copy(dynamic_options)); ScanContextBuilder scan_context_builder(table_path); scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "6"); scan_context_builder.AddOption(Options::BUCKET, current_bucket_count); scan_context_builder.SetPartitionFilter({{{"f1", "10"}}}); scan_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, rescaled_table->NewScan(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); AddReadOptionsForPrefetch(&read_context_builder); @@ -4107,8 +4049,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKBucketSelectByPredicate) { .SetPartitionFilter({{{"f1", "10"}}}) .SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto cached_context, FinishScanContext(cached_builder)); - ASSERT_OK_AND_ASSIGN(auto cached_scan, - rescaled_table->NewScan(std::move(cached_context))); + ASSERT_OK_AND_ASSIGN(auto cached_scan, TableScan::Create(std::move(cached_context))); ASSERT_OK_AND_ASSIGN(result_plan, cached_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(uint64_t hit, cached_scan->GetMetrics()->GetCounter( ScanMetrics::LAST_SNAPSHOT_CACHE_HIT)); diff --git a/test/inte/scan_inte_test.cpp b/test/inte/scan_inte_test.cpp index 1502c39a7..3f949484d 100644 --- a/test/inte/scan_inte_test.cpp +++ b/test/inte/scan_inte_test.cpp @@ -32,9 +32,6 @@ #include "arrow/api.h" #include "gtest/gtest.h" -#include "paimon/catalog/catalog.h" -#include "paimon/catalog/identifier.h" -#include "paimon/catalog/table.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/data/data_define.h" #include "paimon/common/utils/decimal_utils.h" @@ -1164,18 +1161,10 @@ TEST_P(ScanInteTest, TestScanAppendWithInvalidOptions) { "scan.snapshot-id or scan.tag-name must be set when startup mode is FROM_SNAPSHOT"); } { - // `bucket` = -2 is refused by the scan itself, which a copy of the table carrying it - // reaches: an option on the context would contradict the table the catalog loaded, - // which a scan through that table refuses. - ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, - Catalog::Create(paimon::test::GetDataDir() + "orc", {})); - ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, - catalog->GetTable(Identifier("append_09", "append_09"))); - ASSERT_OK_AND_ASSIGN(std::shared_ptr
copied, table->Copy({{Options::BUCKET, "-2"}})); ScanContextBuilder context_builder(table_path); - context_builder.WithStreamingMode(true); + context_builder.AddOption(Options::BUCKET, "-2").WithStreamingMode(true); ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_NOK_WITH_MSG(copied->NewScan(std::move(scan_context)), + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), "do not support bucket=-2 in scan process"); } } @@ -1737,18 +1726,10 @@ TEST_P(ScanInteTest, TestScanPKWithSnapshot1WithBucketStats) { TEST_P(ScanInteTest, TestScanPKWithInvalidOptions) { std::string table_path = paimon::test::GetDataDir() + "orc/pk_table_with_total_buckets.db/pk_table_with_total_buckets"; - // Like the append case above, the invalid bucket count rides on a copy of the table: the - // scan refuses it where the bucket count is validated. - ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, - Catalog::Create(paimon::test::GetDataDir() + "orc", {})); - ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, - catalog->GetTable(Identifier("pk_table_with_total_buckets", - "pk_table_with_total_buckets"))); - ASSERT_OK_AND_ASSIGN(std::shared_ptr
copied, table->Copy({{Options::BUCKET, "-1"}})); ScanContextBuilder context_builder(table_path); - context_builder.WithStreamingMode(true); + context_builder.AddOption(Options::BUCKET, "-1").WithStreamingMode(true); ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_NOK_WITH_MSG(copied->NewScan(std::move(scan_context)), + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), "do not support pk table bucket=-1 in scan process"); } diff --git a/test/inte/write_and_read_inte_with_index_test.cpp b/test/inte/write_and_read_inte_with_index_test.cpp index 9123a3385..efda3de03 100644 --- a/test/inte/write_and_read_inte_with_index_test.cpp +++ b/test/inte/write_and_read_inte_with_index_test.cpp @@ -1250,7 +1250,8 @@ TEST_P(WriteAndReadInteWithIndexTest, TestReadWithLimits) { PredicateBuilder::NotEqual(/*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, Literal(FieldType::STRING, "Alice", 5)); ReadContextBuilder context_builder(path); - context_builder.AddOption("test.enable-adaptive-prefetch-strategy", "false") + context_builder.AddOption(Options::FILE_FORMAT, "orc") + .AddOption("test.enable-adaptive-prefetch-strategy", "false") .AddOption(Options::READ_BATCH_SIZE, "1") .AddOption("orc.read.enable-metrics", "true") .SetPredicate(predicate); From 81fb658666581158ae4f31350bfcc2b135cb1bce Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Tue, 27 Oct 2026 09:42:57 +0800 Subject: [PATCH 15/30] fix --- src/paimon/core/catalog/snapshot_commit.cpp | 59 +++ .../core/table/file_store_table_factory.cpp | 346 ++++++++++++++++++ .../core/table/file_store_table_factory.h | 51 +++ src/paimon/core/utils/context_utils.h | 51 +++ 4 files changed, 507 insertions(+) create mode 100644 src/paimon/core/catalog/snapshot_commit.cpp create mode 100644 src/paimon/core/table/file_store_table_factory.cpp create mode 100644 src/paimon/core/table/file_store_table_factory.h create mode 100644 src/paimon/core/utils/context_utils.h diff --git a/src/paimon/core/catalog/snapshot_commit.cpp b/src/paimon/core/catalog/snapshot_commit.cpp new file mode 100644 index 000000000..35efe767c --- /dev/null +++ b/src/paimon/core/catalog/snapshot_commit.cpp @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/core/catalog/snapshot_commit.h" + +#include +#include + +#include "paimon/core/catalog/catalog_snapshot_commit.h" +#include "paimon/core/catalog/renaming_snapshot_commit.h" +#include "paimon/core/catalog/version_managed_catalog.h" +#include "paimon/result.h" +#include "paimon/status.h" + +namespace paimon { + +Result> SnapshotCommit::Create( + const std::shared_ptr& catalog, const std::optional& catalog_identifier, + const std::optional& table_id, bool use_rest_catalog_commit, + const std::shared_ptr& file_system, + const std::shared_ptr& snapshot_manager) { + if (AsVersionManaged(catalog) != nullptr) { + if (!catalog_identifier) { + return Status::Invalid("a catalog commit requires a table identifier"); + } + // Reuse the caller's UUID so table recreation cannot redirect this commit. + return std::shared_ptr( + std::make_shared(catalog, catalog_identifier.value(), table_id)); + } + if (use_rest_catalog_commit) { + return std::shared_ptr(std::make_shared(table_id)); + } + // Catalogs without version management may still supply a table ID. + if (table_id && catalog == nullptr) { + return Status::Invalid( + "a commit which publishes the snapshot by writing it to the table directory sends no " + "commit table request, so there is nowhere to put a table id; commit through a " + "catalog with WithCatalog() or ask for the request with UseRESTCatalogCommit()"); + } + return std::shared_ptr( + std::make_shared(file_system, snapshot_manager)); +} + +} // namespace paimon diff --git a/src/paimon/core/table/file_store_table_factory.cpp b/src/paimon/core/table/file_store_table_factory.cpp new file mode 100644 index 000000000..2b5c9a40c --- /dev/null +++ b/src/paimon/core/table/file_store_table_factory.cpp @@ -0,0 +1,346 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/core/table/file_store_table_factory.h" + +#include +#include +#include +#include +#include + +#include "fmt/format.h" +#include "paimon/catalog/catalog.h" +#include "paimon/catalog/identifier.h" +#include "paimon/catalog/table.h" +#include "paimon/commit_context.h" +#include "paimon/common/utils/checked_cast.h" +#include "paimon/core/catalog/catalog_utils.h" +#include "paimon/core/catalog/version_managed_catalog.h" +#include "paimon/core/core_options.h" +#include "paimon/core/options/table_type.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/file_store.h" +#include "paimon/core/table/file_store_table.h" +#include "paimon/core/table/system/system_table.h" +#include "paimon/core/utils/branch_manager.h" +#include "paimon/fs/file_system.h" +#include "paimon/predicate/predicate_utils.h" +#include "paimon/read_context.h" +#include "paimon/result.h" +#include "paimon/scan_context.h" +#include "paimon/schema/schema.h" +#include "paimon/status.h" +#include "paimon/table/format/format_table.h" +#include "paimon/write_context.h" + +namespace paimon { + +namespace { + +/// Recognises a format table behind `table_path`, or returns null when it is not one. The scan, +/// read, write and commit entry points below all dispatch on the same question, so one place +/// answers it and they cannot disagree. Null is also the answer when no schema was found under +/// the path at all: what to say about a table that is not there is the managed path's to decide. +/// +/// The table type is read from the schema, never from `options`: `type` is structural, and one +/// call's options must not decide what kind of table this is. +Result> TryLoadFormatTable( + const std::shared_ptr& file_system, const std::string& table_path, + const std::string& branch, const std::map& options, + const std::shared_ptr& table_schema) { + if (table_schema == nullptr) { + return std::shared_ptr(); + } + // From the schema alone: see `FormatTable::Create()`. + PAIMON_ASSIGN_OR_RAISE(bool is_format_table, + TableTypeDefine::IsFormatTable(table_schema->Options())); + if (!is_format_table) { + return std::shared_ptr(); + } + // A format table has no branches, so it is refused rather than served from the table path + // under another branch's name. See `SchemaValidation::ValidateFormatTableSchema()`. + if (BranchManager::NormalizeBranch(branch) != BranchManager::DEFAULT_MAIN_BRANCH) { + return Status::Invalid( + fmt::format("a format table has no branches, so {} cannot be read on branch '{}'", + table_path, branch)); + } + Identifier identifier = CatalogUtils::IdentifierFromPath(table_path, branch); + return FormatTable::Create(file_system, table_path, identifier, + checked_pointer_cast(table_schema), + /*location_carries_paimon_metadata=*/true, options); +} + +/// The options a context carries, merged into the table the load hands back: a table is the +/// product of the load, so the options speak there - the way `Copy` merges them - and `NewXxx` on +/// the product serves them without merging again. A caller holding a table already changes it +/// through `Copy` itself, and an option that contradicts the table that way is refused. +Result> WithContextOptions( + const std::shared_ptr
& table, const std::shared_ptr& schema, + const std::map& options) { + if (table == nullptr || schema == nullptr || options.empty()) { + return table; + } + const std::map& schema_options = schema->Options(); + for (const auto& [key, value] : options) { + auto schema_option = schema_options.find(key); + // A key the schema does not carry, or carries differently: the merge has something to do. + if (schema_option == schema_options.end() || schema_option->second != value) { + return table->Copy(options); + } + } + return table; +} + +/// The tail every `Create` ends with: the schema one of them read says what the path names. A +/// format table is handed back as the `FormatTable` it is, everything else as the `FileStoreTable` +/// a managed table is, carrying the schema for its `NewXxx` to build from. A null schema is handed +/// over as it is - see `TryLoadFormatTable`. +/// +/// @param file_system The file system the schema above was read through, which a format table +/// is served by - it has no schema options of its own to name another one. +/// @param explicit_file_system The file system the context supplied, which a managed table is +/// opened through when there is one. A null one leaves the table to resolve from its +/// schema's own access configuration rather than being pinned to the file system this load +/// happened to read the schema through. +Result> TableOfLoadedSchema( + const std::shared_ptr& file_system, + const std::shared_ptr& explicit_file_system, const std::string& path, + const std::string& branch, const std::map& options, + const std::shared_ptr& schema_manager, + const std::shared_ptr& latest_schema, const std::optional& identifier, + const std::string& uuid = "", const std::map& fs_scheme_map = {}) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, + TryLoadFormatTable(file_system, path, branch, options, latest_schema)); + if (format_table != nullptr) { + return std::shared_ptr
(std::move(format_table)); + } + // The branch the schema was read on is folded into the object name. + Identifier table_identifier = + identifier.value_or(CatalogUtils::IdentifierFromPath(path, branch)); + // The manager was built over the file system the context's own options resolve, while the + // store resolves the schema's: a schema carrying access configuration names a backend the + // context's options never did, and a manager kept through that disagreement would read the + // table's metadata through the wrong one. The schema is in hand already, so dropping the + // manager costs its cache alone. A context that supplied a file system wins on both sides and + // keeps it; a scheme map only the store's resolution takes part in is the same disagreement. + std::shared_ptr manager = schema_manager; + if (manager != nullptr && latest_schema != nullptr && explicit_file_system == nullptr && + (!fs_scheme_map.empty() || + !FileStore::FileSystemResolutionAgrees(latest_schema->Options(), options))) { + manager = nullptr; + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, + FileStoreTable::Create(explicit_file_system, path, latest_schema, + table_identifier, uuid, branch, manager, + /*snapshot_manager=*/nullptr, fs_scheme_map)); + return WithContextOptions(table, latest_schema, options); +} + +/// The managed table a context names behind a path: the schema is read through a manager of the +/// context's branch - handed over instead when the caller already read it - and what the schema +/// says is there decides the table `TableOfLoadedSchema` hands back. +/// +/// @param schema_json The schema of the table, when the caller already read it, or null when it +/// has to be read here. It is honoured on the main branch only; a branch publishes its own +/// under `branch/branch-`, which is where the read below goes. +/// @param explicit_file_system The file system the context supplied, which the managed table the +/// schema describes is opened through - see `TableOfLoadedSchema`. +Result> ManagedTableOf( + const std::shared_ptr& file_system, + const std::shared_ptr& explicit_file_system, const std::string& path, + const std::string& branch, const std::map& options, + const std::optional& schema_json, const std::optional& identifier, + const std::string& uuid = "", const std::map& fs_scheme_map = {}) { + auto schema_manager = std::make_shared(file_system, path, branch); + std::shared_ptr latest_schema; + if (branch == BranchManager::DEFAULT_MAIN_BRANCH && schema_json) { + // Handing the schema over only saves reading it. A context always names a table path and + // this library writes the metadata under it, so `schema` and `branch` stay metadata. + PAIMON_ASSIGN_OR_RAISE(latest_schema, TableSchema::CreateFromJson(schema_json.value())); + } else { + PAIMON_ASSIGN_OR_RAISE(std::optional> latest, + schema_manager->Latest()); + if (latest) { + latest_schema = *latest; + } + } + return TableOfLoadedSchema(file_system, explicit_file_system, path, branch, options, + schema_manager, latest_schema, identifier, uuid, fs_scheme_map); +} + +/// The schema a catalog may hold for the table, read from it on the main branch only and null +/// otherwise: a branch publishes its own under `branch/branch-`, which is where +/// `ManagedTableOf` reads instead, so a catalog answering for no branch schema does not fail the +/// `kind` operation to one. `kind` also names the operator the messages speak of. +Result> CatalogMainBranchSchemaJson( + const std::shared_ptr& catalog, const std::optional& catalog_identifier, + const std::string& branch, const char* kind) { + if (catalog == nullptr) { + return std::optional(); + } + if (!catalog_identifier) { + return Status::Invalid(fmt::format("a catalog {} requires a table identifier", kind)); + } + if (BranchManager::IsMainBranch(branch)) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, + catalog->LoadTableSchema(catalog_identifier.value())); + PAIMON_ASSIGN_OR_RAISE(std::string schema_json, schema->GetJsonSchema()); + return std::optional(std::move(schema_json)); + } + return std::optional(); +} + +} // namespace + +/// The table a scan context names: the format table the builder was handed, a system table behind +/// a `T$system` path, or the table the schema under the context's path says is there. +Result> FileStoreTableFactory::Create(const ScanContext& ctx) { + if (ctx.GetFormatTable() != nullptr) { + return std::shared_ptr
(ctx.GetFormatTable()); + } + PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, + CoreOptions::FromMap(ctx.GetOptions(), ctx.GetSpecificFileSystem(), {})); + PAIMON_ASSIGN_OR_RAISE(std::optional system_table_path, + SystemTableLoader::TryParsePath(ctx.GetPath())); + if (system_table_path) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, + SystemTableLoader::LoadFromPath(tmp_options.GetFileSystem(), + ctx.GetPath(), ctx.GetOptions())); + return std::shared_ptr
(std::move(system_table)); + } + // One `TableScan` interface serves a format table too, the way Java Paimon serves both + // through one `ReadBuilder`; a format table is planned by listing directories instead. + const std::string branch = BranchManager::NormalizeBranch(tmp_options.GetBranch()); + return ManagedTableOf(tmp_options.GetFileSystem(), ctx.GetSpecificFileSystem(), ctx.GetPath(), + branch, ctx.GetOptions(), ctx.GetSpecificTableSchema(), + ctx.GetIdentifier()); +} + +/// The table a read context names: the format table the builder was handed, a system table behind +/// a `T$system` path, or the table the schema under the context's path says is there. +Result> FileStoreTableFactory::Create(const ReadContext& ctx) { + if (ctx.GetFormatTable() != nullptr) { + return std::shared_ptr
(ctx.GetFormatTable()); + } + PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_core_options, + CoreOptions::FromMap(ctx.GetOptions(), ctx.GetSpecificFileSystem(), + ctx.GetFileSystemSchemeToIdentifierMap())); + PAIMON_ASSIGN_OR_RAISE(std::optional system_table_path, + SystemTableLoader::TryParsePath(ctx.GetPath())); + if (system_table_path) { + if (PredicateUtils::ContainsFullTextMatch(ctx.GetPredicate())) { + return Status::NotImplemented( + "FullTextMatch predicate execution is not implemented yet"); + } + if (ctx.HasVectorOrFullTextSearch()) { + return Status::NotImplemented( + "VectorSearch or FullTextSearch is not supported for system tables"); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, + SystemTableLoader::LoadFromPath(tmp_core_options.GetFileSystem(), + ctx.GetPath(), ctx.GetOptions())); + return std::shared_ptr
(std::move(system_table)); + } + + // A format table has no manifest and a file another engine wrote carries no field ids, so how + // a row is put back together is its own; how a file is opened is not - both paths go through + // `DataFileReaderFactory`. + const std::string branch = ctx.GetBranch(); + // The scheme map the schema was read through, so the table's data resolves custom schemes the + // same way its metadata does. + return ManagedTableOf(tmp_core_options.GetFileSystem(), ctx.GetSpecificFileSystem(), + ctx.GetPath(), branch, ctx.GetOptions(), ctx.GetSpecificTableSchema(), + ctx.GetIdentifier(), /*uuid=*/"", + ctx.GetFileSystemSchemeToIdentifierMap()); +} + +/// The table a write context names: the format table the builder was handed, or the table the +/// schema under the context's root path says is there. +Result> FileStoreTableFactory::Create(const WriteContext& ctx) { + if (ctx.GetFormatTable() != nullptr) { + return std::shared_ptr
(ctx.GetFormatTable()); + } + PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx.GetCatalog())); + std::shared_ptr specific_fs = ctx.GetSpecificFileSystem(); + if (specific_fs == nullptr && ctx.GetFileSystemSchemeToIdentifierMap().empty() && + ctx.GetCatalog() != nullptr) { + if (!ctx.GetIdentifier()) { + return Status::Invalid("a catalog write requires a table identifier"); + } + // A catalog issuing per-table temporary credentials only hands them out through + // GetTableFileSystem, so the write uses this table's. + PAIMON_ASSIGN_OR_RAISE(specific_fs, + ctx.GetCatalog()->GetTableFileSystem(ctx.GetIdentifier().value())); + } + PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, + CoreOptions::FromMap(ctx.GetOptions(), specific_fs, + ctx.GetFileSystemSchemeToIdentifierMap())); + std::string branch = ctx.GetBranch(); + PAIMON_ASSIGN_OR_RAISE(std::optional catalog_identifier, + CatalogUtils::BranchIdentifier(ctx.GetIdentifier(), branch)); + // Only the main branch may have a current schema the catalog alone holds: a branch publishes + // its own under `branch/branch-`, which is where the load below reads it instead. + PAIMON_ASSIGN_OR_RAISE( + std::optional catalog_table_schema, + CatalogMainBranchSchemaJson(ctx.GetCatalog(), catalog_identifier, branch, "write")); + return ManagedTableOf(tmp_options.GetFileSystem(), specific_fs, ctx.GetRootPath(), branch, + ctx.GetOptions(), catalog_table_schema, ctx.GetIdentifier(), /*uuid=*/"", + ctx.GetFileSystemSchemeToIdentifierMap()); +} + +/// The table a commit context names: the format table the builder was handed, or the table the +/// schema under the context's root path says is there. +Result> FileStoreTableFactory::Create(const CommitContext& ctx) { + if (ctx.GetFormatTable() != nullptr) { + return std::shared_ptr
(ctx.GetFormatTable()); + } + PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx.GetCatalog())); + // Use this table's catalog credentials unless the caller supplied a file system: a catalog + // issuing per-table temporary ones only hands them out through GetTableFileSystem. + std::shared_ptr specific_fs = ctx.GetSpecificFileSystem(); + if (specific_fs == nullptr && ctx.GetCatalog() != nullptr) { + if (!ctx.GetIdentifier()) { + return Status::Invalid("a catalog commit requires a table identifier"); + } + PAIMON_ASSIGN_OR_RAISE(specific_fs, + ctx.GetCatalog()->GetTableFileSystem(ctx.GetIdentifier().value())); + } + PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, + CoreOptions::FromMap(ctx.GetOptions(), specific_fs)); + const std::string& root_path = ctx.GetRootPath(); + // The branch the context was built for, already validated by `CommitContextBuilder`. + PAIMON_ASSIGN_OR_RAISE(std::string branch, BranchManager::ResolveBranch( + ctx.GetIdentifier(), ctx.GetOptions(), + /*explicit_branch=*/std::nullopt, "commit")); + PAIMON_ASSIGN_OR_RAISE(std::optional catalog_identifier, + CatalogUtils::BranchIdentifier(ctx.GetIdentifier(), branch)); + // Only the main branch may have a current schema the catalog alone holds: a branch publishes + // its own under `branch/branch-`, which is where the load below reads it instead. + PAIMON_ASSIGN_OR_RAISE( + std::optional catalog_table_schema, + CatalogMainBranchSchemaJson(ctx.GetCatalog(), catalog_identifier, branch, "commit")); + // The managed path below reads the commit's branch, so this reads the same one: the two must + // not dispatch on different schemas. + return ManagedTableOf(tmp_options.GetFileSystem(), specific_fs, root_path, branch, + ctx.GetOptions(), catalog_table_schema, ctx.GetIdentifier(), + ctx.GetTableId().value_or("")); +} + +} // namespace paimon diff --git a/src/paimon/core/table/file_store_table_factory.h b/src/paimon/core/table/file_store_table_factory.h new file mode 100644 index 000000000..175834ff6 --- /dev/null +++ b/src/paimon/core/table/file_store_table_factory.h @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include "paimon/result.h" + +namespace paimon { + +class CommitContext; +class ReadContext; +class ScanContext; +class Table; +class WriteContext; + +/// The table a context names, Java Paimon's `FileStoreTableFactory`: the format table the builder +/// was handed, a system table behind a `T$system` path, or the table the schema under the +/// context's path says is there. The four operators' static `Create` entries all dispatch on the +/// same question, so one place answers it and they cannot disagree. +class FileStoreTableFactory { + public: + FileStoreTableFactory() = delete; + ~FileStoreTableFactory() = delete; + + static Result> Create(const ScanContext& context); + + static Result> Create(const ReadContext& context); + + static Result> Create(const WriteContext& context); + + static Result> Create(const CommitContext& context); +}; + +} // namespace paimon diff --git a/src/paimon/core/utils/context_utils.h b/src/paimon/core/utils/context_utils.h new file mode 100644 index 000000000..57e337904 --- /dev/null +++ b/src/paimon/core/utils/context_utils.h @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "fmt/format.h" +#include "paimon/status.h" + +namespace paimon { + +/// The checks every kind of table's operator entry points share. +class ContextUtils { + public: + ContextUtils() = delete; + ~ContextUtils() = delete; + + /// The checks every operator entry point opens with: a context, a memory pool and an executor + /// must all be there for the operator to be built at all, so a null one is refused rather than + /// dereferenced. `kind` names the operator the first message speaks of, which is all that sets + /// the entry points' messages apart. + template + static Status CheckOperatorContext(const ContextPointer& context, const char* kind) { + if (context == nullptr) { + return Status::Invalid(fmt::format("{} context is null pointer", kind)); + } + if (context->GetMemoryPool() == nullptr) { + return Status::Invalid("memory pool is null pointer"); + } + if (context->GetExecutor() == nullptr) { + return Status::Invalid("executor is null pointer"); + } + return Status::OK(); + } +}; + +} // namespace paimon From a87479e02e6c2bbc8785473452a38d3b82c7eec5 Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Tue, 27 Oct 2026 16:10:48 +0800 Subject: [PATCH 16/30] fix --- benchmark/benchmark_suite.cpp | 2 +- docs/source/user_guide/catalog.rst | 20 +-- docs/source/user_guide/format_table.rst | 8 +- examples/read_write_demo.cpp | 2 +- include/paimon/catalog/catalog.h | 11 +- include/paimon/catalog/table.h | 16 +-- include/paimon/file_store_commit.h | 4 +- include/paimon/file_store_write.h | 4 +- include/paimon/table/source/table_read.h | 4 +- include/paimon/table/source/table_scan.h | 4 +- src/paimon/core/catalog/catalog.cpp | 15 ++- .../core/catalog/file_system_catalog.cpp | 15 ++- .../core/catalog/file_system_catalog_test.cpp | 16 ++- ...eted_primary_key_index_maintainer_test.cpp | 4 +- .../core/operation/file_store_write_test.cpp | 5 +- .../core/operation/internal_read_context.cpp | 30 +++-- .../core/operation/internal_read_context.h | 41 +++--- .../core/table/append_only_file_store_table.h | 2 +- src/paimon/core/table/file_store.h | 114 ++++++++--------- src/paimon/core/table/file_store_table.cpp | 6 +- src/paimon/core/table/file_store_table.h | 120 +++++++++--------- .../core/table/file_store_table_factory.cpp | 18 +-- .../core/table/file_store_table_factory.h | 8 +- src/paimon/core/table/format/format_table.cpp | 30 +++-- .../core/table/format/format_table_test.cpp | 29 +++++ .../core/table/primary_key_file_store_table.h | 2 +- .../core/table/source/table_read_test.cpp | 10 +- .../table/system/audit_log_system_table.cpp | 15 ++- .../table/system/audit_log_system_table.h | 7 +- .../core/table/system/binlog_system_table.cpp | 8 +- .../core/table/system/binlog_system_table.h | 1 - .../table/system/global_system_tables.cpp | 70 +++++----- .../core/table/system/global_system_tables.h | 25 ++-- .../table/system/in_memory_system_table.cpp | 4 +- .../table/system/in_memory_system_table.h | 3 +- .../table/system/metadata_system_tables.cpp | 49 ++----- .../table/system/metadata_system_tables.h | 8 -- .../system/read_optimized_system_table.cpp | 8 +- .../system/read_optimized_system_table.h | 1 - src/paimon/core/table/system/system_table.cpp | 19 ++- src/paimon/core/table/system/system_table.h | 48 ++++--- .../core/table/system/system_table_test.cpp | 63 +++++++-- src/paimon/core/table/table_test.cpp | 26 +++- src/paimon/rest/rest_catalog.cpp | 23 ++-- src/paimon/rest/rest_catalog_test.cpp | 38 +++++- src/paimon/testing/mock/mock_catalog.h | 4 +- test/inte/read_inte_test.cpp | 29 +++-- test/inte/realtime_write_inte_test.cpp | 4 +- 48 files changed, 571 insertions(+), 422 deletions(-) diff --git a/benchmark/benchmark_suite.cpp b/benchmark/benchmark_suite.cpp index 0a48a9173..6ec44f0d2 100644 --- a/benchmark/benchmark_suite.cpp +++ b/benchmark/benchmark_suite.cpp @@ -368,7 +368,7 @@ Status EnsureTable(const std::string& root_path, const std::string& db_name, const std::shared_ptr& schema, const std::vector& primary_keys = {}) { PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr catalog, + std::shared_ptr catalog, AddContext(paimon::Catalog::Create(root_path, options), "create catalog")); PAIMON_RETURN_NOT_OK( AddContext(catalog->CreateDatabase(db_name, options, true), "create database")); diff --git a/docs/source/user_guide/catalog.rst b/docs/source/user_guide/catalog.rst index 2d789f839..0ccb976dd 100644 --- a/docs/source/user_guide/catalog.rst +++ b/docs/source/user_guide/catalog.rst @@ -86,7 +86,7 @@ registered on the REST server. The catalog is configured through the {"token.provider", "bear"}, {"token", ""}, }; - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr catalog, + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr catalog, paimon::Catalog::Create(/*root_path=*/"my_instance", options)); For DLF, configure one credential source. Static AK/SK credentials, an optional @@ -127,15 +127,12 @@ without an atomic snapshot-file rename: .. code-block:: cpp - // Writers and committers share ownership of the catalog. - std::shared_ptr shared_catalog(std::move(catalog)); - // Load the table's identity before preparing changes. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, - shared_catalog->GetTable(paimon::Identifier("db", "tbl"))); + catalog->GetTable(paimon::Identifier("db", "tbl"))); paimon::CommitContextBuilder builder(table_path, "commit-user"); - builder.WithCatalog(shared_catalog, paimon::Identifier("db", "tbl")); + builder.WithCatalog(catalog, paimon::Identifier("db", "tbl")); // CatalogUuid() stays null when the server supplies no id. if (std::optional table_uuid = table->CatalogUuid()) { builder.WithTableId(table_uuid.value()); @@ -152,7 +149,7 @@ offsets when a real-time writer is created: .. code-block:: cpp paimon::WriteContextBuilder write_builder(table_path, "commit-user"); - write_builder.WithCatalog(shared_catalog, paimon::Identifier("db", "tbl")); + write_builder.WithCatalog(catalog, paimon::Identifier("db", "tbl")); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, write_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr writer, @@ -188,7 +185,7 @@ returns credentials that are still valid and reloads them before they expire. "oss://bucket/tbl", options)); // MyFileSystem merges provider->GetCredentials() over its own options per access. auto my_fs = std::make_shared(provider); - write_builder.WithCatalog(shared_catalog, paimon::Identifier("db", "tbl")) + write_builder.WithCatalog(catalog, paimon::Identifier("db", "tbl")) .WithFileSystem(my_fs); The snapshot being committed carries a uuid generated on the client, and the @@ -319,18 +316,15 @@ for that table's file system and passing it in through ``WithFileSystem``: .. code-block:: cpp - // Readers and scanners share ownership of the catalog. - std::shared_ptr shared_catalog(std::move(catalog)); - paimon::ScanContextBuilder scan_builder(table_path); - scan_builder.WithCatalog(shared_catalog, paimon::Identifier("db", "tbl")); + scan_builder.WithCatalog(catalog, paimon::Identifier("db", "tbl")); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, paimon::TableScan::Create(std::move(scan_context))); paimon::ReadContextBuilder read_builder(table_path); - read_builder.WithCatalog(shared_catalog, paimon::Identifier("db", "tbl")); + read_builder.WithCatalog(catalog, paimon::Identifier("db", "tbl")); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, diff --git a/docs/source/user_guide/format_table.rst b/docs/source/user_guide/format_table.rst index 4be21a1dd..7ee90976d 100644 --- a/docs/source/user_guide/format_table.rst +++ b/docs/source/user_guide/format_table.rst @@ -53,9 +53,9 @@ That path needs a schema under the table's own location, which is what every tab catalog serves has. A table loaded from a catalog that keeps the schema elsewhere, such as the REST catalog, has none: nothing under the location says that it is a format table, nor that ``schema`` and ``branch`` below it are data rather than metadata. Such a table is read and written -by handing the loaded ``FormatTable`` to the context instead of a path - every context builder has -a constructor taking one - which is what Java does through ``FormatTable.newReadBuilder()`` and -``newBatchWriteBuilder()``: +by handing the loaded ``FormatTable`` to the context builder instead of a path - every builder has +a constructor taking one - and calling the table's own operator, which is what Java does through +``FormatTable.newReadBuilder()`` and ``newBatchWriteBuilder()``: .. code-block:: cpp @@ -68,7 +68,7 @@ a constructor taking one - which is what Java does through ``FormatTable.newRead PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, - paimon::TableScan::Create(std::move(scan_context))); + table->NewScan(std::move(scan_context))); The table already carries its schema and the file system it was loaded through, so a context built from one refuses ``SetTableSchema()``, ``WithFileSystem()``, diff --git a/examples/read_write_demo.cpp b/examples/read_write_demo.cpp index df46fcc54..e911bedb1 100644 --- a/examples/read_write_demo.cpp +++ b/examples/read_write_demo.cpp @@ -61,7 +61,7 @@ paimon::Status Run(const std::string& root_path, const std::string& db_name, {paimon::Options::FILE_SYSTEM, "local"}}; // create table - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr catalog, + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr catalog, paimon::Catalog::Create(root_path, options)); PAIMON_RETURN_NOT_OK(catalog->CreateDatabase(db_name, options, /*ignore_if_exists=*/false)); arrow::FieldVector fields = { diff --git a/include/paimon/catalog/catalog.h b/include/paimon/catalog/catalog.h index 033e7a822..3898a1967 100644 --- a/include/paimon/catalog/catalog.h +++ b/include/paimon/catalog/catalog.h @@ -68,8 +68,11 @@ class PAIMON_EXPORT Catalog { /// schemes, including the file systems it builds from per-table data /// tokens, keeps routing each scheme to its backend. Ignored when /// `file_system` is supplied, which serves every scheme itself. - /// @return A result containing a unique pointer to a `Catalog` instance, or an error status. - static Result> Create( + /// @return A result containing a shared pointer to a `Catalog` instance, or an error status. + /// @note Shared rather than unique ownership: a global system table of the `sys` database + /// builds its rows lazily through the catalog it was handed, so the table keeps the + /// catalog alive. A catalog built by this factory always satisfies that. + static Result> Create( const std::string& root_path, const std::map& options, const std::shared_ptr& file_system = nullptr, const std::map& fs_scheme_to_identifier_map = {}); @@ -154,7 +157,9 @@ class PAIMON_EXPORT Catalog { /// `NewRead()` serve any of them, `NewWrite()` and `NewCommit()` the writable ones. /// /// A global system table of the "sys" database reads its rows lazily through the catalog it - /// came from, so a catalog that serves one must be owned by a `std::shared_ptr`. + /// came from, so the table keeps shared ownership of that catalog. `Catalog::Create()` hands + /// back a `std::shared_ptr`, which satisfies this. A catalog held some other way, such as + /// through a `std::unique_ptr`, cannot be kept and is refused by name. /// /// @param identifier Identifier of the table to get. /// @return A result containing the table, or an error status. diff --git a/include/paimon/catalog/table.h b/include/paimon/catalog/table.h index 3364f6eff..9164623c0 100644 --- a/include/paimon/catalog/table.h +++ b/include/paimon/catalog/table.h @@ -45,10 +45,10 @@ class WriteContext; /// A table provides basic abstraction for table type. /// /// The state every kind of table carries - file system, root path, schema and identifier - lives -/// here, and each kind fills the operations in. The `NewXxx` operators below are the table-first -/// entry points: `TableScan::Create()` and friends load the table a context names and hand the -/// context to them. What a context asks for that the table cannot honour is refused by name rather -/// than silently dropped. +/// here, and each kind fills in the operations. The `NewXxx` operators below are the table-first +/// entry points: `TableScan::Create()` and its friends load the table a context names and hand the +/// context to it. A table refuses by name whatever a context asks for that it cannot honour, +/// rather than dropping it silently. class PAIMON_EXPORT Table { public: virtual ~Table() = default; @@ -65,8 +65,8 @@ class PAIMON_EXPORT Table { return root_path_; } - /// A name to identify this table: a managed or format table answers its identifier's object - /// name, a system table its own kind. + /// A name to identify this table. A managed or format table answers the object name of its + /// identifier, a system table its own name. virtual std::string Name() const = 0; /// Full name of the table: `database.tableName` for a managed or format table, the same name @@ -90,9 +90,9 @@ class PAIMON_EXPORT Table { return schema_; } - /// Copies this table with `dynamic_options` on top of the options it carries: an option given + /// Copies this table with `dynamic_options` on top of the options it carries. An option given /// at the call wins over the one the schema stored, the precedence every context builder - /// promises. `type` is not overridable - it is structural and read from the schema alone. + /// promises. `type` is not overridable: it is structural and read from the schema alone. /// /// @param dynamic_options Options given at the call. /// @return A result containing the copied table, or the error copying it reports. diff --git a/include/paimon/file_store_commit.h b/include/paimon/file_store_commit.h index 581cadf2d..8a6eaad5c 100644 --- a/include/paimon/file_store_commit.h +++ b/include/paimon/file_store_commit.h @@ -50,8 +50,8 @@ class CommitMessage; /// manages versions. `UseRESTCatalogCommit()` only prepares a request for the caller to send. class PAIMON_EXPORT FileStoreCommit { public: - /// Create an instance of `FileStoreCommit`: the table the context names is loaded first, then - /// the context is handed to that table's `NewCommit`. + /// Create an instance of `FileStoreCommit`. A context that already carries a loaded table goes + /// straight to that table's `NewCommit`; otherwise the table the context names is loaded first. /// /// @param context A unique pointer to the `CommitContext` used for commit operations. /// diff --git a/include/paimon/file_store_write.h b/include/paimon/file_store_write.h index 5f74d6536..0778e5bb7 100644 --- a/include/paimon/file_store_write.h +++ b/include/paimon/file_store_write.h @@ -42,8 +42,8 @@ class WriteContext; /// Interface for write operations in a file store. class PAIMON_EXPORT FileStoreWrite { public: - /// Create an instance of `FileStoreWrite`: the table the context names is loaded first, then - /// the context is handed to that table's `NewWrite`. + /// Create an instance of `FileStoreWrite`. A context that already carries a loaded table goes + /// straight to that table's `NewWrite`; otherwise the table the context names is loaded first. /// /// @param context A unique pointer to the `WriteContext` used for write operations. /// diff --git a/include/paimon/table/source/table_read.h b/include/paimon/table/source/table_read.h index 2bfd84fff..5f8303fe3 100644 --- a/include/paimon/table/source/table_read.h +++ b/include/paimon/table/source/table_read.h @@ -42,8 +42,8 @@ class PAIMON_EXPORT TableRead { public: virtual ~TableRead() = default; - /// Create an instance of `TableRead`: the table the context names is loaded first, then the - /// context is handed to that table's `NewRead`. + /// Create an instance of `TableRead`. A context that already carries a loaded table goes + /// straight to that table's `NewRead`; otherwise the table the context names is loaded first. /// /// @param context A unique pointer to the `ReadContext` used for read operations. /// @return A Result containing a unique pointer to the `TableRead` instance. diff --git a/include/paimon/table/source/table_scan.h b/include/paimon/table/source/table_scan.h index 39f0453c8..4944dafd2 100644 --- a/include/paimon/table/source/table_scan.h +++ b/include/paimon/table/source/table_scan.h @@ -36,8 +36,8 @@ class ScanContext; /// A scanner interface for reading table's meta and create a plan. class PAIMON_EXPORT TableScan { public: - /// Create an instance of `TableScan`: the table the context names is loaded first, then the - /// context is handed to that table's `NewScan`. + /// Create an instance of `TableScan`. A context that already carries a loaded table goes + /// straight to that table's `NewScan`; otherwise the table the context names is loaded first. /// /// @param context A unique pointer to the `ScanContext` used for scan operations. /// @return A Result containing a unique pointer to the `TableScan` instance. diff --git a/src/paimon/core/catalog/catalog.cpp b/src/paimon/core/catalog/catalog.cpp index 318125ec6..b14a53eab 100644 --- a/src/paimon/core/catalog/catalog.cpp +++ b/src/paimon/core/catalog/catalog.cpp @@ -40,7 +40,7 @@ const char Catalog::SYSTEM_TABLE_SPLITTER[] = "$"; const char Catalog::DB_SUFFIX[] = ".db"; const char Catalog::DB_LOCATION_PROP[] = "location"; -Result> Catalog::Create( +Result> Catalog::Create( const std::string& root_path, const std::map& options, const std::shared_ptr& file_system, const std::map& fs_scheme_to_identifier_map) { @@ -53,8 +53,14 @@ Result> Catalog::Create( } if (metastore == "rest") { #ifdef PAIMON_ENABLE_REST - return RestCatalog::Create(root_path, options, file_system, RestHttpClient::Config(), - fs_scheme_to_identifier_map); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr rest_catalog, + RestCatalog::Create(root_path, options, file_system, RestHttpClient::Config(), + fs_scheme_to_identifier_map)); + // Built from the derived type so that the `enable_shared_from_this` base a + // `RestCatalog` carries is initialized: a `sys` table keeps its catalog alive through + // `weak_from_this()`, which stays empty when the pointer is widened first. + return std::shared_ptr(std::move(rest_catalog)); #else return Status::NotImplemented( "the rest catalog requires building paimon with PAIMON_ENABLE_REST=ON"); @@ -65,7 +71,8 @@ Result> Catalog::Create( } PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(options, file_system, fs_scheme_to_identifier_map)); - return std::make_unique(core_options.GetFileSystem(), root_path, options); + return std::shared_ptr( + std::make_unique(core_options.GetFileSystem(), root_path, options)); } bool Catalog::SupportsVersionManagement() const { diff --git a/src/paimon/core/catalog/file_system_catalog.cpp b/src/paimon/core/catalog/file_system_catalog.cpp index fdcddf20e..d061acbf2 100644 --- a/src/paimon/core/catalog/file_system_catalog.cpp +++ b/src/paimon/core/catalog/file_system_catalog.cpp @@ -287,16 +287,15 @@ Result> FileSystemCatalog::LoadTableSchema( if (!supported) { return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); } - // The table built here is transient - its schema is read within this call and nothing - // outlives it - so its context keeps no shared ownership of the catalog. A catalog that - // hands a global system table back out (`RestCatalog::GetTable`) retains it instead. + // The table built here is transient: only its schema is read and nothing outlives the + // call. It builds no rows, so the context carries no catalog. GlobalSystemTableContext context; - context.catalog = this; context.fs = fs_; context.warehouse = warehouse_; context.catalog_options = catalog_options_; - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, - GlobalSystemTableLoader::Load(identifier.GetObjectName(), context)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr system_table, + GlobalSystemTableLoader::LoadForSchema(identifier.GetObjectName(), context)); return system_table->LatestSchema(); } // The system table suffix is dropped when the data table identifier is rebuilt below, so the @@ -351,7 +350,9 @@ Result> FileSystemCatalog::GetTable(const Identifier& ide // The branch the schema above was read on, so the table's managers read the same one. PAIMON_ASSIGN_OR_RAISE(std::string branch, identifier.GetBranchNameOrDefault()); return FileStoreTable::Create(fs_, table_path, checked_pointer_cast(*latest_schema), - identifier, /*uuid=*/"", branch); + identifier, /*uuid=*/"", branch, /*schema_manager=*/nullptr, + /*snapshot_manager=*/nullptr, /*fs_scheme_map=*/{}, + /*dynamic_options=*/{}); } Result> FileSystemCatalog::LoadSystemTable( diff --git a/src/paimon/core/catalog/file_system_catalog_test.cpp b/src/paimon/core/catalog/file_system_catalog_test.cpp index f31c0391d..294b7e9b9 100644 --- a/src/paimon/core/catalog/file_system_catalog_test.cpp +++ b/src/paimon/core/catalog/file_system_catalog_test.cpp @@ -231,11 +231,12 @@ TEST(FileSystemCatalogTest, TestOptionsSystemTableCatalog) { ASSERT_OK_AND_ASSIGN(auto system_table, catalog.GetTable(options_identifier)); // A system table comes back as the `SystemTable` it is: it answers for itself with its own - // read-only schema and the root path of the data table it reads. + // read-only schema and the root path of the data table it reads. Its name carries the data + // table it belongs to. ASSERT_TRUE(std::dynamic_pointer_cast(system_table) != nullptr); ASSERT_TRUE(std::dynamic_pointer_cast(system_table) == nullptr); - ASSERT_EQ("options", system_table->Name()); - ASSERT_EQ("options", system_table->FullName()); + ASSERT_EQ("tbl1$options", system_table->Name()); + ASSERT_EQ("tbl1$options", system_table->FullName()); ASSERT_EQ(system_table->GetRootPath(), table_path); ASSERT_EQ((std::vector{"key", "value"}), system_table->LatestSchema()->FieldNames()); @@ -281,7 +282,7 @@ TEST(FileSystemCatalogTest, TestGetTableServesASystemTable) { ASSERT_OK_AND_ASSIGN(auto system_table, catalog.GetTable(Identifier("db1", "tbl1$options"))); ASSERT_TRUE(std::dynamic_pointer_cast(system_table) != nullptr); - ASSERT_EQ("options", system_table->Name()); + ASSERT_EQ("tbl1$options", system_table->Name()); // The scan and the read work from a context of the caller's, like every other table. ScanContextBuilder scan_builder(system_table->GetRootPath()); @@ -310,13 +311,14 @@ TEST(FileSystemCatalogTest, TestGetTableServesASystemTable) { write_builder.SetOptions(options); ASSERT_OK_AND_ASSIGN(auto write_context, write_builder.Finish()); ASSERT_NOK_WITH_MSG(system_table->NewWrite(std::move(write_context)), - "options system table is read-only: writing is not supported"); + "tbl1$options system table is read-only: writing is not supported"); CommitContextBuilder commit_builder(system_table->GetRootPath(), "commit_user"); commit_builder.SetOptions(options); ASSERT_OK_AND_ASSIGN(auto commit_context, commit_builder.Finish()); - ASSERT_NOK_WITH_MSG(system_table->NewCommit(std::move(commit_context)), - "options system table is read-only: committing for it is not supported"); + ASSERT_NOK_WITH_MSG( + system_table->NewCommit(std::move(commit_context)), + "tbl1$options system table is read-only: committing for it is not supported"); } TEST(FileSystemCatalogTest, TestBranchTableLoadsBranchSchema) { diff --git a/src/paimon/core/index/pk/bucketed_primary_key_index_maintainer_test.cpp b/src/paimon/core/index/pk/bucketed_primary_key_index_maintainer_test.cpp index 81803ed90..85690b25f 100644 --- a/src/paimon/core/index/pk/bucketed_primary_key_index_maintainer_test.cpp +++ b/src/paimon/core/index/pk/bucketed_primary_key_index_maintainer_test.cpp @@ -298,7 +298,7 @@ class BucketedPrimaryKeyIndexMaintainerTest : public ::testing::TestWithParam catalog, + ASSERT_OK_AND_ASSIGN(std::shared_ptr catalog, Catalog::Create(directory_->Str(), {})); ASSERT_OK(catalog->CreateDatabase("db", {}, /*ignore_if_exists=*/false)); ASSERT_OK(catalog->CreateTable(Identifier("db", "table"), &c_schema, @@ -373,7 +373,7 @@ TEST_P(BucketedPrimaryKeyIndexMaintainerTest, BuildsIndexForColumnNamedLikeInter schema_ = arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false), arrow::field(kColumnName, arrow::utf8(), /*nullable=*/false)}); options_[Options::PK_BTREE_INDEX_COLUMNS] = kColumnName; - ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, Catalog::Create(directory_->Str(), {})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr catalog, Catalog::Create(directory_->Str(), {})); for (bool spillable : {false, true}) { SCOPED_TRACE(spillable); const std::string table_name = spillable ? "row_id_spill" : "row_id_memory"; diff --git a/src/paimon/core/operation/file_store_write_test.cpp b/src/paimon/core/operation/file_store_write_test.cpp index 520e850e5..ffee33161 100644 --- a/src/paimon/core/operation/file_store_write_test.cpp +++ b/src/paimon/core/operation/file_store_write_test.cpp @@ -170,14 +170,13 @@ TEST(FileStoreWriteTest, TestCreateAppendTable) { ASSERT_OK(catalog->CreateTable(Identifier("foo", "bar"), &schema, /*partition_keys=*/{"f0", "f3"}, /*primary_keys=*/{}, /*options=*/{}, /*ignore_if_exists=*/false)); - std::shared_ptr shared_catalog(std::move(catalog)); - ASSERT_FALSE(shared_catalog->SupportsVersionManagement()); + ASSERT_FALSE(catalog->SupportsVersionManagement()); for (bool use_catalog : {false, true}) { SCOPED_TRACE(use_catalog); WriteContextBuilder context_builder(PathUtil::JoinPath(dir->Str(), "foo.db/bar"), "commit_user_1"); if (use_catalog) { - context_builder.WithCatalog(shared_catalog, Identifier("foo", "bar")); + context_builder.WithCatalog(catalog, Identifier("foo", "bar")); } ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, diff --git a/src/paimon/core/operation/internal_read_context.cpp b/src/paimon/core/operation/internal_read_context.cpp index 8866eae9f..3ee0eb755 100644 --- a/src/paimon/core/operation/internal_read_context.cpp +++ b/src/paimon/core/operation/internal_read_context.cpp @@ -277,17 +277,24 @@ std::optional InternalReadContext::TryResolveSpecialFieldByName( return std::nullopt; } -Result> InternalReadContext::Create( +Result> InternalReadContext::CreateForTable( const std::shared_ptr& context, const std::shared_ptr& table_schema, - CoreOptions core_options, const std::string* path) { + CoreOptions core_options, const std::string& root_path) { core_options.WithCache(context->GetCache()); - return CreateFromCoreOptions(context, table_schema, std::move(core_options), path); + return CreateFromCoreOptions(context, table_schema, std::move(core_options), root_path); } Result> InternalReadContext::Create( + const std::shared_ptr& context, const std::shared_ptr& table_schema, + const std::map& options) { + return CreateFromOptions(context, table_schema, options, /*file_system=*/nullptr, + /*path=*/std::nullopt); +} + +Result> InternalReadContext::CreateFromOptions( const std::shared_ptr& context, const std::shared_ptr& table_schema, const std::map& options, - const std::shared_ptr& file_system, const std::string* path) { + const std::shared_ptr& file_system, std::optional path) { // The file system the caller named, or null for the read context's own, which callers // without a table hand over. The scheme map the context carries does not take part: a file // system handed over wins and one missing falls to the resolving one, the same way a scan @@ -297,7 +304,7 @@ Result> InternalReadContext::Create( CoreOptions::FromMap( options, file_system != nullptr ? file_system : context->GetSpecificFileSystem())); core_options.WithCache(context->GetCache()); - return CreateFromCoreOptions(context, table_schema, std::move(core_options), path); + return CreateFromCoreOptions(context, table_schema, std::move(core_options), std::move(path)); } Result> InternalReadContext::CreateForFallbackBranch( @@ -341,12 +348,12 @@ Result> InternalReadContext::CreateForFallb } else { options.erase(Options::BRANCH); } - return Create(context, table_schema, options, file_system, &root_path); + return CreateFromOptions(context, table_schema, options, file_system, root_path); } Result> InternalReadContext::CreateFromCoreOptions( const std::shared_ptr& context, const std::shared_ptr& table_schema, - CoreOptions core_options, const std::string* path) { + CoreOptions core_options, std::optional path) { if (context->HasVectorOrFullTextSearch()) { if (!core_options.FileIndexReadEnabled()) { return Status::Invalid( @@ -458,8 +465,7 @@ Result> InternalReadContext::CreateFromCore } std::shared_ptr arrow_pool = GetArrowPool(context->GetMemoryPool()); auto internal_context = std::unique_ptr(new InternalReadContext( - context, table_schema, read_schema, core_options, arrow_pool, - path != nullptr ? std::optional(*path) : std::nullopt)); + context, table_schema, read_schema, core_options, std::move(path), arrow_pool)); PAIMON_RETURN_NOT_OK(internal_context->BindIndexScores()); return internal_context; } @@ -539,8 +545,8 @@ InternalReadContext::InternalReadContext(const std::shared_ptr& rea const std::shared_ptr& table_schema, const std::shared_ptr& read_schema, const CoreOptions& options, - const std::shared_ptr& arrow_pool, - std::optional path) + std::optional path, + const std::shared_ptr& arrow_pool) : read_context_(read_context), table_schema_(table_schema), read_schema_(read_schema), @@ -560,7 +566,7 @@ Result> InternalReadContext::CreateWithSche // The new read_schema is the minimal column set for COUNT(*). return std::shared_ptr( new InternalReadContext(original->read_context_, original->table_schema_, new_read_schema, - original->options_, original->arrow_pool_, original->path_)); + original->options_, original->path_, original->arrow_pool_)); } } // namespace paimon diff --git a/src/paimon/core/operation/internal_read_context.h b/src/paimon/core/operation/internal_read_context.h index 43783e950..9e5d91ce0 100644 --- a/src/paimon/core/operation/internal_read_context.h +++ b/src/paimon/core/operation/internal_read_context.h @@ -58,24 +58,23 @@ class InternalReadContext { std::string field_name; }; - /// Builds the context of a read a table serves: the options are the table's own, already - /// parsed, and the cache the context carries is armed on them here. `path` names the root path - /// the table was opened with, which the read reads from in place of the context's own. - static Result> Create( + /// Builds the context of the read a table serves: the options are the table's own, already + /// parsed, and the cache the context carries is armed on them here. + /// @param root_path The root path the table was opened with. The read reads from it in place of + /// the path the context names. + static Result> CreateForTable( const std::shared_ptr& read_context, const std::shared_ptr& table_schema, CoreOptions core_options, - const std::string* path = nullptr); + const std::string& root_path); /// Builds the context of a read whose options are composed on the spot rather than the table's - /// own - the fallback branch of a read, whose schema belongs to another branch, and the reads a - /// write or a lookup serve inside themselves. The read's own file system is `file_system`, or - /// the context's when null; the scheme map the context carries does not take part. + /// own: the reads a write or a lookup serve inside themselves. Such a read serves the file + /// system the context resolved and reads from the path the context names. A fallback branch + /// reads through `CreateForFallbackBranch` instead, whose schema belongs to another branch. static Result> Create( const std::shared_ptr& read_context, const std::shared_ptr& table_schema, - const std::map& options, - const std::shared_ptr& file_system = nullptr, - const std::string* path = nullptr); + const std::map& options); /// The read context of a fallback branch, whose schema is not the table's own: it is read /// through a manager of that branch, and the read serves the options that schema was loaded @@ -195,16 +194,24 @@ class InternalReadContext { InternalReadContext(const std::shared_ptr& read_context, const std::shared_ptr& table_schema, const std::shared_ptr& read_schema, - const CoreOptions& options, - const std::shared_ptr& arrow_pool, - std::optional path = std::nullopt); + const CoreOptions& options, std::optional path, + const std::shared_ptr& arrow_pool); + + /// The tail of a read whose options are a map: they resolve to `CoreOptions` against + /// `file_system`, or against the context's own when it is null, and then go where + /// `CreateFromCoreOptions` takes them. + static Result> CreateFromOptions( + const std::shared_ptr& read_context, + const std::shared_ptr& table_schema, + const std::map& options, + const std::shared_ptr& file_system, std::optional path); /// The tail both `Create` entries share: the options arrive parsed and cached, and the /// validations and the read schema below are the same whichever way they were resolved. static Result> CreateFromCoreOptions( const std::shared_ptr& read_context, const std::shared_ptr& table_schema, CoreOptions core_options, - const std::string* path); + std::optional path); static std::optional TryResolveSpecialFieldById(int32_t field_id, const CoreOptions& core_options, @@ -224,8 +231,8 @@ class InternalReadContext { std::shared_ptr arrow_pool_; CoreOptions options_; std::map index_score_bindings_; - /// The root path a table reads through, or null for the read context's own, which callers - /// without a table hand over. + /// The root path a table reads through, or empty for the read context's own, which a caller + /// without a table leaves it as. std::optional path_; }; diff --git a/src/paimon/core/table/append_only_file_store_table.h b/src/paimon/core/table/append_only_file_store_table.h index 9b4af46ae..1f94d580a 100644 --- a/src/paimon/core/table/append_only_file_store_table.h +++ b/src/paimon/core/table/append_only_file_store_table.h @@ -43,7 +43,7 @@ class AppendOnlyFileStoreTable : public FileStoreTable { /// The constructor `FileStoreTable::Create` builds an append-only table through: the store /// carries every state, and the base class holds it. AppendOnlyFileStoreTable(const std::shared_ptr& store, const Identifier& identifier, - const std::string& uuid = ""); + const std::string& uuid); private: Result> CreateWrite( diff --git a/src/paimon/core/table/file_store.h b/src/paimon/core/table/file_store.h index e343ec2c4..339dd3810 100644 --- a/src/paimon/core/table/file_store.h +++ b/src/paimon/core/table/file_store.h @@ -46,51 +46,47 @@ class ManifestFile; class ManifestList; class TableSchema; -/// The storage a managed table is served through: the file system, root path, schema, branch, -/// options and managers it was loaded with - the state Java Paimon's `AbstractFileStore` carries, -/// parsed once here rather than per operator. `FileStoreTable` hands the state it was loaded with -/// to one of these and asks it back for the pieces every operator opens with. +/// Holds the storage a managed table reads and writes through: the file system, root path, +/// schema, branch, options and managers. Java Paimon carries the same state in `AbstractFileStore`. +/// `FileStoreTable` builds one when it loads and asks it for the pieces each operator opens with. /// -/// Every operator serves the options the table was loaded with, so the factories below take the -/// options of the call: a cache is not part of the table's, `WithCache()` arms a copy an operator -/// makes. They build a fresh instance on every call, never a cached one - a snapshot manager a -/// write arms with a catalog loader is not one a scan should be served through afterwards. +/// The factories below take the options of the call, so an operator can serve a cache of its own +/// over the options the table was loaded with. Each call builds a fresh instance and never a +/// cached one: a snapshot manager a write armed with a catalog loader must not serve a scan. class FileStore { public: - /// @param file_system The file system the caller opened the table through, which wins over - /// the access configuration the schema's options may carry when it is not null. A null - /// one hands the resolution to that configuration, and to the default local file system - /// when the options name none. + /// Builds the store. It parses the schema's options and fails when they do not parse, so the + /// constructor is private and callers come here. + /// @param file_system The file system the caller opened the table through. It wins over the + /// access configuration in the schema's options. Null leaves the resolution to that + /// configuration, and to the default local file system when the options name none. /// @param root_path Root path of the table in the file system. - /// @param schema The schema the table was loaded with, whose partition keys name the partitions - /// the paths below are built for and whose options `GetOptions()` parses. Null when the + /// @param schema The schema the table was loaded with. Its partition keys name the partitions + /// the paths below are built for, and `GetOptions()` parses its options. Null when the /// caller had none, which the operators then report. /// @param branch The branch the store serves, normalized here. - /// @param schema_manager A schema manager to carry instead of building one - the one a context + /// @param schema_manager A schema manager to carry instead of building one: the one a context /// dispatch read the schema through, whose cache the operators then serve. - /// @param snapshot_manager A snapshot manager to carry instead of building one; a null one is - /// built over `root_path` on `branch`. + /// @param snapshot_manager A snapshot manager to carry instead of building one. Null builds one + /// over `root_path` on `branch`. /// @param fs_scheme_map The scheme-to-identifier map the caller's contexts resolve custom /// schemes through, kept so a copy resolves as this store does. - /// - /// Building the store parses the schema's options, so it fails when they do not parse - which - /// is why a store is built through this factory rather than the constructor. static Result> Create( const std::shared_ptr& file_system, const std::string& root_path, const std::shared_ptr& schema, const std::string& branch, - const std::shared_ptr& schema_manager = nullptr, - const std::shared_ptr& snapshot_manager = nullptr, - const std::map& fs_scheme_map = {}); - - /// Whether a copy's dynamic options change the file system its store resolves to. If they do, - /// the managers of the table it was copied from are not its to inherit: built over that - /// table's resolution, they would keep reading metadata through the backend the copy just - /// left. + const std::shared_ptr& schema_manager, + const std::shared_ptr& snapshot_manager, + const std::map& fs_scheme_map); + + /// Returns whether a copy's dynamic options change the file system its store resolves to. When + /// they do, the copy must not inherit the managers of the table it was copied from: they were + /// built over that table's resolution and would keep reading metadata through the backend the + /// copy left. static bool FileSystemResolutionMayChange( const TableSchema& schema, const std::map& dynamic_options); - /// Whether two options maps resolve to the same file system: every resolution key has to - /// agree, a missing one on both sides included. + /// Returns whether two options maps resolve to the same file system. Every resolution key has + /// to agree, including a key missing on both sides. static bool FileSystemResolutionAgrees(const std::map& left, const std::map& right); @@ -98,8 +94,8 @@ class FileStore { return branch_; } - /// The schema manager the operators read through, carrying the schema cache of `root_path` on - /// `CurrentBranch()`. + /// The schema manager the operators read through. It carries the schema cache of `root_path` + /// on `CurrentBranch()`. const std::shared_ptr& GetSchemaManager() const { return schema_manager_; } @@ -115,20 +111,20 @@ class FileStore { return schema_; } - /// The file system the table's data is accessed through: the one the options resolve, which - /// the file system the caller opened the table through wins over. + /// The file system the table's data is accessed through: the one the options resolve, unless + /// the caller opened the table through one of its own. const std::shared_ptr& GetFileSystem() const { return file_system_; } - /// The file system the caller opened the table through, as handed to `Create`: null when the + /// The file system the caller opened the table through, as handed to `Create`. Null when the /// call had none, so a copy resolves from its own options again. const std::shared_ptr& GetSpecifiedFileSystem() const { return specified_file_system_; } - /// The scheme-to-identifier map the options were resolved with, kept for a copy to resolve - /// through again. + /// The scheme-to-identifier map the options were resolved with, kept so a copy can resolve + /// through it again. const std::map& GetFsSchemeMap() const { return fs_scheme_map_; } @@ -137,59 +133,59 @@ class FileStore { return root_path_; } - /// The options the schema was loaded with on `CurrentBranch()`. They exist only over a schema: - /// an operator reads them after reporting that none was found when there is not. + /// The options of the schema loaded on `CurrentBranch()`. They exist only over a schema, so an + /// operator reports that no schema was found before it ever reads them. const CoreOptions& GetOptions() const { assert(core_options_.has_value()); return *core_options_; } - /// The path factory an operator builds its metadata paths through: over this store's root - /// path and the partition keys of its schema, which no schema change can alter, so an - /// operator serving a schema read after the table was loaded still names the same partitions. - /// The file format identifier is the caller's to name: a write names the format its level-0 - /// files are written in, which the options may set per level, while a scan, a read and a - /// commit name the table's format. + /// Builds the path factory an operator names its metadata paths through, over this store's + /// root path and the partition keys of its schema. No schema change can alter the partition + /// keys, so an operator serving a schema read after the table was loaded still names the same + /// partitions. The caller names the file format identifier: a write names the format of its + /// level-0 files, which the options may set per level, while a scan, a read and a commit name + /// the table's format. Result> NewPathFactory( const std::shared_ptr& arrow_schema, const CoreOptions& options, const std::string& file_format_identifier, const std::shared_ptr& memory_pool) const; - /// The snapshot manager a write or a commit reads snapshots through, over this store's root - /// path and branch. The file system is the caller's to name, from the options its operator - /// serves. A catalog that version-manages the table hands its snapshots out itself, so its - /// loader is attached and the catalog captured to keep it alive; such a catalog names the - /// table by an identifier, which the context must have supplied. + /// Builds the snapshot manager a write or a commit reads snapshots through, over this store's + /// root path and branch. The caller names the file system, from the options its operator + /// serves. A catalog that version-manages the table hands out its own snapshots, so its loader + /// is attached and the catalog is captured to keep it alive. Such a catalog names the table by + /// an identifier, which the context must have supplied. Result> NewSnapshotManager( const CoreOptions& options, const std::shared_ptr& catalog, const std::optional& catalog_identifier) const; - /// The manifest list a scan or a commit reads the table's manifest entries through. + /// Builds the manifest list a scan or a commit reads the table's manifest entries through. Result> NewManifestList( const CoreOptions& options, const std::shared_ptr& path_factory, const std::shared_ptr& memory_pool) const; - /// The manifest file a scan or a commit reads a single manifest's entries through, pruned by - /// `partition_schema`. + /// Builds the manifest file a scan or a commit reads a single manifest's entries through, + /// pruned by `partition_schema`. Result> NewManifestFile( const CoreOptions& options, const std::shared_ptr& path_factory, const std::shared_ptr& partition_schema, const std::shared_ptr& memory_pool) const; - /// The index manifest file a write, a scan or a commit reads the table's index manifest - /// through. + /// Builds the index manifest file a write, a scan or a commit reads the table's index + /// manifest through. Result> NewIndexManifestFile( const CoreOptions& options, const std::shared_ptr& path_factory, const std::shared_ptr& memory_pool) const; - /// The index file handler a write maintains its deletion vectors and indexes through, or a - /// scan reads them through. + /// Builds the index file handler a write maintains its deletion vectors and indexes through, + /// or a scan reads them through. Result> NewIndexFileHandler( const CoreOptions& options, const std::shared_ptr& path_factory, const std::shared_ptr& memory_pool) const; - /// The partition computer a commit names the partitions it writes with, over the partition - /// keys of this store's schema and the caller's arrow schema. + /// Builds the partition computer a commit names the partitions it writes with, over the + /// partition keys of this store's schema and the caller's arrow schema. Result> NewPartitionComputer( const std::shared_ptr& arrow_schema, const CoreOptions& options, const std::shared_ptr& memory_pool) const; diff --git a/src/paimon/core/table/file_store_table.cpp b/src/paimon/core/table/file_store_table.cpp index c7bc8261b..0f9e0ed34 100644 --- a/src/paimon/core/table/file_store_table.cpp +++ b/src/paimon/core/table/file_store_table.cpp @@ -216,7 +216,7 @@ Result> FileStoreTable::SwitchToBranch(const std::string& std::shared_ptr(std::move(branch_schema)), identifier_, CatalogUuid().value_or(""), target_branch, branch_schema_manager, /*snapshot_manager=*/nullptr, - store_->GetFsSchemeMap()); + store_->GetFsSchemeMap(), /*dynamic_options=*/{}); } Status FileStoreTable::RestoreRealtimeCommittedProgress( @@ -472,8 +472,8 @@ Result> FileStoreTable::NewRead( PAIMON_RETURN_NOT_OK(CheckOptionsAgreeWithTable(*loaded_schema, read_context->GetOptions())); // The read reads through the branch the table serves, not the context's own. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr internal_context, - InternalReadContext::Create(read_context, loaded_schema, - store_->GetOptions(), &root_path_)); + InternalReadContext::CreateForTable(read_context, loaded_schema, + store_->GetOptions(), root_path_)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, CreateTableRead(internal_context, read_context->GetMemoryPool(), diff --git a/src/paimon/core/table/file_store_table.h b/src/paimon/core/table/file_store_table.h index 1fb307495..04b088ab8 100644 --- a/src/paimon/core/table/file_store_table.h +++ b/src/paimon/core/table/file_store_table.h @@ -43,44 +43,44 @@ class RealtimeContext; class Schema; class SplitGenerator; -/// The managed table a catalog hands out: one with snapshots and manifests, which the schema under -/// its root path describes. A format table - a directory of plain data files - is a `FormatTable`, -/// the other kind of `Table`. +/// The managed table a catalog hands out: one with snapshots and manifests, described by the +/// schema under its root path. A format table - a directory of plain data files - is a +/// `FormatTable`, the other kind of `Table`. /// -/// This is the flow the four operators share, Java Paimon's `AbstractFileStoreTable`: what the -/// table was constructed with - the schema, file system, root path, branch and managers - decides -/// what every operator serves, so it is read here once and handed down. The schema's primary keys -/// decide the kind of managed table it is - `AppendOnlyFileStoreTable` or -/// `PrimaryKeyFileStoreTable`, the `Create` factory below - and that kind builds the pieces the -/// flows hand down: the write itself, the scans, the read and the split generator. +/// The four operators share one flow, the one Java Paimon's `AbstractFileStoreTable` carries. What +/// the table was loaded with - schema, file system, root path, branch and managers - decides what +/// every operator serves, so it is read once here and handed down. The schema's primary keys decide +/// the kind of managed table it is, `AppendOnlyFileStoreTable` or `PrimaryKeyFileStoreTable`, which +/// the `Create` factory below picks. That kind then builds the pieces the flow hands down: the +/// write itself, the scans, the read and the split generator. class FileStoreTable : public Table { public: - /// The managed table the schema describes, built as the kind its primary keys name: an - /// `AppendOnlyFileStoreTable` when there are none, a `PrimaryKeyFileStoreTable` when there - /// are, the way Java Paimon's `FileStoreTableFactory` does. Primary keys cannot be added or - /// dropped by a schema change, so the kind is the same for every schema the root path held. - /// @param schema The schema the table was loaded with, whose primary keys name the kind and - /// whose options the operators serve. Null when the caller had none, in which case the - /// table is built as the append-only kind and its operators report that none was found. + /// Builds the managed table the schema describes, as the kind its primary keys name: an + /// `AppendOnlyFileStoreTable` when there are none, a `PrimaryKeyFileStoreTable` when there are. + /// Java Paimon's `FileStoreTableFactory` picks the kind the same way. A schema change can + /// neither add nor drop primary keys, so every schema the root path ever held names one kind. + /// @param schema The schema the table was loaded with. Its primary keys name the kind and its + /// options are what the operators serve. Null when the caller had none, in which case + /// the table is built as the append-only kind and its operators report that none was + /// found. /// @param branch The branch the table serves, which the managers below read and write on. - /// @param schema_manager A schema manager to carry instead of building one - see + /// @param schema_manager A schema manager to carry instead of building one. See /// `FileStore::Create`. - /// @param snapshot_manager A snapshot manager to carry instead of building one, the way `Copy` - /// hands its table's over so both share one - see `FileStore::Create`. + /// @param snapshot_manager A snapshot manager to carry instead of building one. `Copy` hands + /// its table's over so both share one. See `FileStore::Create`. /// @param fs_scheme_map The scheme-to-identifier map the caller's contexts resolve custom /// schemes through, which the store's options resolve through as well. `Copy` hands its /// table's map over. /// @param dynamic_options The options a copy was given over the ones its schema stored, which - /// `Copy` accumulates - see the member of the same name. + /// `Copy` accumulates. See the member of the same name. static Result> Create( const std::shared_ptr& file_system, const std::string& root_path, const std::shared_ptr& schema, const Identifier& identifier, - const std::string& uuid = "", - const std::string& branch = BranchManager::DEFAULT_MAIN_BRANCH, - const std::shared_ptr& schema_manager = nullptr, - const std::shared_ptr& snapshot_manager = nullptr, - const std::map& fs_scheme_map = {}, - const std::map& dynamic_options = {}); + const std::string& uuid, const std::string& branch, + const std::shared_ptr& schema_manager, + const std::shared_ptr& snapshot_manager, + const std::map& fs_scheme_map, + const std::map& dynamic_options); const std::string& CurrentBranch() const { return store_->CurrentBranch(); @@ -102,15 +102,15 @@ class FileStoreTable : public Table { return identifier_.GetFullName(); } - /// The file system, root path and identifier stay as they are: they name this table, which an - /// option does not rename. + /// Returns a copy that serves `dynamic_options`. The file system, root path and identifier + /// stay as they are: they name this table, and an option does not rename it. Result> Copy( const std::map& dynamic_options) const override; /// Returns the table of `branch`, as Java Paimon's `switchToBranch` does. The branch switched /// to is served as it is: its schema is read from its own directory, so this table's dynamic - /// options are not carried over. Switching to the branch already served hands back a copy of - /// this table, schema and managers shared, and a branch with no schema does not exist. + /// options are not carried over. Switching to the branch this table already serves hands back a + /// copy of it with the schema and the managers shared. A branch with no schema does not exist. Result> SwitchToBranch(const std::string& branch) const; Result> NewScan(std::unique_ptr context) const override; @@ -124,47 +124,47 @@ class FileStoreTable : public Table { std::unique_ptr context) const override; private: - /// The options a context carries against the ones the table was loaded with. A table serves - /// the options it was loaded with, so an option the context contradicts is refused rather than - /// honoured - a caller that means to change them copies the table. A table a load hands back - /// carries the context's own options merged in - `FileStoreTableFactory` does the merging - so - /// it agrees by construction; the refusal belongs to a table the caller held already, whose - /// load is over. A key the table was not loaded with is no contradiction: it names a runtime - /// setting - a scan mode, a batch size - which is none of the table's business, and is served - /// as given. + /// Checks the options a context carries against the ones the table was loaded with. A table + /// serves the options it was loaded with, so an option the context contradicts is refused: a + /// caller that means to change them copies the table. A table a load hands back carries the + /// context's own options merged in - `FileStoreTableFactory` merges them - so it agrees by + /// construction. The refusal belongs to a table the caller already held, whose load is over. A + /// key the table was not loaded with is no contradiction: it names a runtime setting, a scan + /// mode or a batch size, which is none of the table's business and is served as given. static Status CheckOptionsAgreeWithTable( const TableSchema& table_schema, const std::map& context_options); - /// The read `NewRead` hands back, from the internal context it read the schema into: the path - /// factory comes from the store, and the reader is selected by the schema the read serves - the - /// primary keys it carries - rather than by the table's own kind. + /// Builds the read `NewRead` hands back, from the internal context it read the schema into. The + /// path factory comes from the store. The reader is picked by the schema the read serves - the + /// primary keys it carries - not by the table's own kind. Result> CreateTableRead( const std::shared_ptr& internal_context, const std::shared_ptr& memory_pool, const std::shared_ptr& executor) const; protected: - /// The base constructor the kinds of managed table build through: the store carries every state - /// the kinds hand down - see `Create` for what each piece of it is. + /// The base constructor the kinds of managed table build through. The store carries every state + /// they hand down; see `Create` for what each piece of it is. FileStoreTable(const std::shared_ptr& store, const Identifier& identifier, const std::string& uuid); - /// The committed progress a real-time write restores, from the latest snapshot the snapshot - /// manager reads: where the write picks up after, which both kinds ask for the same way. + /// Restores the committed progress of a real-time write from the latest snapshot the snapshot + /// manager reads. It tells the write where to pick up, and both kinds ask for it the same way. static Status RestoreRealtimeCommittedProgress( const std::shared_ptr& realtime_context, const std::shared_ptr& snapshot_manager, const CoreOptions& options); - /// The scan the table's root path names, which the kinds of file store scan read from. + /// Returns the scan restricted to the table's root path, which the kinds of file store scan + /// read from. static std::unique_ptr WithTablePath(std::unique_ptr&& scan, const std::string& root_path); - /// The write of this kind of table, which `NewWrite` hands the shared flow's pieces to. The - /// schema and the branch the write serves are the table's own. + /// Builds the write of this kind of table. `NewWrite` hands the pieces of the shared flow to + /// it. The schema and the branch the write serves are the table's own. /// @param io_manager The spill manager the context's temp directory names, or null when the /// context named none. - /// @param ignore_previous_files Whether the write starts over rather than appending, which - /// the flow resolved with this kind's own rules. + /// @param ignore_previous_files Whether the write starts over rather than appends, which the + /// flow resolved with this kind's own rules. virtual Result> CreateWrite( const WriteContext& ctx, const std::shared_ptr& arrow_schema, const std::shared_ptr& partition_schema, const CoreOptions& options, @@ -172,8 +172,8 @@ class FileStoreTable : public Table { const std::shared_ptr& snapshot_manager, const std::shared_ptr& io_manager, bool ignore_previous_files) const = 0; - /// The scan of this kind of table, which `NewScan` hands the manifest list and file it built - /// to, over the table's own schema and managers. + /// Builds the scan of this kind of table. `NewScan` hands it the manifest list and the manifest + /// file it built, over the table's own schema and managers. virtual Result> NewFileStoreScan( const std::shared_ptr& manifest_list, const std::shared_ptr& manifest_file, @@ -181,12 +181,12 @@ class FileStoreTable : public Table { const std::shared_ptr& executor, const std::shared_ptr& memory_pool, const ScanContext* context) const = 0; - /// The split generator of this kind of table, Java Paimon's `splitGenerator()`. + /// Builds the split generator of this kind of table, Java Paimon's `splitGenerator()`. virtual Result> NewSplitGenerator( const CoreOptions& core_options) const = 0; - /// The manifest entry scan supplier of this kind of table, which `NewCommit` hands the manifest - /// list and file it built to. + /// Builds the manifest entry scan supplier of this kind of table. `NewCommit` hands it the + /// manifest list and the manifest file it built. virtual CommitScanner::ScanSupplier NewCommitScanSupplier( const std::shared_ptr& snapshot_manager, const std::shared_ptr& manifest_list, @@ -195,13 +195,13 @@ class FileStoreTable : public Table { const std::shared_ptr& executor, const std::shared_ptr& pool) const = 0; - /// The storage this table serves - see `FileStore`. + /// The storage this table reads and writes through. See `FileStore`. std::shared_ptr store_; - /// The dynamic options the table was loaded with over the ones its schema stored, which `Copy` - /// accumulates into the copy it derives. They stay separate from the schema they were merged - /// into because the fallback reader of a read needs them alone: it serves another branch's - /// schema, over which the same options are merged again. + /// The dynamic options the table was loaded with, over the ones its schema stored. `Copy` + /// accumulates them into the copy it derives. They stay separate from the schema they were + /// merged into because a fallback reader needs them on their own: it serves another branch's + /// schema and merges the same options over it again. std::map dynamic_options_; }; diff --git a/src/paimon/core/table/file_store_table_factory.cpp b/src/paimon/core/table/file_store_table_factory.cpp index 2b5c9a40c..d5082ef73 100644 --- a/src/paimon/core/table/file_store_table_factory.cpp +++ b/src/paimon/core/table/file_store_table_factory.cpp @@ -125,7 +125,7 @@ Result> TableOfLoadedSchema( const std::string& branch, const std::map& options, const std::shared_ptr& schema_manager, const std::shared_ptr& latest_schema, const std::optional& identifier, - const std::string& uuid = "", const std::map& fs_scheme_map = {}) { + const std::string& uuid, const std::map& fs_scheme_map) { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, TryLoadFormatTable(file_system, path, branch, options, latest_schema)); if (format_table != nullptr) { @@ -146,10 +146,11 @@ Result> TableOfLoadedSchema( !FileStore::FileSystemResolutionAgrees(latest_schema->Options(), options))) { manager = nullptr; } - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, - FileStoreTable::Create(explicit_file_system, path, latest_schema, - table_identifier, uuid, branch, manager, - /*snapshot_manager=*/nullptr, fs_scheme_map)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr table, + FileStoreTable::Create(explicit_file_system, path, latest_schema, table_identifier, uuid, + branch, manager, /*snapshot_manager=*/nullptr, fs_scheme_map, + /*dynamic_options=*/{})); return WithContextOptions(table, latest_schema, options); } @@ -167,7 +168,7 @@ Result> ManagedTableOf( const std::shared_ptr& explicit_file_system, const std::string& path, const std::string& branch, const std::map& options, const std::optional& schema_json, const std::optional& identifier, - const std::string& uuid = "", const std::map& fs_scheme_map = {}) { + const std::string& uuid, const std::map& fs_scheme_map) { auto schema_manager = std::make_shared(file_system, path, branch); std::shared_ptr latest_schema; if (branch == BranchManager::DEFAULT_MAIN_BRANCH && schema_json) { @@ -230,7 +231,8 @@ Result> FileStoreTableFactory::Create(const ScanContext& const std::string branch = BranchManager::NormalizeBranch(tmp_options.GetBranch()); return ManagedTableOf(tmp_options.GetFileSystem(), ctx.GetSpecificFileSystem(), ctx.GetPath(), branch, ctx.GetOptions(), ctx.GetSpecificTableSchema(), - ctx.GetIdentifier()); + ctx.GetIdentifier(), + /*uuid=*/"", /*fs_scheme_map=*/{}); } /// The table a read context names: the format table the builder was handed, a system table behind @@ -340,7 +342,7 @@ Result> FileStoreTableFactory::Create(const CommitContext // not dispatch on different schemas. return ManagedTableOf(tmp_options.GetFileSystem(), specific_fs, root_path, branch, ctx.GetOptions(), catalog_table_schema, ctx.GetIdentifier(), - ctx.GetTableId().value_or("")); + ctx.GetTableId().value_or(""), /*fs_scheme_map=*/{}); } } // namespace paimon diff --git a/src/paimon/core/table/file_store_table_factory.h b/src/paimon/core/table/file_store_table_factory.h index 175834ff6..a1bb77062 100644 --- a/src/paimon/core/table/file_store_table_factory.h +++ b/src/paimon/core/table/file_store_table_factory.h @@ -30,10 +30,10 @@ class ScanContext; class Table; class WriteContext; -/// The table a context names, Java Paimon's `FileStoreTableFactory`: the format table the builder -/// was handed, a system table behind a `T$system` path, or the table the schema under the -/// context's path says is there. The four operators' static `Create` entries all dispatch on the -/// same question, so one place answers it and they cannot disagree. +/// Loads the table a context names, as Java Paimon's `FileStoreTableFactory` does: the format +/// table the builder was handed, the system table behind a `T$system` path, or the managed table +/// the schema under the context's path describes. The static `Create` of all four operators asks +/// the same question, so one place answers it and they cannot disagree. class FileStoreTableFactory { public: FileStoreTableFactory() = delete; diff --git a/src/paimon/core/table/format/format_table.cpp b/src/paimon/core/table/format/format_table.cpp index d4b611eac..f14ac66d1 100644 --- a/src/paimon/core/table/format/format_table.cpp +++ b/src/paimon/core/table/format/format_table.cpp @@ -224,6 +224,9 @@ Result> FormatTable::GetArrowSchema() const { return schema_->GetArrowSchema(); } +// The refusals below use two codes. `Invalid` is a setting that clashes with what a format table +// is, and will never work: no snapshots, no buckets, no real-time store, no manifests, no +// catalog-side version. `NotImplemented` is a capability this code lacks today and could gain. Result> FormatTable::NewScan( std::unique_ptr context) const { std::shared_ptr shared_context = std::move(context); @@ -233,15 +236,15 @@ Result> FormatTable::NewScan( // Anything a `ScanContext` carries that a format table cannot honour is refused rather than // silently dropped. if (shared_context->IsStreamingMode()) { - return Status::NotImplemented( + return Status::Invalid( "a format table has no snapshots, so there is nothing for a streaming scan to follow"); } if (shared_context->GetRealtimeContext() != nullptr) { - return Status::NotImplemented( + return Status::Invalid( "a format table has no real-time store to union with what is on disk"); } if (shared_context->GetGlobalIndexResult() != nullptr) { - return Status::NotImplemented("a format table carries no global index"); + return Status::Invalid("a format table carries no global index"); } std::map partition_filter; std::shared_ptr filters = shared_context->GetScanFilters(); @@ -253,12 +256,12 @@ Result> FormatTable::NewScan( "implemented yet; give the partition values as a partition filter instead"); } if (filters->GetBucketFilter()) { - return Status::NotImplemented("a format table has no buckets to filter by"); + return Status::Invalid("a format table has no buckets to filter by"); } const std::vector>& partition_filters = filters->GetPartitionFilters(); if (partition_filters.size() > 1) { - return Status::NotImplemented("a format table scan takes at most one partition filter"); + return Status::Invalid("a format table scan takes at most one partition filter"); } if (partition_filters.size() == 1) { partition_filter = partition_filters.front(); @@ -298,12 +301,12 @@ Result> FormatTable::NewWrite( "format table"); } if (context->IsStreamingMode()) { - return Status::NotImplemented( + return Status::Invalid( "a format table has no snapshots, so there is nothing a streaming write could " "commit against"); } if (context->GetRealtimeContext() != nullptr) { - return Status::NotImplemented("a format table has no real-time store to write into"); + return Status::Invalid("a format table has no real-time store to write into"); } if (!context->GetWriteSchema().empty()) { return Status::NotImplemented( @@ -313,8 +316,7 @@ Result> FormatTable::NewWrite( // A write id prefixes a postpone-bucket writer's files so that one compaction reader can // put them back in order. A format table has no buckets, so it would identify nothing. if (context->GetWriteId().has_value()) { - return Status::NotImplemented( - "a format table has no buckets, so a write id would name nothing"); + return Status::Invalid("a format table has no buckets, so a write id would name nothing"); } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr format_write, FormatTableFileStoreWrite::Create(table, context->GetMemoryPool())); @@ -327,26 +329,26 @@ Result> FormatTable::NewCommit( PAIMON_ASSIGN_OR_RAISE(std::shared_ptr given_table, CopySelf(context->GetOptions())); if (!context->IgnoreEmptyCommit()) { - return Status::NotImplemented( + return Status::Invalid( "a format table cannot record an empty commit: keeping one means writing a " "snapshot that adds no files, and there are no snapshots here"); } if (context->UseRESTCatalogCommit()) { - return Status::NotImplemented( + return Status::Invalid( "a format table commits by renaming files into place, not by sending a snapshot " "to a rest catalog"); } if (context->GetTableId()) { - return Status::NotImplemented( + return Status::Invalid( "a format table sends no commit table request, so there is nowhere to put a table id"); } if (context->GetCatalog() != nullptr) { - return Status::NotImplemented( + return Status::Invalid( "a format table keeps no snapshot for a catalog to take, so it commits by renaming " "files into place rather than through the catalog it was loaded from"); } if (context->AppendCommitCheckConflict()) { - return Status::NotImplemented( + return Status::Invalid( "a format table has no manifests to check a concurrent commit against"); } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr format_commit, diff --git a/src/paimon/core/table/format/format_table_test.cpp b/src/paimon/core/table/format/format_table_test.cpp index 2dac6c102..84fbce351 100644 --- a/src/paimon/core/table/format/format_table_test.cpp +++ b/src/paimon/core/table/format/format_table_test.cpp @@ -2766,6 +2766,35 @@ TEST(FormatTableTest, TestTheGenericEntryPointsReachAFormatTable) { reader->Close(); } +TEST(FormatTableTest, TestACatalogInTheContextIsRefusedAlikeByWriteAndCommit) { + // `WithCatalog()` asks for a catalog-side version of the commit, which a format table has no + // way to produce. Both halves of the write path answer with the same code, so a caller reads + // one kind of failure out of them rather than "cannot be done" from one and "not yet" from + // the other. + std::unique_ptr dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN(std::shared_ptr table, + CreateTable(dir->GetFileSystem(), dir->Str(), {"dt"})); + + auto catalog = std::make_shared(); + catalog->SetTableSchema(table->LatestSchema()); + Identifier identifier("db", "fmt"); + + WriteContextBuilder write_builder(dir->Str(), "test-user"); + write_builder.WithCatalog(catalog, identifier); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); + Result> write = table->NewWrite(std::move(write_context)); + ASSERT_FALSE(write.ok()); + ASSERT_TRUE(write.status().IsInvalid()); + + CommitContextBuilder commit_builder(dir->Str(), "test-user"); + commit_builder.WithCatalog(catalog, identifier); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); + Result> commit = table->NewCommit(std::move(commit_context)); + ASSERT_FALSE(commit.ok()); + ASSERT_TRUE(commit.status().IsInvalid()); +} + TEST(FormatTableTest, TestTheGenericEntryPointsTakeALoadedFormatTable) { // A table a catalog keeps the schema for, the way a rest catalog does. Nothing under its // location says that it is a format table, nor that everything below the location is data, so diff --git a/src/paimon/core/table/primary_key_file_store_table.h b/src/paimon/core/table/primary_key_file_store_table.h index ce7547a71..2d3390e86 100644 --- a/src/paimon/core/table/primary_key_file_store_table.h +++ b/src/paimon/core/table/primary_key_file_store_table.h @@ -43,7 +43,7 @@ class PrimaryKeyFileStoreTable : public FileStoreTable { /// The constructor `FileStoreTable::Create` builds a primary key table through: the store /// carries every state, and the base class holds it. PrimaryKeyFileStoreTable(const std::shared_ptr& store, const Identifier& identifier, - const std::string& uuid = ""); + const std::string& uuid); private: Result> CreateWrite( diff --git a/src/paimon/core/table/source/table_read_test.cpp b/src/paimon/core/table/source/table_read_test.cpp index 3cd3b22de..3e79c537d 100644 --- a/src/paimon/core/table/source/table_read_test.cpp +++ b/src/paimon/core/table/source/table_read_test.cpp @@ -137,7 +137,10 @@ TEST(TableReadTest, TestFullTextMatchExecutionIsNotImplemented) { ASSERT_OK_AND_ASSIGN(std::shared_ptr schema, schema_manager.ReadSchema(0)); ASSERT_OK_AND_ASSIGN( std::shared_ptr table, - FileStoreTable::Create(fs, table_path, schema, Identifier("append_09.db", "append_09"))); + FileStoreTable::Create(fs, table_path, schema, Identifier("append_09.db", "append_09"), + /*uuid=*/"", /*branch=*/BranchManager::DEFAULT_MAIN_BRANCH, + /*schema_manager=*/nullptr, /*snapshot_manager=*/nullptr, + /*fs_scheme_map=*/{}, /*dynamic_options=*/{})); for (bool table_entry : {false, true}) { ReadContextBuilder builder(table_path); builder.SetReadFieldNames({"f0", "f1"}).SetPredicate(predicate); @@ -164,7 +167,10 @@ TEST(TableReadTest, TestPerFieldScoreExecutionIsNotImplemented) { ASSERT_OK_AND_ASSIGN(std::shared_ptr schema, schema_manager.ReadSchema(0)); ASSERT_OK_AND_ASSIGN( std::shared_ptr table, - FileStoreTable::Create(fs, path, schema, Identifier("append_09.db", "append_09"))); + FileStoreTable::Create(fs, path, schema, Identifier("append_09.db", "append_09"), + /*uuid=*/"", /*branch=*/BranchManager::DEFAULT_MAIN_BRANCH, + /*schema_manager=*/nullptr, /*snapshot_manager=*/nullptr, + /*fs_scheme_map=*/{}, /*dynamic_options=*/{})); for (bool table_entry : {false, true}) { ReadContextBuilder builder(path); builder.SetFullTextSearch(search).SetReadFieldNames({"f1", "_f0_INDEX_SCORE"}); diff --git a/src/paimon/core/table/system/audit_log_system_table.cpp b/src/paimon/core/table/system/audit_log_system_table.cpp index 09fd6328f..0bf82ead3 100644 --- a/src/paimon/core/table/system/audit_log_system_table.cpp +++ b/src/paimon/core/table/system/audit_log_system_table.cpp @@ -359,16 +359,19 @@ std::unique_ptr CreateChangelogBatchReader( AuditLogSystemTable::AuditLogSystemTable(std::shared_ptr fs, std::string table_path, std::shared_ptr table_schema, std::map options) - : SystemTable(fs, table_path), + : AuditLogSystemTable(std::move(fs), table_path, std::move(table_schema), std::move(options), + ComposeName(table_path, kName)) {} + +AuditLogSystemTable::AuditLogSystemTable(std::shared_ptr fs, std::string table_path, + std::shared_ptr table_schema, + std::map options, + std::string name) + : SystemTable(fs, table_path, std::move(name)), fs_(std::move(fs)), table_path_(std::move(table_path)), table_schema_(std::move(table_schema)), options_(std::move(options)) {} -std::string AuditLogSystemTable::Name() const { - return kName; -} - Result> AuditLogSystemTable::ArrowSchema() const { std::shared_ptr row_kind_field = DataField::ConvertDataFieldToArrowField(SpecialFields::RowKind()); @@ -398,6 +401,8 @@ Result> AuditLogSystemTable::NewScan( .WithMemoryPool(context->GetMemoryPool()) .WithExecutor(context->GetExecutor()) // The view's own file system when the context names none, rather than the default backend. + // A `ScanContext` carries no scheme-to-identifier map, so unlike `CreateDataReadContext` + // below there is no map here to resolve through instead. .WithFileSystem(context->GetSpecificFileSystem() != nullptr ? context->GetSpecificFileSystem() : GetFileSystem()) diff --git a/src/paimon/core/table/system/audit_log_system_table.h b/src/paimon/core/table/system/audit_log_system_table.h index 7395c9797..4cc324b1d 100644 --- a/src/paimon/core/table/system/audit_log_system_table.h +++ b/src/paimon/core/table/system/audit_log_system_table.h @@ -58,12 +58,17 @@ class AuditLogSystemTable : public SystemTable { std::shared_ptr table_schema, std::map options); - std::string Name() const override; Result> ArrowSchema() const override; Result> NewScan(std::unique_ptr context) const override; Result> NewRead(std::unique_ptr context) const override; protected: + /// For a derived changelog view that answers with a kind of its own: `BinlogSystemTable` names + /// itself `T$binlog`, not the `T$audit_log` the public constructor composes. + AuditLogSystemTable(std::shared_ptr fs, std::string table_path, + std::shared_ptr table_schema, + std::map options, std::string name); + Result> NewChangelogRead( const std::shared_ptr& context, std::shared_ptr converter) const; diff --git a/src/paimon/core/table/system/binlog_system_table.cpp b/src/paimon/core/table/system/binlog_system_table.cpp index a3c39078d..8299f17f5 100644 --- a/src/paimon/core/table/system/binlog_system_table.cpp +++ b/src/paimon/core/table/system/binlog_system_table.cpp @@ -74,12 +74,8 @@ std::shared_ptr CreateBinlogBatchConverter() { BinlogSystemTable::BinlogSystemTable(std::shared_ptr fs, std::string table_path, std::shared_ptr table_schema, std::map options) - : AuditLogSystemTable(std::move(fs), std::move(table_path), std::move(table_schema), - std::move(options)) {} - -std::string BinlogSystemTable::Name() const { - return kName; -} + : AuditLogSystemTable(std::move(fs), table_path, std::move(table_schema), std::move(options), + ComposeName(table_path, kName)) {} Result> BinlogSystemTable::ArrowSchema() const { std::shared_ptr rowkind_field = diff --git a/src/paimon/core/table/system/binlog_system_table.h b/src/paimon/core/table/system/binlog_system_table.h index 3a939fcef..3a28ffa93 100644 --- a/src/paimon/core/table/system/binlog_system_table.h +++ b/src/paimon/core/table/system/binlog_system_table.h @@ -40,7 +40,6 @@ class BinlogSystemTable : public AuditLogSystemTable { std::shared_ptr table_schema, std::map options); - std::string Name() const override; Result> ArrowSchema() const override; Result> NewRead(std::unique_ptr context) const override; }; diff --git a/src/paimon/core/table/system/global_system_tables.cpp b/src/paimon/core/table/system/global_system_tables.cpp index a66362b39..92f10c49d 100644 --- a/src/paimon/core/table/system/global_system_tables.cpp +++ b/src/paimon/core/table/system/global_system_tables.cpp @@ -126,6 +126,18 @@ Result IsEnabled(const GlobalSystemTableRegistryEntry& entry, CatalogOptionsSystemTable::kEnabledOption, false); } +// Whether a table reads its rows by enumerating what the catalog holds, and so cannot be built +// without one. +bool RequiresCatalog(const std::string& table_name) { + std::string normalized = StringUtils::ToLowerCase(table_name); + for (const auto& entry : GlobalSystemTableRegistry()) { + if (entry.name == normalized) { + return entry.requires_catalog; + } + } + return false; +} + struct CatalogTableInfo { std::string database_name; std::string table_name; @@ -134,11 +146,16 @@ struct CatalogTableInfo { // Match Java CatalogUtils::listAllTables: tolerate databases or tables removed concurrently, but // propagate all other catalog and schema errors instead of returning incomplete system-table rows. -Result> LoadAllDataTables(const Catalog& catalog) { +// `table_name` names the table asking, for the refusal a missing catalog reports. +Result> LoadAllDataTables( + const std::shared_ptr& catalog, const std::string& table_name) { + if (catalog == nullptr) { + return Status::NotImplemented("global system table requires catalog context: ", table_name); + } std::vector result; - PAIMON_ASSIGN_OR_RAISE(std::vector databases, catalog.ListDatabases()); + PAIMON_ASSIGN_OR_RAISE(std::vector databases, catalog->ListDatabases()); for (const std::string& database : databases) { - Result> tables_result = catalog.ListTables(database); + Result> tables_result = catalog->ListTables(database); if (!tables_result.ok()) { if (tables_result.status().IsNotExist()) { continue; @@ -147,7 +164,7 @@ Result> LoadAllDataTables(const Catalog& catalog) } for (const std::string& table : tables_result.value()) { Identifier identifier(database, table); - Result> schema_result = catalog.LoadTableSchema(identifier); + Result> schema_result = catalog->LoadTableSchema(identifier); if (!schema_result.ok()) { if (schema_result.status().IsNotExist()) { continue; @@ -288,6 +305,16 @@ Result GlobalSystemTableLoader::IsSupported( } Result> GlobalSystemTableLoader::Load( + const std::string& table_name, const GlobalSystemTableContext& context) { + // Refused here rather than at the first read: a table handed back to a caller that can never + // build its rows is a failure the caller finds out about late and far from its cause. + if (context.catalog == nullptr && RequiresCatalog(table_name)) { + return Status::NotImplemented("global system table requires catalog context: ", table_name); + } + return LoadForSchema(table_name, context); +} + +Result> GlobalSystemTableLoader::LoadForSchema( const std::string& table_name, const GlobalSystemTableContext& context) { std::string normalized = StringUtils::ToLowerCase(table_name); for (const auto& entry : GlobalSystemTableRegistry()) { @@ -296,10 +323,6 @@ Result> GlobalSystemTableLoader::Load( if (!enabled) { return Status::NotExist("global system table is disabled: ", table_name); } - if (entry.requires_catalog && context.catalog == nullptr) { - return Status::NotImplemented("global system table requires catalog context: ", - table_name); - } PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, entry.factory(context)); // The table is handed out through the `Table` interface, which serves the schema @@ -330,11 +353,7 @@ Result> GlobalSystemTableLoader::GetSupportedTableNames // ============================================================================= CatalogOptionsSystemTable::CatalogOptionsSystemTable(GlobalSystemTableContext context) - : InMemorySystemTable(context.fs, "sys/catalog_options"), context_(std::move(context)) {} - -std::string CatalogOptionsSystemTable::Name() const { - return kName; -} + : InMemorySystemTable(context.fs, "sys/catalog_options", kName), context_(std::move(context)) {} Result> CatalogOptionsSystemTable::ArrowSchema() const { return arrow::schema({ @@ -366,11 +385,8 @@ Result> CatalogOptionsSystemTable::BuildRows() const { // ============================================================================= AllTableOptionsSystemTable::AllTableOptionsSystemTable(GlobalSystemTableContext context) - : InMemorySystemTable(context.fs, "sys/all_table_options"), context_(std::move(context)) {} - -std::string AllTableOptionsSystemTable::Name() const { - return kName; -} + : InMemorySystemTable(context.fs, "sys/all_table_options", kName), + context_(std::move(context)) {} Result> AllTableOptionsSystemTable::ArrowSchema() const { return arrow::schema({ @@ -386,7 +402,7 @@ Result> AllTableOptionsSystemTable::BuildRows() const { std::vector rows; PAIMON_ASSIGN_OR_RAISE(std::vector tables, - LoadAllDataTables(*context_.catalog)); + LoadAllDataTables(context_.catalog, kName)); for (const CatalogTableInfo& table : tables) { for (const auto& [key, value] : table.schema->Options()) { GenericRow row(schema->num_fields()); @@ -405,11 +421,7 @@ Result> AllTableOptionsSystemTable::BuildRows() const { // ============================================================================= TablesSystemTable::TablesSystemTable(GlobalSystemTableContext context) - : InMemorySystemTable(context.fs, "sys/tables"), context_(std::move(context)) {} - -std::string TablesSystemTable::Name() const { - return kName; -} + : InMemorySystemTable(context.fs, "sys/tables", kName), context_(std::move(context)) {} Result> TablesSystemTable::ArrowSchema() const { return arrow::schema({ @@ -435,7 +447,7 @@ Result> TablesSystemTable::BuildRows() const { std::vector rows; PAIMON_ASSIGN_OR_RAISE(std::vector tables, - LoadAllDataTables(*context_.catalog)); + LoadAllDataTables(context_.catalog, kName)); for (const CatalogTableInfo& table : tables) { const std::shared_ptr& data_schema = table.schema; @@ -477,11 +489,7 @@ Result> TablesSystemTable::BuildRows() const { // ============================================================================= PartitionsSystemTable::PartitionsSystemTable(GlobalSystemTableContext context) - : InMemorySystemTable(context.fs, "sys/partitions"), context_(std::move(context)) {} - -std::string PartitionsSystemTable::Name() const { - return kName; -} + : InMemorySystemTable(context.fs, "sys/partitions", kName), context_(std::move(context)) {} Result> PartitionsSystemTable::ArrowSchema() const { return arrow::schema({ @@ -501,7 +509,7 @@ Result> PartitionsSystemTable::BuildRows() const { std::vector rows; PAIMON_ASSIGN_OR_RAISE(std::vector tables, - LoadAllDataTables(*context_.catalog)); + LoadAllDataTables(context_.catalog, kName)); for (const CatalogTableInfo& table : tables) { const std::shared_ptr& data_schema = table.schema; Identifier id(table.database_name, table.table_name); diff --git a/src/paimon/core/table/system/global_system_tables.h b/src/paimon/core/table/system/global_system_tables.h index eadc83af1..280f57479 100644 --- a/src/paimon/core/table/system/global_system_tables.h +++ b/src/paimon/core/table/system/global_system_tables.h @@ -33,14 +33,11 @@ class FileSystem; /// Context passed to global system table constructors, providing catalog-level /// access for enumerating databases, tables, and reading metadata. struct GlobalSystemTableContext { - /// The catalog the table reads its rows through, non-owning: the rows are built lazily, - /// so a table that outlives the call that built it must be handed the shared ownership - /// below too, or this pointer dangles at the first `BuildRows`. - const Catalog* catalog = nullptr; - /// The shared ownership a returned table keeps of its catalog, so its lazy `BuildRows` - /// cannot read through one already destroyed. Empty where the table is transient - used - /// within the call, before the catalog can go away. - std::shared_ptr catalog_retainer; + /// The catalog this table reads its rows through. Rows are built lazily, so a table handed back + /// to a caller keeps shared ownership of it. Empty when the table is transient and only its + /// schema is read. `GlobalSystemTableLoader::Load` refuses a table that enumerates data tables + /// when this is empty. + std::shared_ptr catalog; std::shared_ptr fs; std::string warehouse; std::map catalog_options; @@ -55,7 +52,6 @@ class CatalogOptionsSystemTable : public InMemorySystemTable { explicit CatalogOptionsSystemTable(GlobalSystemTableContext context); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -71,7 +67,6 @@ class AllTableOptionsSystemTable : public InMemorySystemTable { explicit AllTableOptionsSystemTable(GlobalSystemTableContext context); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -87,7 +82,6 @@ class TablesSystemTable : public InMemorySystemTable { explicit TablesSystemTable(GlobalSystemTableContext context); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -103,7 +97,6 @@ class PartitionsSystemTable : public InMemorySystemTable { explicit PartitionsSystemTable(GlobalSystemTableContext context); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -120,9 +113,17 @@ class GlobalSystemTableLoader { static Result IsSupported(const std::string& table_name, const std::map& catalog_options); + /// Loads a global system table for a caller to read. A table that enumerates data tables needs + /// the catalog in the context, so it is refused here when the context carries none. static Result> Load(const std::string& table_name, const GlobalSystemTableContext& context); + /// Loads a global system table for the schema it exposes alone, the way + /// `Catalog::LoadTableSchema` does. No rows are built, so a context that carries no catalog is + /// accepted here and refused by `Load` instead. + static Result> LoadForSchema( + const std::string& table_name, const GlobalSystemTableContext& context); + static Result> GetSupportedTableNames( const std::map& catalog_options); }; diff --git a/src/paimon/core/table/system/in_memory_system_table.cpp b/src/paimon/core/table/system/in_memory_system_table.cpp index d21d6fb4d..87fdd1d29 100644 --- a/src/paimon/core/table/system/in_memory_system_table.cpp +++ b/src/paimon/core/table/system/in_memory_system_table.cpp @@ -107,8 +107,8 @@ class InMemorySystemTableRead : public TableRead { } // namespace InMemorySystemTable::InMemorySystemTable(const std::shared_ptr& fs, - std::string table_path) - : SystemTable(fs, table_path), table_path_(std::move(table_path)) {} + std::string table_path, std::string name) + : SystemTable(fs, table_path, std::move(name)), table_path_(std::move(table_path)) {} Result> InMemorySystemTable::NewScan( std::unique_ptr /*context*/) const { diff --git a/src/paimon/core/table/system/in_memory_system_table.h b/src/paimon/core/table/system/in_memory_system_table.h index 90cd46d84..c72d7389e 100644 --- a/src/paimon/core/table/system/in_memory_system_table.h +++ b/src/paimon/core/table/system/in_memory_system_table.h @@ -36,7 +36,8 @@ class FileSystem; /// It provides the common singleton split scan and one-shot batch reader. class InMemorySystemTable : public SystemTable { public: - InMemorySystemTable(const std::shared_ptr& fs, std::string table_path); + InMemorySystemTable(const std::shared_ptr& fs, std::string table_path, + std::string name); Result> NewScan( std::unique_ptr /*context*/) const override; diff --git a/src/paimon/core/table/system/metadata_system_tables.cpp b/src/paimon/core/table/system/metadata_system_tables.cpp index e9e00ea42..1b5927004 100644 --- a/src/paimon/core/table/system/metadata_system_tables.cpp +++ b/src/paimon/core/table/system/metadata_system_tables.cpp @@ -507,11 +507,8 @@ Result> WriteColsValue( OptionsSystemTable::OptionsSystemTable(const std::shared_ptr& fs, std::string table_path, std::shared_ptr table_schema) - : InMemorySystemTable(fs, table_path), table_schema_(std::move(table_schema)) {} - -std::string OptionsSystemTable::Name() const { - return kName; -} + : InMemorySystemTable(fs, table_path, ComposeName(table_path, kName)), + table_schema_(std::move(table_schema)) {} Result> OptionsSystemTable::ArrowSchema() const { return arrow::schema({arrow::field("key", arrow::utf8(), /*nullable=*/false), @@ -533,13 +530,9 @@ Result> OptionsSystemTable::BuildRows() const { SnapshotsSystemTable::SnapshotsSystemTable(std::shared_ptr fs, std::string table_path, std::string branch) - : InMemorySystemTable(fs, table_path), + : InMemorySystemTable(fs, table_path, ComposeName(table_path, kName)), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch))) {} -std::string SnapshotsSystemTable::Name() const { - return kName; -} - Result> SnapshotsSystemTable::ArrowSchema() const { return arrow::schema({ arrow::field("snapshot_id", arrow::int64(), /*nullable=*/false), @@ -595,13 +588,9 @@ Result> SnapshotsSystemTable::BuildRows() const { SchemasSystemTable::SchemasSystemTable(std::shared_ptr fs, std::string table_path, std::string branch) - : InMemorySystemTable(fs, table_path), + : InMemorySystemTable(fs, table_path, ComposeName(table_path, kName)), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch))) {} -std::string SchemasSystemTable::Name() const { - return kName; -} - Result> SchemasSystemTable::ArrowSchema() const { return arrow::schema({ arrow::field("schema_id", arrow::int64(), /*nullable=*/false), @@ -651,13 +640,9 @@ Result> SchemasSystemTable::BuildRows() const { TagsSystemTable::TagsSystemTable(std::shared_ptr fs, std::string table_path, std::string branch) - : InMemorySystemTable(fs, table_path), + : InMemorySystemTable(fs, table_path, ComposeName(table_path, kName)), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch))) {} -std::string TagsSystemTable::Name() const { - return kName; -} - Result> TagsSystemTable::ArrowSchema() const { return arrow::schema({ arrow::field("tag_name", arrow::utf8(), /*nullable=*/false), @@ -701,13 +686,9 @@ Result> TagsSystemTable::BuildRows() const { BranchesSystemTable::BranchesSystemTable(std::shared_ptr fs, std::string table_path, std::string branch) - : InMemorySystemTable(fs, table_path), + : InMemorySystemTable(fs, table_path, ComposeName(table_path, kName)), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch))) {} -std::string BranchesSystemTable::Name() const { - return kName; -} - Result> BranchesSystemTable::ArrowSchema() const { return arrow::schema({ arrow::field("branch_name", arrow::utf8(), /*nullable=*/false), @@ -740,13 +721,9 @@ Result> BranchesSystemTable::BuildRows() const { ConsumersSystemTable::ConsumersSystemTable(std::shared_ptr fs, std::string table_path, std::string branch) - : InMemorySystemTable(fs, table_path), + : InMemorySystemTable(fs, table_path, ComposeName(table_path, kName)), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch))) {} -std::string ConsumersSystemTable::Name() const { - return kName; -} - Result> ConsumersSystemTable::ArrowSchema() const { return arrow::schema({ arrow::field("consumer_id", arrow::utf8(), /*nullable=*/false), @@ -775,14 +752,10 @@ ManifestsSystemTable::ManifestsSystemTable(std::shared_ptr fs, std:: std::string branch, std::shared_ptr table_schema, std::map options) - : InMemorySystemTable(fs, table_path), + : InMemorySystemTable(fs, table_path, ComposeName(table_path, kName)), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch), std::move(table_schema), std::move(options))) {} -std::string ManifestsSystemTable::Name() const { - return kName; -} - Result> ManifestsSystemTable::ArrowSchema() const { return arrow::schema({ arrow::field("file_name", arrow::utf8(), /*nullable=*/false), @@ -844,14 +817,10 @@ Result> ManifestsSystemTable::BuildRows() const { FilesSystemTable::FilesSystemTable(std::shared_ptr fs, std::string table_path, std::string branch, std::shared_ptr table_schema, std::map options) - : InMemorySystemTable(fs, table_path), + : InMemorySystemTable(fs, table_path, ComposeName(table_path, kName)), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch), std::move(table_schema), std::move(options))) {} -std::string FilesSystemTable::Name() const { - return kName; -} - Result> FilesSystemTable::ArrowSchema() const { return arrow::schema({ arrow::field("partition", arrow::utf8(), /*nullable=*/true), diff --git a/src/paimon/core/table/system/metadata_system_tables.h b/src/paimon/core/table/system/metadata_system_tables.h index ae2e5155b..1127e37ed 100644 --- a/src/paimon/core/table/system/metadata_system_tables.h +++ b/src/paimon/core/table/system/metadata_system_tables.h @@ -38,7 +38,6 @@ class OptionsSystemTable : public InMemorySystemTable { OptionsSystemTable(const std::shared_ptr& fs, std::string table_path, std::shared_ptr table_schema); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -63,7 +62,6 @@ class SnapshotsSystemTable : public InMemorySystemTable { SnapshotsSystemTable(std::shared_ptr fs, std::string table_path, std::string branch); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -78,7 +76,6 @@ class SchemasSystemTable : public InMemorySystemTable { SchemasSystemTable(std::shared_ptr fs, std::string table_path, std::string branch); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -93,7 +90,6 @@ class TagsSystemTable : public InMemorySystemTable { TagsSystemTable(std::shared_ptr fs, std::string table_path, std::string branch); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -108,7 +104,6 @@ class BranchesSystemTable : public InMemorySystemTable { BranchesSystemTable(std::shared_ptr fs, std::string table_path, std::string branch); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -124,7 +119,6 @@ class ConsumersSystemTable : public InMemorySystemTable { ConsumersSystemTable(std::shared_ptr fs, std::string table_path, std::string branch); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -141,7 +135,6 @@ class ManifestsSystemTable : public InMemorySystemTable { std::shared_ptr table_schema, std::map options); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -158,7 +151,6 @@ class FilesSystemTable : public InMemorySystemTable { std::shared_ptr table_schema, std::map options); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; diff --git a/src/paimon/core/table/system/read_optimized_system_table.cpp b/src/paimon/core/table/system/read_optimized_system_table.cpp index 65a48c048..2c4123f2f 100644 --- a/src/paimon/core/table/system/read_optimized_system_table.cpp +++ b/src/paimon/core/table/system/read_optimized_system_table.cpp @@ -40,15 +40,11 @@ ReadOptimizedSystemTable::ReadOptimizedSystemTable(const std::shared_ptr table_schema, std::map options) - : SystemTable(fs, table_path), + : SystemTable(fs, table_path, ComposeName(table_path, kName)), table_path_(std::move(table_path)), table_schema_(std::move(table_schema)), options_(std::move(options)) {} -std::string ReadOptimizedSystemTable::Name() const { - return kName; -} - Result> ReadOptimizedSystemTable::ArrowSchema() const { return DataField::ConvertDataFieldsToArrowSchema(table_schema_->Fields()); } @@ -74,6 +70,8 @@ Result> ReadOptimizedSystemTable::NewScan( .WithMemoryPool(context->GetMemoryPool()) .WithExecutor(context->GetExecutor()) // The view's own file system when the context names none, rather than the default backend. + // A `ScanContext` carries no scheme-to-identifier map, so unlike `CreateDataReadContext` + // below there is no map here to resolve through instead. .WithFileSystem(context->GetSpecificFileSystem() != nullptr ? context->GetSpecificFileSystem() : GetFileSystem()) diff --git a/src/paimon/core/table/system/read_optimized_system_table.h b/src/paimon/core/table/system/read_optimized_system_table.h index 7f92e04e2..82fec3ef4 100644 --- a/src/paimon/core/table/system/read_optimized_system_table.h +++ b/src/paimon/core/table/system/read_optimized_system_table.h @@ -38,7 +38,6 @@ class ReadOptimizedSystemTable : public SystemTable { std::shared_ptr table_schema, std::map options); - std::string Name() const override; Result> ArrowSchema() const override; Result> NewScan(std::unique_ptr context) const override; Result> NewRead(std::unique_ptr context) const override; diff --git a/src/paimon/core/table/system/system_table.cpp b/src/paimon/core/table/system/system_table.cpp index 0161ad6b7..7a8146534 100644 --- a/src/paimon/core/table/system/system_table.cpp +++ b/src/paimon/core/table/system/system_table.cpp @@ -42,6 +42,7 @@ #include "paimon/core/utils/branch_manager.h" #include "paimon/file_store_commit.h" #include "paimon/file_store_write.h" +#include "paimon/predicate/predicate_utils.h" #include "paimon/read_context.h" #include "paimon/status.h" #include "paimon/write_context.h" @@ -182,15 +183,27 @@ std::optional FindSystemTableFactory(const std::string& syst } // namespace -SystemTable::SystemTable(const std::shared_ptr& fs, const std::string& table_path) +SystemTable::SystemTable(const std::shared_ptr& fs, const std::string& table_path, + std::string name) : Table(fs, table_path, /*schema=*/nullptr, - Identifier(Identifier::kUnknownDatabase, "system_table"), /*uuid=*/"") {} + Identifier(Identifier::kUnknownDatabase, "system_table"), /*uuid=*/""), + name_(std::move(name)) {} + +std::string SystemTable::ComposeName(const std::string& table_path, const std::string& kind) { + return PathUtil::GetName(table_path) + Catalog::SYSTEM_TABLE_SPLITTER + kind; +} Status SystemTable::CheckNoSearch(const std::shared_ptr& context) { if (context->HasVectorOrFullTextSearch()) { return Status::NotImplemented( "VectorSearch or FullTextSearch does not support system-table reads yet"); } + // A `FullTextMatch` predicate is not a TopK request, so the check above does not see it. The + // factory refuses one for a system table with the same message. An in-memory view serves rows + // of its own and never evaluates a predicate, so without this it answers with all of them. + if (PredicateUtils::ContainsFullTextMatch(context->GetPredicate())) { + return Status::NotImplemented("FullTextMatch predicate execution is not implemented yet"); + } return Status::OK(); } @@ -282,7 +295,7 @@ Result> SystemTableLoader::LoadFromPath( // The warehouse is the grandparent of the sys/
path context.warehouse = PathUtil::GetParentDirPath(PathUtil::GetParentDirPath(path)); context.catalog_options = dynamic_options; - // Note: context.catalog is intentionally left as nullptr here. + // Note: context.catalog is intentionally left empty here. // Global tables loaded from path do not have a Catalog reference and // cannot enumerate databases/tables. Only tables that don't require // catalog enumeration (e.g. catalog_options) will work in this path. diff --git a/src/paimon/core/table/system/system_table.h b/src/paimon/core/table/system/system_table.h index 4c235d3cf..640672236 100644 --- a/src/paimon/core/table/system/system_table.h +++ b/src/paimon/core/table/system/system_table.h @@ -50,21 +50,29 @@ struct SystemTablePath { bool is_global = false; }; -/// Base class for table-scoped system tables such as `T$options` and `T$snapshots`: the read-only -/// view a `T$system` identifier names, which exposes a schema of its own and serves scans and -/// reads from the data table it belongs to. `Name()` - which each kind answers for itself - is -/// its full name as well: the identifier it is opened by belongs to the data table underneath. +/// Base class for the table-scoped system tables such as `T$options` and `T$snapshots`. A +/// `T$system` identifier names a read-only view that exposes a schema of its own and serves scans +/// and reads from the data table it belongs to. +/// +/// A system table of a data table answers `$` for `Name()`. One of the `sys` +/// database answers the bare kind name. `FullName()` returns the same as `Name()` in both cases, +/// which is what Java Paimon's `Table.fullName()` defaults to. class SystemTable : public Table, public std::enable_shared_from_this { public: + std::string Name() const override { + return name_; + } + std::string FullName() const override { - return Name(); + return name_; } - /// The read-only schema this system table exposes, which is not the data table's schema. + /// Returns the read-only schema this system table exposes. It is not the data table's schema. virtual Result> ArrowSchema() const = 0; - /// A system table is read-only: copying it, writing it and committing for it are refused - /// rather than routed to the data table underneath, which its own `Table` serves. + /// A system table is read-only, so copying it, writing it and committing for it are refused + /// instead of routed to the data table underneath. That table serves them through its own + /// `Table`. Result> Copy( const std::map& dynamic_options) const override; @@ -77,19 +85,29 @@ class SystemTable : public Table, public std::enable_shared_from_this& fs, const std::string& table_path); - - /// Refuses a read whose context asks for a search: each system table builds a context of its - /// own for the data table underneath, which carries no search, so answering one would return - /// plain rows and quietly drop the search and its top K. + /// @param name The name this table answers with: `ComposeName(table_path, kind)` for a + /// table-scoped system table, the bare kind name for one of the `sys` database. + SystemTable(const std::shared_ptr& fs, const std::string& table_path, + std::string name); + + /// Composes the name of a table-scoped system table. Java Paimon's `OptionsTable.name()` + /// composes it the same way: `location.getName() + SYSTEM_TABLE_SPLITTER + OPTIONS`. + static std::string ComposeName(const std::string& table_path, const std::string& kind); + + /// Refuses a read whose context asks for a search: a root TopK request or a `FullTextMatch` + /// predicate. Neither is evaluated on the rows a system table serves, so answering one + /// would return plain rows and quietly drop the search and its top K. static Status CheckNoSearch(const std::shared_ptr& context); - /// Loaders call this once the object exists, because `ArrowSchema()` is virtual and cannot run - /// from a constructor: it fills in this table's own read-only `SystemTableSchema`. + /// Fills in this table's own read-only `SystemTableSchema`. A loader calls it once the object + /// exists, because `ArrowSchema()` is virtual and cannot run from a constructor. Status AdoptOwnSchema(); friend class SystemTableLoader; friend class GlobalSystemTableLoader; + + private: + std::string name_; }; /// Loads system table implementations from parsed table identifiers or table paths. diff --git a/src/paimon/core/table/system/system_table_test.cpp b/src/paimon/core/table/system/system_table_test.cpp index f1f0cbfb1..d6333475b 100644 --- a/src/paimon/core/table/system/system_table_test.cpp +++ b/src/paimon/core/table/system/system_table_test.cpp @@ -38,6 +38,7 @@ #include "paimon/core/table/system/audit_log_system_table.h" #include "paimon/core/table/system/binlog_system_table.h" #include "paimon/core/table/system/global_system_tables.h" +#include "paimon/core/table/system/metadata_system_tables.h" #include "paimon/core/table/system/read_optimized_system_table.h" #include "paimon/core/table/system/system_table_scan.h" #include "paimon/defs.h" @@ -45,6 +46,9 @@ #include "paimon/fs/file_system_factory.h" #include "paimon/memory/memory_pool.h" #include "paimon/metrics.h" +#include "paimon/predicate/full_text_match.h" +#include "paimon/predicate/predicate.h" +#include "paimon/predicate/predicate_builder.h" #include "paimon/predicate/vector_search.h" #include "paimon/read_context.h" #include "paimon/reader/batch_reader.h" @@ -222,7 +226,9 @@ TEST(SystemTableTest, TestReadOptimizedSystemTableRegistration) { SystemTableLoader::Load(ReadOptimizedSystemTable::kName, /*fs=*/nullptr, "/tmp/table", table_schema, /*dynamic_options=*/{})); - ASSERT_EQ(system_table->Name(), ReadOptimizedSystemTable::kName); + // The name carries the data table it belongs to, the way Java Paimon's + // `ReadOptimizedTable.name()` does. + ASSERT_EQ(system_table->Name(), "table$ro"); ASSERT_OK_AND_ASSIGN(std::shared_ptr arrow_schema, system_table->ArrowSchema()); ASSERT_EQ(arrow_schema->field_names(), (std::vector{"pk", "v"})); @@ -360,6 +366,39 @@ TEST(SystemTableTest, TestNewReadRefusesASearch) { "VectorSearch or FullTextSearch does not support system-table reads yet"); } +// `SetPredicate` carries a search too, as a `FullTextMatch` leaf. An in-memory view reads rows +// of its own and never evaluates a predicate, so without the check it would answer with all of +// them where `TableRead::Create` refuses the same request. +TEST(SystemTableTest, TestNewReadRefusesAFullTextMatchPredicate) { + std::map options = {{Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "orc"}}; + ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, + CreateTableSchemaForTest(options)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr in_memory, + SystemTableLoader::Load(OptionsSystemTable::kName, /*fs=*/nullptr, + "/tmp/table", table_schema, + /*dynamic_options=*/{})); + ReadOptimizedSystemTable read_optimized(/*fs=*/nullptr, "/tmp/table", table_schema, options); + + auto match = std::make_shared("pk", R"({"match":{"query":"apple"}})"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr predicate, + PredicateBuilder::FullTextMatch(/*field_index=*/0, match)); + + auto make_context = [&predicate] { + ReadContextBuilder builder("/tmp/table"); + builder.SetPredicate(predicate); + return builder.Finish(); + }; + + ASSERT_OK_AND_ASSIGN(std::unique_ptr in_memory_context, make_context()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_optimized_context, make_context()); + + ASSERT_NOK_WITH_MSG(in_memory->NewRead(std::move(in_memory_context)), + "FullTextMatch predicate execution is not implemented yet"); + ASSERT_NOK_WITH_MSG(read_optimized.NewRead(std::move(read_optimized_context)), + "FullTextMatch predicate execution is not implemented yet"); +} + namespace { /// A catalog that serves the files of every table through `GetTableFileSystem`, like one @@ -392,21 +431,25 @@ TEST(SystemTableTest, TestGlobalPartitionsUsesTheFileSystemOfEachTable) { std::unique_ptr dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); - PerTableFileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str(), options); - ASSERT_OK(catalog.CreateDatabase("db1", options, /*ignore_if_exists=*/true)); + // Held by a `shared_ptr` because a global system table keeps shared ownership of the catalog + // it reads its rows through. + std::shared_ptr catalog = + std::make_shared(core_options.GetFileSystem(), dir->Str(), + options); + ASSERT_OK(catalog->CreateDatabase("db1", options, /*ignore_if_exists=*/true)); arrow::Schema typed_schema( {arrow::field("dt", arrow::utf8()), arrow::field("v", arrow::int32(), /*nullable=*/true)}); ::ArrowSchema c_schema; ASSERT_TRUE(arrow::ExportSchema(typed_schema, &c_schema).ok()); - Status created = catalog.CreateTable(Identifier("db1", "t1"), &c_schema, - /*partition_keys=*/{"dt"}, /*primary_keys=*/{}, options, - /*ignore_if_exists=*/false); + Status created = catalog->CreateTable(Identifier("db1", "t1"), &c_schema, + /*partition_keys=*/{"dt"}, /*primary_keys=*/{}, options, + /*ignore_if_exists=*/false); ArrowSchemaRelease(&c_schema); ASSERT_OK(created); GlobalSystemTableContext context; - context.catalog = &catalog; + context.catalog = catalog; context.fs = core_options.GetFileSystem(); context.warehouse = dir->Str(); context.catalog_options = options; @@ -416,16 +459,16 @@ TEST(SystemTableTest, TestGlobalPartitionsUsesTheFileSystemOfEachTable) { // still the ones asked for before its files are listed. ASSERT_OK_AND_ASSIGN(std::vector rows, partitions.BuildRows()); ASSERT_TRUE(rows.empty()); - ASSERT_EQ((std::vector{"db1.t1"}), catalog.requested); + ASSERT_EQ((std::vector{"db1.t1"}), catalog->requested); // A table whose credentials cannot be issued fails the query, so that a partial result // is never mistaken for the whole one. - catalog.failure = Status::Invalid("no credentials for the table"); + catalog->failure = Status::Invalid("no credentials for the table"); ASSERT_NOK_WITH_MSG(partitions.BuildRows(), "no credentials for the table"); // A table that went away between being listed and being read is skipped instead, like // one whose location has already been removed. - catalog.failure = Status::NotExist("table dropped"); + catalog->failure = Status::NotExist("table dropped"); ASSERT_OK_AND_ASSIGN(std::vector skipped, partitions.BuildRows()); ASSERT_TRUE(skipped.empty()); } diff --git a/src/paimon/core/table/table_test.cpp b/src/paimon/core/table/table_test.cpp index 055fc5995..5100dfd78 100644 --- a/src/paimon/core/table/table_test.cpp +++ b/src/paimon/core/table/table_test.cpp @@ -75,7 +75,8 @@ Result> CreateManagedTable( return FileStoreTable::Create( file_system, path, checked_pointer_cast(std::shared_ptr(std::move(created))), identifier, - /*uuid=*/"", branch); + /*uuid=*/"", branch, /*schema_manager=*/nullptr, /*snapshot_manager=*/nullptr, + /*fs_scheme_map=*/{}, /*dynamic_options=*/{}); } /// Writes two rows through the table's own write and commit, so a scan below has data to find. @@ -243,7 +244,8 @@ TEST(TableTest, TestACopyHandsItsDynamicOptionsToTheFallbackReader) { FileStoreTable::Create( fs, path, checked_pointer_cast(std::shared_ptr(std::move(created))), - identifier, /*uuid=*/"", /*branch=*/"main")); + identifier, /*uuid=*/"", /*branch=*/"main", /*schema_manager=*/nullptr, + /*snapshot_manager=*/nullptr, /*fs_scheme_map=*/{}, /*dynamic_options=*/{})); // The copy turns the sequence-number column on, and a read projects it through a context // that carries no options of its own. The main reader serves the copy's merged options, so @@ -286,7 +288,8 @@ TEST(TableTest, TestACopyOfACopyServesTheLatestValueOfARepeatedDynamicOption) { FileStoreTable::Create( fs, path, checked_pointer_cast(std::shared_ptr(std::move(created))), - identifier, /*uuid=*/"", /*branch=*/"main")); + identifier, /*uuid=*/"", /*branch=*/"main", /*schema_manager=*/nullptr, + /*snapshot_manager=*/nullptr, /*fs_scheme_map=*/{}, /*dynamic_options=*/{})); // The option is turned off and then on again by the copy the read serves, so the copy's // schema carries the latest value. The fallback branch's schema does not carry the option @@ -328,7 +331,8 @@ TEST(TableTest, TestAFallbackReaderServesItsBranchsOwnSchema) { FileStoreTable::Create( fs, path, checked_pointer_cast(std::shared_ptr(std::move(main_created))), - identifier, /*uuid=*/"", /*branch=*/"main")); + identifier, /*uuid=*/"", /*branch=*/"main", /*schema_manager=*/nullptr, + /*snapshot_manager=*/nullptr, /*fs_scheme_map=*/{}, /*dynamic_options=*/{})); SchemaManager fallback_schema_manager(fs, path, "fb"); ASSERT_OK_AND_ASSIGN( @@ -341,7 +345,8 @@ TEST(TableTest, TestAFallbackReaderServesItsBranchsOwnSchema) { FileStoreTable::Create( fs, path, checked_pointer_cast(std::shared_ptr(std::move(fallback_created))), - identifier, /*uuid=*/"", /*branch=*/"fb")); + identifier, /*uuid=*/"", /*branch=*/"fb", /*schema_manager=*/nullptr, + /*snapshot_manager=*/nullptr, /*fs_scheme_map=*/{}, /*dynamic_options=*/{})); WriteAndCommitRow(fallback, path, /*id=*/1, /*v=*/10); WriteAndCommitRow(fallback, path, /*id=*/1, /*v=*/20); @@ -679,7 +684,11 @@ TEST(TableTest, TestMainBranchAliasResolvesToTheMainBranch) { ASSERT_OK_AND_ASSIGN(std::string branch, main_identifier.GetBranchNameOrDefault()); ASSERT_OK_AND_ASSIGN(std::shared_ptr table, FileStoreTable::Create(fs, dir->Str(), schema, main_identifier, - /*uuid=*/"", branch)); + /*uuid=*/"", branch, + /*schema_manager=*/nullptr, + /*snapshot_manager=*/nullptr, + /*fs_scheme_map=*/{}, + /*dynamic_options=*/{})); EXPECT_EQ("main", table->CurrentBranch()); } } @@ -1112,7 +1121,10 @@ TEST(TableTest, TestCreateFailsWhenTheSchemaOptionsDoNotParse) { /*partition_keys=*/{}, /*primary_keys=*/{}, {{Options::TARGET_FILE_ROW_NUM, "0"}})); auto schema = checked_pointer_cast(std::shared_ptr(std::move(created))); ASSERT_NOK_WITH_MSG( - FileStoreTable::Create(dir->GetFileSystem(), dir->Str(), schema, Identifier("db", "tbl")), + FileStoreTable::Create(dir->GetFileSystem(), dir->Str(), schema, Identifier("db", "tbl"), + /*uuid=*/"", /*branch=*/BranchManager::DEFAULT_MAIN_BRANCH, + /*schema_manager=*/nullptr, /*snapshot_manager=*/nullptr, + /*fs_scheme_map=*/{}, /*dynamic_options=*/{}), "target-file-row-num should be at least 1"); } diff --git a/src/paimon/rest/rest_catalog.cpp b/src/paimon/rest/rest_catalog.cpp index cae923d7f..167f3530c 100644 --- a/src/paimon/rest/rest_catalog.cpp +++ b/src/paimon/rest/rest_catalog.cpp @@ -384,13 +384,15 @@ Result> RestCatalog::LoadTableSchema(const Identifier& i if (!supported) { return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); } + // Only the schema is read here and the table does not outlive the call, so it needs no + // catalog and this one hands over no ownership of itself. GlobalSystemTableContext context; - context.catalog = this; context.fs = fs_; context.warehouse = warehouse_; context.catalog_options = api_->GetMergedOptions(); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, - GlobalSystemTableLoader::Load(identifier.GetObjectName(), context)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr system_table, + GlobalSystemTableLoader::LoadForSchema(identifier.GetObjectName(), context)); return system_table->LatestSchema(); } PAIMON_ASSIGN_OR_RAISE(bool is_system_table, identifier.IsSystemTable()); @@ -445,18 +447,17 @@ Result> RestCatalog::GetTable(const Identifier& identifie if (!supported) { return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); } - GlobalSystemTableContext context; - context.catalog = this; - // A global system table builds its rows lazily through the catalog above, so the table - // handed back has to keep it alive or the pointer dangles at the first `BuildRows`. A - // catalog not owned by a `std::shared_ptr` cannot be kept, so it is refused by name. + // A global system table builds its rows lazily through this catalog, so the table handed + // back has to keep it alive or the pointer dangles at the first `BuildRows`. A catalog + // not owned by a `std::shared_ptr` cannot be kept, so it is refused by name. std::shared_ptr self = weak_from_this().lock(); if (self == nullptr) { return Status::Invalid( "a global system table retains its catalog, so the catalog must be owned by " "a std::shared_ptr before it is asked for one"); } - context.catalog_retainer = std::move(self); + GlobalSystemTableContext context; + context.catalog = std::move(self); context.fs = fs_; context.warehouse = warehouse_; context.catalog_options = api_->GetMergedOptions(); @@ -497,7 +498,9 @@ Result> RestCatalog::GetTable(const Identifier& identifie PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); return FileStoreTable::Create( table_file_system, table_path, schema, identifier, uuid, - NormalizeBranch(std::move(branch)).value_or(Identifier::kDefaultMainBranch)); + NormalizeBranch(std::move(branch)).value_or(Identifier::kDefaultMainBranch), + /*schema_manager=*/nullptr, /*snapshot_manager=*/nullptr, /*fs_scheme_map=*/{}, + /*dynamic_options=*/{}); } std::string RestCatalog::GetRootPath() const { diff --git a/src/paimon/rest/rest_catalog_test.cpp b/src/paimon/rest/rest_catalog_test.cpp index 54929fb48..6a8a202e7 100644 --- a/src/paimon/rest/rest_catalog_test.cpp +++ b/src/paimon/rest/rest_catalog_test.cpp @@ -810,7 +810,7 @@ TEST_F(RestCatalogTest, RecreatedTableLoadsNewCredentialsBeforeOldTokenExpires) } TEST_F(RestCatalogTest, CatalogFactoryMetastoreDispatch) { - ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, Catalog::Create(kWarehouse, options_)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr catalog, Catalog::Create(kWarehouse, options_)); ASSERT_OK_AND_ASSIGN(std::vector databases, catalog->ListDatabases()); ASSERT_TRUE(databases.empty()); options_[CatalogOptions::METASTORE] = "something-else"; @@ -1070,13 +1070,13 @@ TEST_F(RestCatalogTest, SystemTableSchema) { requests_before = state_->get_table_requests; } ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog->GetTable(system_identifier)); - // A system table comes back as the read-only `SystemTable` it is, answering for itself - // with the kind it serves rather than the identifier it was opened by. + // A system table comes back as the read-only `SystemTable` it is, naming itself after the + // data table it belongs to. std::shared_ptr system_table = std::dynamic_pointer_cast(table); ASSERT_NE(nullptr, system_table); - ASSERT_EQ("options", system_table->Name()); + ASSERT_EQ("t1$options", system_table->Name()); ASSERT_EQ((std::vector{"key", "value"}), table->LatestSchema()->FieldNames()); - ASSERT_EQ("options", table->Uuid()); + ASSERT_EQ("t1$options", table->Uuid()); { std::lock_guard lock(state_->mutex); ASSERT_EQ(requests_before + 1, state_->get_table_requests); @@ -1578,6 +1578,34 @@ TEST_F(RestCatalogTest, GlobalSystemTableRetainsItsCatalog) { ASSERT_GT(array->length(), 0); } +TEST_F(RestCatalogTest, GlobalSystemTableWorksThroughThePublicCatalogFactory) { + // `Catalog::Create()` is the factory a caller outside paimon uses, and it hands back a + // `Catalog` rather than a `RestCatalog`. A global system table keeps its catalog alive + // through that pointer, so this path has to work too - a catalog whose shared ownership + // was not set up would fail here with the refusal naming the missing ownership. + ASSERT_OK_AND_ASSIGN(std::shared_ptr catalog, Catalog::Create(kWarehouse, options_)); + ASSERT_OK(catalog->CreateDatabase("db1", {}, /*ignore_if_exists=*/false)); + ASSERT_OK(CreateSampleTable(catalog.get(), Identifier("db1", "t1"))); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + catalog->GetTable(Identifier("sys", "tables"))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, + ScanContextBuilder("sys/tables").Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_EQ(plan->Splits().size(), 1); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, + ReadContextBuilder("sys/tables").Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, table->NewRead(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, + read->CreateReader(plan->Splits())); + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, batch_reader->NextBatch()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr array, + ReadResultCollector::GetArray(std::move(batch))); + ASSERT_TRUE(array); + ASSERT_GT(array->length(), 0); +} + TEST_F(RestCatalogTest, ListSnapshots) { ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); ASSERT_OK(catalog->CreateDatabase("db1", {}, /*ignore_if_exists=*/false)); diff --git a/src/paimon/testing/mock/mock_catalog.h b/src/paimon/testing/mock/mock_catalog.h index 32a684c34..346203c74 100644 --- a/src/paimon/testing/mock/mock_catalog.h +++ b/src/paimon/testing/mock/mock_catalog.h @@ -118,7 +118,9 @@ class MockVersionManagedCatalog : public Catalog, public VersionManagedCatalog { // this mock's `GetTableLocation` is the unsupported stub. return FileStoreTable::Create( table_file_system_ != nullptr ? table_file_system_ : file_system_, - /*root_path=*/"", table_schema_, identifier, table_uuid_); + /*root_path=*/"", table_schema_, identifier, table_uuid_, + /*branch=*/BranchManager::DEFAULT_MAIN_BRANCH, /*schema_manager=*/nullptr, + /*snapshot_manager=*/nullptr, /*fs_scheme_map=*/{}, /*dynamic_options=*/{}); } void SetTableSchema(const std::shared_ptr& table_schema) { diff --git a/test/inte/read_inte_test.cpp b/test/inte/read_inte_test.cpp index 2c24607a2..ef8e5c853 100644 --- a/test/inte/read_inte_test.cpp +++ b/test/inte/read_inte_test.cpp @@ -4173,8 +4173,9 @@ TEST_P(ReadInteTest, TestSpecificFs) { namespace { Result> ReadGlobalSystemTable( - const std::string& table_name, Catalog* catalog, const std::shared_ptr& fs, - const std::string& warehouse, const std::map& options) { + const std::string& table_name, const std::shared_ptr& catalog, + const std::shared_ptr& fs, const std::string& warehouse, + const std::map& options) { GlobalSystemTableContext ctx; ctx.catalog = catalog; ctx.fs = fs; @@ -4229,8 +4230,8 @@ TEST(SystemTableReadInteTest, TestReadGlobalCatalogOptions) { std::string warehouse = PathUtil::JoinPath(dir->Str(), "warehouse"); ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(warehouse, options)); ASSERT_OK_AND_ASSIGN(auto result, - ReadGlobalSystemTable("catalog_options", catalog.get(), - catalog->GetFileSystem(), warehouse, options)); + ReadGlobalSystemTable("catalog_options", catalog, catalog->GetFileSystem(), + warehouse, options)); auto struct_array = SingleStructChunk(result); ASSERT_TRUE(struct_array); auto key_array = std::dynamic_pointer_cast(struct_array->field(0)); @@ -4301,7 +4302,7 @@ TEST(SystemTableReadInteTest, TestReadGlobalAllTableOptions) { // Directly test BuildRows { GlobalSystemTableContext ctx; - ctx.catalog = catalog.get(); + ctx.catalog = catalog; ctx.fs = catalog->GetFileSystem(); ctx.warehouse = warehouse; ctx.catalog_options = options; @@ -4310,9 +4311,9 @@ TEST(SystemTableReadInteTest, TestReadGlobalAllTableOptions) { ASSERT_GT(rows.size(), 0) << "BuildRows returned empty, expected at least 1 row"; } - ASSERT_OK_AND_ASSIGN(auto result, - ReadGlobalSystemTable("all_table_options", catalog.get(), - catalog->GetFileSystem(), warehouse, options)); + ASSERT_OK_AND_ASSIGN( + auto result, ReadGlobalSystemTable("all_table_options", catalog, catalog->GetFileSystem(), + warehouse, options)); auto struct_array = SingleStructChunk(result); ASSERT_TRUE(struct_array); ASSERT_GE(struct_array->length(), 1) << "result has " << struct_array->length() << " rows"; @@ -4392,7 +4393,7 @@ TEST(SystemTableReadInteTest, TestReadGlobalTables) { ArrowSchemaRelease(&no_pk_schema); ASSERT_OK_AND_ASSIGN(auto result, - ReadGlobalSystemTable("tables", catalog.get(), fs, warehouse, options)); + ReadGlobalSystemTable("tables", catalog, fs, warehouse, options)); auto struct_array = SingleStructChunk(result); ASSERT_TRUE(struct_array); ASSERT_GE(struct_array->length(), 1); @@ -4490,8 +4491,8 @@ TEST(SystemTableReadInteTest, TestReadGlobalPartitions) { ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(warehouse, options)); auto fs = catalog->GetFileSystem(); - ASSERT_OK_AND_ASSIGN(auto part_result, ReadGlobalSystemTable("partitions", catalog.get(), fs, - warehouse, options)); + ASSERT_OK_AND_ASSIGN(auto part_result, + ReadGlobalSystemTable("partitions", catalog, fs, warehouse, options)); auto struct_array = SingleStructChunk(part_result); ASSERT_TRUE(struct_array); ASSERT_EQ(StructFieldNames(struct_array), @@ -4542,7 +4543,7 @@ TEST(SystemTableReadInteTest, TestGlobalSystemTablesPropagateCorruptSchema) { /*overwrite=*/true)); for (const std::string system_table : {"all_table_options", "tables", "partitions"}) { - ASSERT_NOK(ReadGlobalSystemTable(system_table, catalog.get(), fs, warehouse, options)); + ASSERT_NOK(ReadGlobalSystemTable(system_table, catalog, fs, warehouse, options)); } } @@ -4578,7 +4579,7 @@ TEST(SystemTableReadInteTest, TestPartitionsSystemTablePropagatesCorruptSnapshot ASSERT_OK(fs->WriteFile(PathUtil::JoinPath(table_path, "snapshot/snapshot-1"), "{invalid-json", /*overwrite=*/true)); - ASSERT_NOK(ReadGlobalSystemTable("partitions", catalog.get(), fs, warehouse, options)); + ASSERT_NOK(ReadGlobalSystemTable("partitions", catalog, fs, warehouse, options)); } TEST(SystemTableReadInteTest, TestPartitionsSystemTablePropagatesCorruptManifest) { @@ -4617,7 +4618,7 @@ TEST(SystemTableReadInteTest, TestPartitionsSystemTablePropagatesCorruptManifest PathUtil::JoinPath(table_path, "manifest/" + snapshot->BaseManifestList()), "corrupt", /*overwrite=*/true)); - ASSERT_NOK(ReadGlobalSystemTable("partitions", catalog.get(), fs, warehouse, options)); + ASSERT_NOK(ReadGlobalSystemTable("partitions", catalog, fs, warehouse, options)); } } // namespace paimon::test diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp index 5de6c9fa7..7111f5302 100644 --- a/test/inte/realtime_write_inte_test.cpp +++ b/test/inte/realtime_write_inte_test.cpp @@ -504,7 +504,7 @@ class RealtimeWriteInteTest : public ::testing::Test { void CreateTable(const std::vector& partition_keys) const { auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*schema_, c_schema.get()).ok()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, + ASSERT_OK_AND_ASSIGN(std::shared_ptr catalog, Catalog::Create(dir_->Str(), options_)); ASSERT_OK(catalog->CreateDatabase("foo", {}, /*ignore_if_exists=*/false)); ASSERT_OK(catalog->CreateTable(Identifier("foo", "bar"), c_schema.get(), partition_keys, @@ -516,7 +516,7 @@ class RealtimeWriteInteTest : public ::testing::Test { const std::vector& primary_keys = {"id"}) const { auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*schema_, c_schema.get()).ok()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, + ASSERT_OK_AND_ASSIGN(std::shared_ptr catalog, Catalog::Create(dir_->Str(), options_)); ASSERT_OK(catalog->CreateDatabase("foo", {}, /*ignore_if_exists=*/false)); std::vector table_primary_keys = partition_keys; From 6c19538c223af95edd6f8c972cefa449fbe3480a Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Tue, 27 Oct 2026 16:51:21 +0800 Subject: [PATCH 17/30] fix --- docs/source/user_guide/catalog.rst | 2 +- .../core/operation/file_store_commit.cpp | 16 ++++++++++++ .../core/operation/file_store_write.cpp | 11 ++++++++ src/paimon/core/schema/schema_validation.cpp | 5 ++++ src/paimon/core/table/file_store_table.cpp | 26 ++++--------------- src/paimon/core/table/source/table_read.cpp | 10 +++++++ src/paimon/core/table/source/table_scan.cpp | 10 +++++++ src/paimon/core/table/table_test.cpp | 7 +++++ 8 files changed, 65 insertions(+), 22 deletions(-) diff --git a/docs/source/user_guide/catalog.rst b/docs/source/user_guide/catalog.rst index 0ccb976dd..cdb721b97 100644 --- a/docs/source/user_guide/catalog.rst +++ b/docs/source/user_guide/catalog.rst @@ -139,7 +139,7 @@ without an atomic snapshot-file rename: } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit, - paimon::FileStoreCommit::Create(std::move(context))); + table->NewCommit(std::move(context))); PAIMON_RETURN_NOT_OK(commit->Commit(commit_messages)); Configure native writers with the same catalog and identifier. They load the diff --git a/src/paimon/core/operation/file_store_commit.cpp b/src/paimon/core/operation/file_store_commit.cpp index 4e0747437..9b6c3975c 100644 --- a/src/paimon/core/operation/file_store_commit.cpp +++ b/src/paimon/core/operation/file_store_commit.cpp @@ -18,3 +18,19 @@ */ #include "paimon/file_store_commit.h" + +#include "paimon/catalog/table.h" +#include "paimon/commit_context.h" +#include "paimon/core/table/file_store_table_factory.h" +#include "paimon/core/utils/context_utils.h" + +namespace paimon { + +Result> FileStoreCommit::Create( + std::unique_ptr ctx) { + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "commit")); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, FileStoreTableFactory::Create(*ctx)); + return table->NewCommit(std::move(ctx)); +} + +} // namespace paimon diff --git a/src/paimon/core/operation/file_store_write.cpp b/src/paimon/core/operation/file_store_write.cpp index c1b889473..ea73b2246 100644 --- a/src/paimon/core/operation/file_store_write.cpp +++ b/src/paimon/core/operation/file_store_write.cpp @@ -18,8 +18,19 @@ #include "paimon/file_store_write.h" +#include "paimon/catalog/table.h" +#include "paimon/core/table/file_store_table_factory.h" +#include "paimon/core/utils/context_utils.h" +#include "paimon/write_context.h" + namespace paimon { +Result> FileStoreWrite::Create(std::unique_ptr ctx) { + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "write")); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, FileStoreTableFactory::Create(*ctx)); + return table->NewWrite(std::move(ctx)); +} + Result> FileStoreWrite::PrepareCommitWithProgress(int64_t) { return Status::Invalid("prepare commit with progress requires a real-time writer"); } diff --git a/src/paimon/core/schema/schema_validation.cpp b/src/paimon/core/schema/schema_validation.cpp index 5d112417f..c799c24f8 100644 --- a/src/paimon/core/schema/schema_validation.cpp +++ b/src/paimon/core/schema/schema_validation.cpp @@ -85,6 +85,11 @@ constexpr char kDeletionVectorsMergeOnRead[] = "deletion-vectors.merge-on-read"; /// into a table. Unlike the Java set, `merge-engine`, `bucket`, `row-tracking.enabled` and /// `data-evolution.enabled` stay servable as dynamic options here, and `type`/`branch` are dropped /// by `Copy` before this point. +// TODO(yonghao.fyh): refuse the rest of Java's immutable set, which this list does not cover yet: +// `blob-descriptor-field` (only its deprecated `blob.stored-descriptor-fields` fallback is above), +// `data-file.path-directory`, `sequence.snapshot-ordering`, `rowkind.field`, `primary-key`, +// `primary-key.nullable`, `partition`, `dynamic-bucket.initial-buckets`, `video-frame-field`, and +// the four servable keys named above, should the C++ side stop serving them as dynamic options. const char* const kImmutableOptions[] = { Options::BUCKET_KEY, Options::BUCKET_FUNCTION_TYPE, diff --git a/src/paimon/core/table/file_store_table.cpp b/src/paimon/core/table/file_store_table.cpp index 0f9e0ed34..8d48d71c6 100644 --- a/src/paimon/core/table/file_store_table.cpp +++ b/src/paimon/core/table/file_store_table.cpp @@ -57,7 +57,6 @@ #include "paimon/core/snapshot.h" #include "paimon/core/table/append_only_file_store_table.h" #include "paimon/core/table/file_store.h" -#include "paimon/core/table/file_store_table_factory.h" #include "paimon/core/table/primary_key_file_store_table.h" #include "paimon/core/table/source/append_only_table_read.h" #include "paimon/core/table/source/data_evolution_batch_scan.h" @@ -212,9 +211,13 @@ Result> FileStoreTable::SwitchToBranch(const std::string& branch_schema->Options(), std::static_pointer_cast(schema_)->Options()))) ? nullptr : schema_manager; + // The identifier names the branch switched to, as Java's `newCatalogEnvironment` rebuilds it, + // so `Name()` and `FullName()` answer for the table served rather than the one left behind. + PAIMON_ASSIGN_OR_RAISE(Identifier branch_identifier, + CatalogUtils::BranchIdentifier(identifier_, target_branch)); return FileStoreTable::Create(store_->GetSpecifiedFileSystem(), root_path_, std::shared_ptr(std::move(branch_schema)), - identifier_, CatalogUuid().value_or(""), target_branch, + branch_identifier, CatalogUuid().value_or(""), target_branch, branch_schema_manager, /*snapshot_manager=*/nullptr, store_->GetFsSchemeMap(), /*dynamic_options=*/{}); } @@ -316,12 +319,6 @@ Result> FileStoreTable::NewWrite( snapshot_manager, io_manager, ignore_previous_files); } -Result> FileStoreWrite::Create(std::unique_ptr ctx) { - PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "write")); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, FileStoreTableFactory::Create(*ctx)); - return table->NewWrite(std::move(ctx)); -} - Result> FileStoreTable::NewScan( std::unique_ptr context) const { PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "scan")); @@ -424,12 +421,6 @@ Result> FileStoreTable::NewScan( return batch_scan; } -Result> TableScan::Create(std::unique_ptr context) { - PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "scan")); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, FileStoreTableFactory::Create(*context)); - return table->NewScan(std::move(context)); -} - Result> FileStoreTable::CreateTableRead( const std::shared_ptr& internal_context, const std::shared_ptr& memory_pool, @@ -651,11 +642,4 @@ Result> FileStoreTable::NewCommit( expire_snapshots, GetSchemaManager(), std::move(scan_supplier)); } -Result> FileStoreCommit::Create( - std::unique_ptr ctx) { - PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "commit")); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, FileStoreTableFactory::Create(*ctx)); - return table->NewCommit(std::move(ctx)); -} - } // namespace paimon diff --git a/src/paimon/core/table/source/table_read.cpp b/src/paimon/core/table/source/table_read.cpp index e723c54bb..5688c7650 100644 --- a/src/paimon/core/table/source/table_read.cpp +++ b/src/paimon/core/table/source/table_read.cpp @@ -19,8 +19,18 @@ #include "paimon/table/source/table_read.h" +#include "paimon/catalog/table.h" +#include "paimon/core/table/file_store_table_factory.h" +#include "paimon/core/utils/context_utils.h" + namespace paimon { +Result> TableRead::Create(std::unique_ptr ctx) { + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "read")); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, FileStoreTableFactory::Create(*ctx)); + return table->NewRead(std::move(ctx)); +} + Result> TableRead::CreateCountReader( const std::vector>& splits) { (void)splits; diff --git a/src/paimon/core/table/source/table_scan.cpp b/src/paimon/core/table/source/table_scan.cpp index be5d5b955..d3dd794b4 100644 --- a/src/paimon/core/table/source/table_scan.cpp +++ b/src/paimon/core/table/source/table_scan.cpp @@ -19,10 +19,20 @@ #include "paimon/table/source/table_scan.h" +#include "paimon/catalog/table.h" #include "paimon/common/metrics/metrics_impl.h" +#include "paimon/core/table/file_store_table_factory.h" +#include "paimon/core/utils/context_utils.h" +#include "paimon/scan_context.h" namespace paimon { +Result> TableScan::Create(std::unique_ptr context) { + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "scan")); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, FileStoreTableFactory::Create(*context)); + return table->NewScan(std::move(context)); +} + std::shared_ptr TableScan::GetMetrics() const { return std::make_shared(); } diff --git a/src/paimon/core/table/table_test.cpp b/src/paimon/core/table/table_test.cpp index 5100dfd78..8b7b2da45 100644 --- a/src/paimon/core/table/table_test.cpp +++ b/src/paimon/core/table/table_test.cpp @@ -456,6 +456,10 @@ TEST(TableTest, TestSwitchToBranch) { auto dev_file_store_table = std::dynamic_pointer_cast(dev_table); ASSERT_TRUE(dev_file_store_table != nullptr); EXPECT_EQ("dev", dev_file_store_table->CurrentBranch()); + // The identifier names the branch switched to, so `Name()` and `FullName()` answer for the + // table served, and a caller that reloads through `FullName()` lands on the same branch. + EXPECT_EQ("tbl$branch_dev", dev_table->Name()); + EXPECT_EQ("tbl$branch_dev", dev_table->FullName()); ASSERT_EQ((std::vector{"c0"}), dev_table->LatestSchema()->FieldNames()); auto dev_table_schema = std::dynamic_pointer_cast(dev_table->LatestSchema()); ASSERT_TRUE(dev_table_schema != nullptr); @@ -480,6 +484,9 @@ TEST(TableTest, TestSwitchToBranch) { auto main_again_file_store_table = std::dynamic_pointer_cast(main_again); ASSERT_TRUE(main_again_file_store_table != nullptr); EXPECT_EQ("main", main_again_file_store_table->CurrentBranch()); + // Switching back leaves no branch in the name either, the way the main branch is addressed. + EXPECT_EQ("tbl", main_again->Name()); + EXPECT_EQ("tbl", main_again->FullName()); ASSERT_EQ((std::vector{"id"}), main_again->LatestSchema()->FieldNames()); auto main_again_schema = std::dynamic_pointer_cast(main_again->LatestSchema()); ASSERT_TRUE(main_again_schema != nullptr); From a0890b02eb4f1075ca64444e5df2bceba29ad9e0 Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Tue, 27 Oct 2026 18:24:11 +0800 Subject: [PATCH 18/30] fix --- docs/source/api/format_table.rst | 28 +- docs/source/user_guide/catalog.rst | 11 +- docs/source/user_guide/format_table.rst | 38 ++- include/paimon/commit_context.h | 24 +- include/paimon/file_store_commit.h | 5 +- include/paimon/file_store_write.h | 5 +- include/paimon/read_context.h | 53 ++-- include/paimon/scan_context.h | 20 +- include/paimon/table/source/table_read.h | 5 +- include/paimon/table/source/table_scan.h | 5 +- include/paimon/write_context.h | 20 -- src/paimon/core/operation/commit_context.cpp | 37 +-- .../core/operation/commit_context_test.cpp | 2 +- .../core/operation/file_store_commit_test.cpp | 3 +- src/paimon/core/operation/read_context.cpp | 50 +--- src/paimon/core/operation/scan_context.cpp | 42 +-- src/paimon/core/operation/write_context.cpp | 42 +-- .../core/table/file_store_table_factory.cpp | 126 +++----- .../core/table/file_store_table_factory.h | 9 +- src/paimon/core/table/format/format_table.cpp | 4 +- .../core/table/format/format_table_test.cpp | 277 +++++++----------- .../core/table/source/table_scan_test.cpp | 12 +- .../core/table/system/system_table_test.cpp | 16 + src/paimon/core/table/table_test.cpp | 24 +- test/inte/read_inte_test.cpp | 48 ++- test/inte/realtime_write_inte_test.cpp | 9 +- 26 files changed, 327 insertions(+), 588 deletions(-) diff --git a/docs/source/api/format_table.rst b/docs/source/api/format_table.rst index 1ec964375..36295493f 100644 --- a/docs/source/api/format_table.rst +++ b/docs/source/api/format_table.rst @@ -35,18 +35,16 @@ and :cpp:class:`paimon::FileStoreCommit`. See :doc:`../user_guide/format_table`. Contexts ======== -Each of those entry points builds its context from a table path, and the schema under that path -says what kind of table it is. A caller that already holds a ``FormatTable`` hands it over instead, -through the constructor each context builder has for one: - -* :cpp:class:`paimon::ScanContextBuilder`\ ``(const std::shared_ptr&)`` -* :cpp:class:`paimon::ReadContextBuilder`\ ``(const std::shared_ptr&)`` -* :cpp:class:`paimon::WriteContextBuilder`\ ``(const std::shared_ptr&)`` -* :cpp:class:`paimon::CommitContextBuilder`\ ``(const std::shared_ptr&)`` - -This is the only way to reach a format table whose schema does not live under its own location, -such as one a REST catalog serves: nothing under the location says that it is a format table, nor -that what sits below it is data rather than metadata. The table knows both, so a setting that would -answer either question again - ``SetTableSchema()``, ``WithFileSystem()`` and a branch among them - -is refused by these builders rather than quietly ignored. Options given at the call still win over -the ones the table carries, as they do everywhere else. +Each of those entry points builds its context the way it does for a data table: from the table's +location, through the builder's usual constructor: + +* :cpp:class:`paimon::ScanContextBuilder`\ ``(const std::string&)`` +* :cpp:class:`paimon::ReadContextBuilder`\ ``(const std::string&)`` +* :cpp:class:`paimon::WriteContextBuilder`\ ``(const std::string&)`` +* :cpp:class:`paimon::CommitContextBuilder`\ ``(const std::string&)`` + +The context then goes to the table's own operator - ``NewScan()``, ``NewRead()``, ``NewWrite()`` or +``NewCommit()`` - which serves a table whose schema does not live under its own location as well, +such as one a REST catalog serves: the table carries its schema and the file system it was loaded +through, so nothing under the location has to answer for them. Options given at the call still win +over the ones the table carries, as they do everywhere else. diff --git a/docs/source/user_guide/catalog.rst b/docs/source/user_guide/catalog.rst index cdb721b97..fd9d16cf3 100644 --- a/docs/source/user_guide/catalog.rst +++ b/docs/source/user_guide/catalog.rst @@ -168,8 +168,8 @@ Both builders use the file system of the table being written or committed for manifests and data, so a catalog that issues temporary credentials per table serves them through ``Catalog::GetTableFileSystem``. ``WithFileSystem`` overrides it; writers also allow ``WithFileSystemSchemeToIdentifierMap`` to override -file-system selection. For format tables, use ``WriteContextBuilder(FormatTable)`` -instead. +file-system selection. A format table is written through the table's own +``NewWrite`` instead; see :doc:`format_table`. A file system passed to ``WithFileSystem`` is used exactly as it is: neither the built-in file systems nor the table's own credentials are involved, so a custom @@ -336,10 +336,9 @@ snapshots, manifests and data under the table path, so it signs every access wit the table's credentials and reloads them as they expire. An explicit ``WithFileSystem`` takes precedence, so the catalog is not asked. A catalog that issues no per-table credentials returns its catalog-level file system, so this is -also how you read with the catalog's own object-store credentials. For a format -table, use ``ReadContextBuilder(FormatTable)`` or ``ScanContextBuilder(FormatTable)`` -instead; ``WithCatalog`` is rejected because the table already carries the file -system it was loaded through. +also how you read with the catalog's own object-store credentials. A format table +is read through the table's own ``NewRead`` or ``NewScan`` instead; see +:doc:`format_table`. The C++ REST catalog covers the database, table, snapshot and commit operations of the ``Catalog`` API. The parts of the Java REST catalog that have no C++ diff --git a/docs/source/user_guide/format_table.rst b/docs/source/user_guide/format_table.rst index 7ee90976d..7579b9dff 100644 --- a/docs/source/user_guide/format_table.rst +++ b/docs/source/user_guide/format_table.rst @@ -44,18 +44,10 @@ A format table is served through the same :cpp:func:`Catalog::GetTable` as every its ``type`` option says what it is, and the call hands back the ``FormatTable`` it describes, through the ``Table`` interface the two kinds of table share. -Reading and writing go through the entry points every other table uses: ``TableScan::Create``, -``TableRead::Create``, ``FileStoreWrite::Create`` and ``FileStoreCommit::Create``, each from its -usual context builder. Each reads the table's schema - from under the table path, or from the one -the context carries - and dispatches to a format table when its ``type`` says so. - -That path needs a schema under the table's own location, which is what every table a file system -catalog serves has. A table loaded from a catalog that keeps the schema elsewhere, such as the -REST catalog, has none: nothing under the location says that it is a format table, nor that -``schema`` and ``branch`` below it are data rather than metadata. Such a table is read and written -by handing the loaded ``FormatTable`` to the context builder instead of a path - every builder has -a constructor taking one - and calling the table's own operator, which is what Java does through -``FormatTable.newReadBuilder()`` and ``newBatchWriteBuilder()``: +Reading and writing go through the table itself: a caller holding one calls ``NewScan()``, +``NewRead()``, ``NewWrite()`` or ``NewCommit()`` on it, with a context its usual builder builds from +the table's location. This is what Java serves through ``FormatTable.newReadBuilder()`` and +``newBatchWriteBuilder()``: .. code-block:: cpp @@ -64,22 +56,26 @@ a constructor taking one - and calling the table's own operator, which is what J // The table's `type` option is `format-table`, so this is a `FormatTable`. std::shared_ptr table = std::dynamic_pointer_cast(loaded); - paimon::ScanContextBuilder scan_builder(table); + paimon::ScanContextBuilder scan_builder(table->Location()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, table->NewScan(std::move(scan_context))); -The table already carries its schema and the file system it was loaded through, so a context built -from one refuses ``SetTableSchema()``, ``WithFileSystem()``, -``WithFileSystemSchemeToIdentifierMap()``, ``WithCatalog()`` and a branch. Options +Because the operator comes from the table itself, this serves a table whose schema does not live +under its own location as well, such as one a REST catalog serves: the table carries its schema and +the file system it was loaded through, so nothing under the location has to answer for them. Options given at the call still win over the ones the schema stored, as they do everywhere else. -``FormatTable`` is a derived class of ``Table``, so the table serves the same calls itself: a -caller holding one through a ``std::shared_ptr`` calls ``NewScan()``, -``NewRead()``, ``NewWrite()`` or ``NewCommit()`` on it, and the operators come back for the table -it is, whether or not the context carries one. This is the C++ side of what Java serves through -one ``Table.newReadBuilder()`` for both kinds of table. +The generic entry points - ``TableScan::Create``, ``TableRead::Create``, ``FileStoreWrite::Create`` +and ``FileStoreCommit::Create`` - serve a data table only: a load from a path is the data table's +way in, and a format table the schema under the path names is refused with a message pointing at +the catalog and the table's own operators above. + +``FormatTable`` is a derived class of ``Table``, so a caller holding one through a +``std::shared_ptr`` calls the same operators on it without the cast: they come back +for the table it is, whatever kind it turns out to be. This is the C++ side of what Java serves +through one ``Table.newReadBuilder()`` for both kinds of table. A batch comes back in the table's column order, or the projection's when the read names one, behind the ``_VALUE_KIND`` field every ``BatchReader`` carries. Which columns a *file* is asked for diff --git a/include/paimon/commit_context.h b/include/paimon/commit_context.h index 556db7bd2..c38cdca4c 100644 --- a/include/paimon/commit_context.h +++ b/include/paimon/commit_context.h @@ -32,7 +32,6 @@ namespace paimon { class Catalog; class Executor; -class FormatTable; class MemoryPool; /// `CommitContext` is some configuration for commit operations. @@ -48,8 +47,7 @@ class PAIMON_EXPORT CommitContext { bool append_commit_check_conflict, const std::shared_ptr& memory_pool, const std::shared_ptr& executor, const std::shared_ptr& specific_file_system, - const std::map& options, - const std::shared_ptr& format_table); + const std::map& options); CommitContext(const std::string& root_path, const std::string& commit_user, bool ignore_empty_commit, bool use_rest_catalog_commit, @@ -59,8 +57,7 @@ class PAIMON_EXPORT CommitContext { const std::shared_ptr& memory_pool, const std::shared_ptr& executor, const std::shared_ptr& specific_file_system, - const std::map& options, - const std::shared_ptr& format_table); + const std::map& options); ~CommitContext(); const std::string& GetRootPath() const { @@ -114,12 +111,6 @@ class PAIMON_EXPORT CommitContext { return options_; } - /// The format table this context was built from, or null when it names a table path and the - /// schema under that path says what kind of table it is. - const std::shared_ptr& GetFormatTable() const { - return format_table_; - } - private: std::string root_path_; std::string commit_user_; @@ -133,7 +124,6 @@ class PAIMON_EXPORT CommitContext { std::shared_ptr executor_; std::shared_ptr specific_file_system_; std::map options_; - std::shared_ptr format_table_; }; /// `CommitContextBuilder` used to build a `CommitContext`, has input validation. @@ -144,16 +134,6 @@ class PAIMON_EXPORT CommitContextBuilder { /// @param commit_user The user identifier for the commit operation. CommitContextBuilder(const std::string& root_path, const std::string& commit_user); - /// Constructs a `CommitContextBuilder` for a format table that is already loaded: the only way - /// to commit to one whose schema lives in a metastore rather than under its location, such as - /// a table a REST catalog serves. The table carries what such a location does not say, so - /// `WithFileSystem()` is refused here rather than ignored. - /// - /// There is no commit user: a format table keeps no snapshot to record one in. - /// - /// @param table The format table to commit to, as `Catalog::GetTable()` hands it back. - explicit CommitContextBuilder(const std::shared_ptr& table); - ~CommitContextBuilder(); /// Set a configuration options map to set some option entries which are not defined in the diff --git a/include/paimon/file_store_commit.h b/include/paimon/file_store_commit.h index 8a6eaad5c..b0fcb17e1 100644 --- a/include/paimon/file_store_commit.h +++ b/include/paimon/file_store_commit.h @@ -50,8 +50,9 @@ class CommitMessage; /// manages versions. `UseRESTCatalogCommit()` only prepares a request for the caller to send. class PAIMON_EXPORT FileStoreCommit { public: - /// Create an instance of `FileStoreCommit`. A context that already carries a loaded table goes - /// straight to that table's `NewCommit`; otherwise the table the context names is loaded first. + /// Create an instance of `FileStoreCommit`. The data table the context names is loaded from its + /// root path - a format table is refused, served by the table a catalog hands back through its + /// own `NewCommit` instead. /// /// @param context A unique pointer to the `CommitContext` used for commit operations. /// diff --git a/include/paimon/file_store_write.h b/include/paimon/file_store_write.h index 0778e5bb7..351b98b06 100644 --- a/include/paimon/file_store_write.h +++ b/include/paimon/file_store_write.h @@ -42,8 +42,9 @@ class WriteContext; /// Interface for write operations in a file store. class PAIMON_EXPORT FileStoreWrite { public: - /// Create an instance of `FileStoreWrite`. A context that already carries a loaded table goes - /// straight to that table's `NewWrite`; otherwise the table the context names is loaded first. + /// Create an instance of `FileStoreWrite`. The data table the context names is loaded from its + /// root path - a format table is refused, served by the table a catalog hands back through its + /// own `NewWrite` instead. /// /// @param context A unique pointer to the `WriteContext` used for write operations. /// diff --git a/include/paimon/read_context.h b/include/paimon/read_context.h index d01e28df5..568315ef6 100644 --- a/include/paimon/read_context.h +++ b/include/paimon/read_context.h @@ -38,7 +38,6 @@ namespace paimon { class Catalog; class Executor; struct FullTextSearch; -class FormatTable; class MemoryPool; class Predicate; class FileSystem; @@ -52,23 +51,25 @@ struct VectorSearch; /// @see ReadContextBuilder class PAIMON_EXPORT ReadContext { public: - ReadContext( - const std::string& path, const std::string& branch, - const std::vector& read_field_names, - const std::vector& read_field_ids, const std::shared_ptr& predicate, - const std::shared_ptr& vector_search, - const std::shared_ptr& full_text_search, bool enable_predicate_filter, - bool enable_prefetch, bool enable_late_materializing, uint32_t prefetch_batch_count, - uint32_t prefetch_max_parallel_num, bool enable_multi_thread_row_to_batch, - uint32_t row_to_batch_thread_number, const std::optional& table_schema, - const std::shared_ptr& memory_pool, const std::shared_ptr& executor, - const std::shared_ptr& specific_file_system, - const std::map& fs_scheme_to_identifier_map, - const std::shared_ptr& realtime_context, - const std::map& options, bool read_ahead_cache_enabled, - const CacheConfig& cache_config, const std::shared_ptr& cache, - const std::shared_ptr& format_table, WarmupLevel warmup_level, - const std::shared_ptr& catalog, const std::optional& identifier); + ReadContext(const std::string& path, const std::string& branch, + const std::vector& read_field_names, + const std::vector& read_field_ids, + const std::shared_ptr& predicate, + const std::shared_ptr& vector_search, + const std::shared_ptr& full_text_search, + bool enable_predicate_filter, bool enable_prefetch, bool enable_late_materializing, + uint32_t prefetch_batch_count, uint32_t prefetch_max_parallel_num, + bool enable_multi_thread_row_to_batch, uint32_t row_to_batch_thread_number, + const std::optional& table_schema, + const std::shared_ptr& memory_pool, + const std::shared_ptr& executor, + const std::shared_ptr& specific_file_system, + const std::map& fs_scheme_to_identifier_map, + const std::shared_ptr& realtime_context, + const std::map& options, bool read_ahead_cache_enabled, + const CacheConfig& cache_config, const std::shared_ptr& cache, + WarmupLevel warmup_level, const std::shared_ptr& catalog, + const std::optional& identifier); ~ReadContext(); @@ -164,12 +165,6 @@ class PAIMON_EXPORT ReadContext { return cache_; } - /// The format table this context was built from, or null when it names a table path and the - /// schema under that path says what kind of table it is. - const std::shared_ptr& GetFormatTable() const { - return format_table_; - } - WarmupLevel GetWarmupLevel() const { return warmup_level_; } @@ -225,7 +220,6 @@ class PAIMON_EXPORT ReadContext { bool read_ahead_cache_enabled_; CacheConfig cache_config_; std::shared_ptr cache_; - std::shared_ptr format_table_; WarmupLevel warmup_level_; std::shared_ptr catalog_; std::optional identifier_; @@ -240,15 +234,6 @@ class PAIMON_EXPORT ReadContextBuilder { /// @param path The root path of the table. explicit ReadContextBuilder(const std::string& path); - /// Constructs a `ReadContextBuilder` for a format table that is already loaded: the only way - /// to read one whose schema lives in a metastore rather than under its location, such as a - /// table a REST catalog serves. The table carries what such a location does not say, so - /// `SetTableSchema()`, `WithFileSystem()`, `WithFileSystemSchemeToIdentifierMap()` and a - /// branch are refused here rather than ignored. - /// - /// @param table The format table to read, as `Catalog::GetTable()` hands it back. - explicit ReadContextBuilder(const std::shared_ptr& table); - ~ReadContextBuilder(); ReadContextBuilder(ReadContextBuilder&&) noexcept; diff --git a/include/paimon/scan_context.h b/include/paimon/scan_context.h index 2e3f8d9a8..99f5057eb 100644 --- a/include/paimon/scan_context.h +++ b/include/paimon/scan_context.h @@ -37,7 +37,6 @@ class ScanContextBuilder; class ScanFilter; class Catalog; class Executor; -class FormatTable; class MemoryPool; class Predicate; @@ -57,9 +56,7 @@ class PAIMON_EXPORT ScanContext { const std::shared_ptr& specific_file_system, const std::optional& table_schema, const std::map& options, - const std::shared_ptr& cache, - const std::shared_ptr& format_table, - const std::shared_ptr& catalog, + const std::shared_ptr& cache, const std::shared_ptr& catalog, const std::optional& identifier); ~ScanContext(); @@ -111,12 +108,6 @@ class PAIMON_EXPORT ScanContext { return cache_; } - /// The format table this context was built from, or null when it names a table path and the - /// schema under that path says what kind of table it is. - const std::shared_ptr& GetFormatTable() const { - return format_table_; - } - /// Returns the catalog supplying table metadata, or null if unset. const std::shared_ptr& GetCatalog() const { return catalog_; @@ -140,7 +131,6 @@ class PAIMON_EXPORT ScanContext { std::optional table_schema_; std::map options_; std::shared_ptr cache_; - std::shared_ptr format_table_; std::shared_ptr catalog_; std::optional identifier_; }; @@ -178,14 +168,6 @@ class PAIMON_EXPORT ScanContextBuilder { /// @param path The root path of the table. explicit ScanContextBuilder(const std::string& path); - /// Constructs a `ScanContextBuilder` for a format table that is already loaded: the only way - /// to scan one whose schema lives in a metastore rather than under its location, such as a - /// table a REST catalog serves. The table carries what such a location does not say, so - /// `SetTableSchema()` and `WithFileSystem()` are refused here rather than ignored. - /// - /// @param table The format table to scan, as `Catalog::GetTable()` hands it back. - explicit ScanContextBuilder(const std::shared_ptr& table); - ~ScanContextBuilder(); /// If limit is not set, it defaults to unlimited. ScanContextBuilder& SetLimit(int32_t limit); diff --git a/include/paimon/table/source/table_read.h b/include/paimon/table/source/table_read.h index 5f8303fe3..fa1aef730 100644 --- a/include/paimon/table/source/table_read.h +++ b/include/paimon/table/source/table_read.h @@ -42,8 +42,9 @@ class PAIMON_EXPORT TableRead { public: virtual ~TableRead() = default; - /// Create an instance of `TableRead`. A context that already carries a loaded table goes - /// straight to that table's `NewRead`; otherwise the table the context names is loaded first. + /// Create an instance of `TableRead`. The data table the context names is loaded from its path + /// - a system table behind a `T$system` path and a format table are refused, served by the + /// table a catalog hands back through its own `NewRead` instead. /// /// @param context A unique pointer to the `ReadContext` used for read operations. /// @return A Result containing a unique pointer to the `TableRead` instance. diff --git a/include/paimon/table/source/table_scan.h b/include/paimon/table/source/table_scan.h index 4944dafd2..edecb59c8 100644 --- a/include/paimon/table/source/table_scan.h +++ b/include/paimon/table/source/table_scan.h @@ -36,8 +36,9 @@ class ScanContext; /// A scanner interface for reading table's meta and create a plan. class PAIMON_EXPORT TableScan { public: - /// Create an instance of `TableScan`. A context that already carries a loaded table goes - /// straight to that table's `NewScan`; otherwise the table the context names is loaded first. + /// Create an instance of `TableScan`. The data table the context names is loaded from its path + /// - a system table behind a `T$system` path and a format table are refused, served by the + /// table a catalog hands back through its own `NewScan` instead. /// /// @param context A unique pointer to the `ScanContext` used for scan operations. /// @return A Result containing a unique pointer to the `TableScan` instance. diff --git a/include/paimon/write_context.h b/include/paimon/write_context.h index f1d3ac06e..ba55f37e8 100644 --- a/include/paimon/write_context.h +++ b/include/paimon/write_context.h @@ -33,7 +33,6 @@ namespace paimon { class Catalog; class Executor; -class FormatTable; class MemoryPool; /// `WriteContext` is some configuration for write operations. @@ -53,7 +52,6 @@ class PAIMON_EXPORT WriteContext { const std::map& fs_scheme_to_identifier_map, const std::shared_ptr& realtime_context, const std::map& options, - const std::shared_ptr& format_table, const std::shared_ptr& catalog = nullptr, const std::optional& identifier = std::nullopt); @@ -124,12 +122,6 @@ class PAIMON_EXPORT WriteContext { return realtime_context_; } - /// The format table this context was built from, or null when it names a table path and the - /// schema under that path says what kind of table it is. - const std::shared_ptr& GetFormatTable() const { - return format_table_; - } - /// Returns the catalog supplying table metadata, or null if unset. const std::shared_ptr& GetCatalog() const { return catalog_; @@ -157,7 +149,6 @@ class PAIMON_EXPORT WriteContext { std::map fs_scheme_to_identifier_map_; std::shared_ptr realtime_context_; std::map options_; - std::shared_ptr format_table_; std::shared_ptr catalog_; std::optional identifier_; }; @@ -170,17 +161,6 @@ class PAIMON_EXPORT WriteContextBuilder { /// @param commit_user The user identifier for commit operations. WriteContextBuilder(const std::string& root_path, const std::string& commit_user); - /// Constructs a `WriteContextBuilder` for a format table that is already loaded: the only way - /// to write one whose schema lives in a metastore rather than under its location, such as a - /// table a REST catalog serves. The table carries what such a location does not say, so - /// `WithFileSystem()`, `WithFileSystemSchemeToIdentifierMap()`, `WithCatalog()` and a branch - /// are refused here rather than ignored. - /// - /// There is no commit user: a format table keeps no snapshot to record one in. - /// - /// @param table The format table to write to, as `Catalog::GetTable()` hands it back. - explicit WriteContextBuilder(const std::shared_ptr& table); - ~WriteContextBuilder(); /// Set a configuration options map to set some option entries which are not defined in the diff --git a/src/paimon/core/operation/commit_context.cpp b/src/paimon/core/operation/commit_context.cpp index e61ec950f..c632ac9c6 100644 --- a/src/paimon/core/operation/commit_context.cpp +++ b/src/paimon/core/operation/commit_context.cpp @@ -33,7 +33,6 @@ #include "paimon/memory/memory_pool.h" #include "paimon/result.h" #include "paimon/status.h" -#include "paimon/table/format/format_table.h" namespace paimon { @@ -43,12 +42,11 @@ CommitContext::CommitContext(const std::string& root_path, const std::string& co const std::shared_ptr& memory_pool, const std::shared_ptr& executor, const std::shared_ptr& specific_file_system, - const std::map& options, - const std::shared_ptr& format_table) + const std::map& options) : CommitContext(root_path, commit_user, ignore_empty_commit, use_rest_catalog_commit, /*catalog=*/nullptr, /*identifier=*/std::nullopt, /*table_id=*/std::nullopt, append_commit_check_conflict, memory_pool, executor, specific_file_system, - options, format_table) {} + options) {} CommitContext::CommitContext(const std::string& root_path, const std::string& commit_user, bool ignore_empty_commit, bool use_rest_catalog_commit, @@ -59,8 +57,7 @@ CommitContext::CommitContext(const std::string& root_path, const std::string& co const std::shared_ptr& memory_pool, const std::shared_ptr& executor, const std::shared_ptr& specific_file_system, - const std::map& options, - const std::shared_ptr& format_table) + const std::map& options) : root_path_(root_path), commit_user_(commit_user), ignore_empty_commit_(ignore_empty_commit), @@ -72,8 +69,7 @@ CommitContext::CommitContext(const std::string& root_path, const std::string& co memory_pool_(memory_pool), executor_(executor), specific_file_system_(specific_file_system), - options_(options), - format_table_(format_table) {} + options_(options) {} CommitContext::~CommitContext() = default; @@ -96,10 +92,6 @@ class CommitContextBuilder::Impl { private: std::string root_path_; - /// Kept across `Reset()`, as `root_path_` is: both name the table this builder builds for, - /// rather than a setting of one commit to it. - std::shared_ptr format_table_; - bool built_from_format_table_ = false; std::string commit_user_; bool ignore_empty_commit_ = true; bool use_rest_catalog_commit_ = false; @@ -120,15 +112,6 @@ CommitContextBuilder::CommitContextBuilder(const std::string& root_path, impl_->commit_user_ = commit_user; } -CommitContextBuilder::CommitContextBuilder(const std::shared_ptr& table) - : impl_(std::make_unique()) { - impl_->format_table_ = table; - impl_->built_from_format_table_ = true; - if (table != nullptr) { - impl_->root_path_ = table->Location(); - } -} - CommitContextBuilder::~CommitContextBuilder() = default; CommitContextBuilder& CommitContextBuilder::AddOption(const std::string& key, @@ -190,9 +173,6 @@ CommitContextBuilder& CommitContextBuilder::WithFileSystem( } Result> CommitContextBuilder::Finish() { - if (impl_->built_from_format_table_ && impl_->format_table_ == nullptr) { - return Status::Invalid("cannot commit with null format table"); - } if (impl_->catalog_ == nullptr && impl_->identifier_) { return Status::Invalid("cannot commit through a null catalog"); } @@ -230,13 +210,6 @@ Result> CommitContextBuilder::Finish() { "a commit either goes to the catalog of WithCatalog() or is handed back as a request " "for the caller to send, which is what UseRESTCatalogCommit() asks for"); } - // The table already carries the file system it was loaded through, and from a source this - // cannot see behind, so a second answer is refused rather than silently dropped. - if (impl_->format_table_ != nullptr && impl_->specific_file_system_ != nullptr) { - return Status::Invalid( - "a format table carries the file system it was loaded through, so WithFileSystem() " - "cannot be used with one"); - } PAIMON_ASSIGN_OR_RAISE(impl_->root_path_, PathUtil::NormalizePath(impl_->root_path_)); if (impl_->root_path_.empty()) { return Status::Invalid("root path is empty"); @@ -245,7 +218,7 @@ Result> CommitContextBuilder::Finish() { impl_->root_path_, impl_->commit_user_, impl_->ignore_empty_commit_, impl_->use_rest_catalog_commit_, impl_->catalog_, impl_->identifier_, impl_->table_id_, impl_->append_commit_check_conflict_, impl_->memory_pool_, impl_->executor_, - impl_->specific_file_system_, impl_->options_, impl_->format_table_); + impl_->specific_file_system_, impl_->options_); impl_->Reset(); return ctx; } diff --git a/src/paimon/core/operation/commit_context_test.cpp b/src/paimon/core/operation/commit_context_test.cpp index 51fcab399..d86a79149 100644 --- a/src/paimon/core/operation/commit_context_test.cpp +++ b/src/paimon/core/operation/commit_context_test.cpp @@ -40,7 +40,7 @@ TEST(CommitContextTest, TestLegacyConstructor) { auto fs = std::make_shared(); const std::map options = {{"key", "value"}}; CommitContext context("table_root_path", "commit_user_1", false, true, true, pool, executor, fs, - options, nullptr); + options); ASSERT_EQ(context.GetRootPath(), "table_root_path"); ASSERT_EQ(context.GetCommitUser(), "commit_user_1"); ASSERT_FALSE(context.IgnoreEmptyCommit()); diff --git a/src/paimon/core/operation/file_store_commit_test.cpp b/src/paimon/core/operation/file_store_commit_test.cpp index a7f39628d..c67c761c5 100644 --- a/src/paimon/core/operation/file_store_commit_test.cpp +++ b/src/paimon/core/operation/file_store_commit_test.cpp @@ -99,8 +99,7 @@ TEST(FileStoreCommitTest, TestCreateWithCatalogRequiresIdentifier) { dir->Str(), "commit_user", /*ignore_empty_commit=*/true, /*use_rest_catalog_commit=*/false, catalog, /*identifier=*/std::nullopt, /*table_id=*/std::nullopt, /*append_commit_check_conflict=*/false, GetDefaultPool(), - CreateDefaultExecutor(), /*specific_file_system=*/nullptr, options, - /*format_table=*/nullptr); + CreateDefaultExecutor(), /*specific_file_system=*/nullptr, options); ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(commit_context)), "a catalog commit requires a table identifier"); } diff --git a/src/paimon/core/operation/read_context.cpp b/src/paimon/core/operation/read_context.cpp index bc65b4661..40ed358de 100644 --- a/src/paimon/core/operation/read_context.cpp +++ b/src/paimon/core/operation/read_context.cpp @@ -32,7 +32,6 @@ #include "paimon/predicate/full_text_search.h" #include "paimon/predicate/vector_search.h" #include "paimon/status.h" -#include "paimon/table/format/format_table.h" namespace paimon { class Predicate; @@ -50,8 +49,7 @@ ReadContext::ReadContext( const std::map& fs_scheme_to_identifier_map, const std::shared_ptr& realtime_context, const std::map& options, bool read_ahead_cache_enabled, - const CacheConfig& cache_config, const std::shared_ptr& cache, - const std::shared_ptr& format_table, WarmupLevel warmup_level, + const CacheConfig& cache_config, const std::shared_ptr& cache, WarmupLevel warmup_level, const std::shared_ptr& catalog, const std::optional& identifier) : path_(path), branch_(branch), @@ -77,7 +75,6 @@ ReadContext::ReadContext( read_ahead_cache_enabled_(read_ahead_cache_enabled), cache_config_(cache_config), cache_(cache), - format_table_(format_table), warmup_level_(warmup_level), catalog_(catalog), identifier_(identifier) {} @@ -135,10 +132,6 @@ class ReadContextBuilder::Impl { private: std::string path_; - /// Kept across `Reset()`, as `path_` is: both name the table this builder builds for, rather - /// than a setting of one read of it. - std::shared_ptr format_table_; - bool built_from_format_table_ = false; /// Unset until `WithBranch()` names one, so that a branch named only by the `branch` /// option is not overruled by the default. std::optional branch_; @@ -175,15 +168,6 @@ ReadContextBuilder::ReadContextBuilder(const std::string& path) impl_->path_ = path; } -ReadContextBuilder::ReadContextBuilder(const std::shared_ptr& table) - : impl_(std::make_unique()) { - impl_->format_table_ = table; - impl_->built_from_format_table_ = true; - if (table != nullptr) { - impl_->path_ = table->Location(); - } -} - ReadContextBuilder::~ReadContextBuilder() = default; ReadContextBuilder::ReadContextBuilder(ReadContextBuilder&&) noexcept = default; @@ -359,34 +343,6 @@ Result> ReadContextBuilder::Finish() { return Status::Invalid("FullTextSearch field name and query must not be empty"); } } - if (impl_->built_from_format_table_ && impl_->format_table_ == nullptr) { - return Status::Invalid("cannot read with null format table"); - } - if (impl_->format_table_ != nullptr) { - // The table already answers each of these, and from a source this cannot see behind, so a - // second answer is refused rather than silently dropped. - if (impl_->catalog_ != nullptr) { - return Status::Invalid( - "a format table carries the file system it was loaded through, so WithCatalog() " - "cannot be used with one"); - } - if (impl_->table_schema_) { - return Status::Invalid( - "a format table carries its own schema, so SetTableSchema() cannot be used with " - "one"); - } - if (impl_->specific_file_system_ != nullptr || - !impl_->fs_scheme_to_identifier_map_.empty()) { - return Status::Invalid( - "a format table carries the file system it was loaded through, so WithFileSystem() " - "and WithFileSystemSchemeToIdentifierMap() cannot be used with one"); - } - if (impl_->branch_ && - !BranchManager::IsMainBranch(BranchManager::NormalizeBranch(impl_->branch_.value()))) { - return Status::Invalid( - "a format table has no branches, so WithBranch() cannot be used with one"); - } - } PAIMON_ASSIGN_OR_RAISE(impl_->path_, PathUtil::NormalizePath(impl_->path_)); if (impl_->path_.empty()) { return Status::Invalid("cannot read with empty table path"); @@ -436,8 +392,8 @@ Result> ReadContextBuilder::Finish() { impl_->row_to_batch_thread_number_, impl_->table_schema_, impl_->memory_pool_, impl_->executor_, impl_->specific_file_system_, impl_->fs_scheme_to_identifier_map_, impl_->realtime_context_, impl_->options_, impl_->read_ahead_cache_enabled_, - impl_->cache_config_, impl_->cache_, impl_->format_table_, impl_->warmup_level_, - impl_->catalog_, impl_->identifier_); + impl_->cache_config_, impl_->cache_, impl_->warmup_level_, impl_->catalog_, + impl_->identifier_); if (impl_->read_schema_ && impl_->read_schema_->release) { ctx->SetReadSchema(std::move(impl_->read_schema_)); } diff --git a/src/paimon/core/operation/scan_context.cpp b/src/paimon/core/operation/scan_context.cpp index 89cccaa69..3fd859c5b 100644 --- a/src/paimon/core/operation/scan_context.cpp +++ b/src/paimon/core/operation/scan_context.cpp @@ -25,7 +25,6 @@ #include "paimon/executor.h" #include "paimon/memory/memory_pool.h" #include "paimon/status.h" -#include "paimon/table/format/format_table.h" namespace paimon { class Predicate; @@ -39,8 +38,7 @@ ScanContext::ScanContext( const std::shared_ptr& specific_file_system, const std::optional& table_schema, const std::map& options, const std::shared_ptr& cache, - const std::shared_ptr& format_table, const std::shared_ptr& catalog, - const std::optional& identifier) + const std::shared_ptr& catalog, const std::optional& identifier) : path_(path), is_streaming_mode_(is_streaming_mode), limit_(limit), @@ -53,7 +51,6 @@ ScanContext::ScanContext( table_schema_(table_schema), options_(options), cache_(cache), - format_table_(format_table), catalog_(catalog), identifier_(identifier) {} @@ -83,10 +80,6 @@ class ScanContextBuilder::Impl { private: std::string path_; - /// Kept across `Reset()`, as `path_` is: both name the table this builder builds for, rather - /// than a setting of one scan of it. - std::shared_ptr format_table_; - bool built_from_format_table_ = false; bool is_streaming_mode_ = false; std::optional limit_; std::optional bucket_filter_; @@ -110,15 +103,6 @@ ScanContextBuilder::ScanContextBuilder(const std::string& path) impl_->path_ = path; } -ScanContextBuilder::ScanContextBuilder(const std::shared_ptr& table) - : impl_(std::make_unique()) { - impl_->format_table_ = table; - impl_->built_from_format_table_ = true; - if (table != nullptr) { - impl_->path_ = table->Location(); - } -} - ScanContextBuilder::~ScanContextBuilder() = default; ScanContextBuilder& ScanContextBuilder::WithStreamingMode(bool is_streaming_mode) { impl_->is_streaming_mode_ = is_streaming_mode; @@ -205,28 +189,6 @@ ScanContextBuilder& ScanContextBuilder::WithCache(const std::shared_ptr& } Result> ScanContextBuilder::Finish() { - if (impl_->built_from_format_table_ && impl_->format_table_ == nullptr) { - return Status::Invalid("cannot scan with null format table"); - } - if (impl_->format_table_ != nullptr) { - // The table already answers both, and from a source this cannot see behind, so a second - // answer is refused rather than silently dropped. - if (impl_->catalog_ != nullptr) { - return Status::Invalid( - "a format table carries the file system it was loaded through, so WithCatalog() " - "cannot be used with one"); - } - if (impl_->table_schema_) { - return Status::Invalid( - "a format table carries its own schema, so SetTableSchema() cannot be used with " - "one"); - } - if (impl_->specific_file_system_ != nullptr) { - return Status::Invalid( - "a format table carries the file system it was loaded through, so WithFileSystem() " - "cannot be used with one"); - } - } PAIMON_ASSIGN_OR_RAISE(impl_->path_, PathUtil::NormalizePath(impl_->path_)); if (impl_->path_.empty()) { return Status::Invalid("cannot scan with empty table path"); @@ -248,7 +210,7 @@ Result> ScanContextBuilder::Finish() { impl_->bucket_filter_), impl_->global_index_result_, impl_->realtime_context_, impl_->memory_pool_, executor, impl_->specific_file_system_, impl_->table_schema_, impl_->options_, impl_->cache_, - impl_->format_table_, impl_->catalog_, impl_->identifier_); + impl_->catalog_, impl_->identifier_); impl_->Reset(); return ctx; } diff --git a/src/paimon/core/operation/write_context.cpp b/src/paimon/core/operation/write_context.cpp index e31d64ac4..2b890f934 100644 --- a/src/paimon/core/operation/write_context.cpp +++ b/src/paimon/core/operation/write_context.cpp @@ -30,7 +30,6 @@ #include "paimon/memory/memory_pool.h" #include "paimon/result.h" #include "paimon/status.h" -#include "paimon/table/format/format_table.h" namespace paimon { @@ -43,8 +42,7 @@ WriteContext::WriteContext( const std::shared_ptr& specific_file_system, const std::map& fs_scheme_to_identifier_map, const std::shared_ptr& realtime_context, - const std::map& options, - const std::shared_ptr& format_table, const std::shared_ptr& catalog, + const std::map& options, const std::shared_ptr& catalog, const std::optional& identifier) : root_path_(root_path), commit_user_(commit_user), @@ -62,7 +60,6 @@ WriteContext::WriteContext( fs_scheme_to_identifier_map_(fs_scheme_to_identifier_map), realtime_context_(realtime_context), options_(options), - format_table_(format_table), catalog_(catalog), identifier_(identifier) {} @@ -93,10 +90,6 @@ class WriteContextBuilder::Impl { private: std::string root_path_; - /// Kept across `Reset()`, as `root_path_` is: both name the table this builder builds for, - /// rather than a setting of one write to it. - std::shared_ptr format_table_; - bool built_from_format_table_ = false; std::string commit_user_; /// Unset until `WithBranch()` names one, so that a branch named only by the identifier or /// the `branch` option is not overruled by the default. @@ -125,15 +118,6 @@ WriteContextBuilder::WriteContextBuilder(const std::string& root_path, impl_->commit_user_ = commit_user; } -WriteContextBuilder::WriteContextBuilder(const std::shared_ptr& table) - : impl_(std::make_unique()) { - impl_->format_table_ = table; - impl_->built_from_format_table_ = true; - if (table != nullptr) { - impl_->root_path_ = table->Location(); - } -} - WriteContextBuilder::~WriteContextBuilder() = default; WriteContextBuilder& WriteContextBuilder::AddOption(const std::string& key, @@ -226,27 +210,6 @@ WriteContextBuilder& WriteContextBuilder::WithCatalog(const std::shared_ptr> WriteContextBuilder::Finish() { - if (impl_->built_from_format_table_ && impl_->format_table_ == nullptr) { - return Status::Invalid("cannot write with null format table"); - } - if (impl_->format_table_ != nullptr) { - if (impl_->catalog_ != nullptr) { - return Status::Invalid("WithCatalog() requires a native table"); - } - // The table already answers each of these, and from a source this cannot see behind, so a - // second answer is refused rather than silently dropped. - if (impl_->specific_file_system_ != nullptr || - !impl_->fs_scheme_to_identifier_map_.empty()) { - return Status::Invalid( - "a format table carries the file system it was loaded through, so WithFileSystem() " - "and WithFileSystemSchemeToIdentifierMap() cannot be used with one"); - } - if (impl_->branch_ && - !BranchManager::IsMainBranch(BranchManager::NormalizeBranch(impl_->branch_.value()))) { - return Status::Invalid( - "a format table has no branches, so WithBranch() cannot be used with one"); - } - } PAIMON_ASSIGN_OR_RAISE(impl_->root_path_, PathUtil::NormalizePath(impl_->root_path_)); if (impl_->root_path_.empty()) { return Status::Invalid("root path is empty"); @@ -286,8 +249,7 @@ Result> WriteContextBuilder::Finish() { impl_->ignore_num_bucket_check_, impl_->ignore_previous_files_, enable_multi_thread_spill, impl_->write_id_, branch, impl_->write_schema_, impl_->memory_pool_, impl_->executor_, impl_->temp_directory_, impl_->specific_file_system_, impl_->fs_scheme_to_identifier_map_, - impl_->realtime_context_, impl_->options_, impl_->format_table_, impl_->catalog_, - impl_->identifier_); + impl_->realtime_context_, impl_->options_, impl_->catalog_, impl_->identifier_); impl_->Reset(); return ctx; } diff --git a/src/paimon/core/table/file_store_table_factory.cpp b/src/paimon/core/table/file_store_table_factory.cpp index d5082ef73..5eca88fcf 100644 --- a/src/paimon/core/table/file_store_table_factory.cpp +++ b/src/paimon/core/table/file_store_table_factory.cpp @@ -29,7 +29,6 @@ #include "paimon/catalog/identifier.h" #include "paimon/catalog/table.h" #include "paimon/commit_context.h" -#include "paimon/common/utils/checked_cast.h" #include "paimon/core/catalog/catalog_utils.h" #include "paimon/core/catalog/version_managed_catalog.h" #include "paimon/core/core_options.h" @@ -41,52 +40,17 @@ #include "paimon/core/table/system/system_table.h" #include "paimon/core/utils/branch_manager.h" #include "paimon/fs/file_system.h" -#include "paimon/predicate/predicate_utils.h" #include "paimon/read_context.h" #include "paimon/result.h" #include "paimon/scan_context.h" #include "paimon/schema/schema.h" #include "paimon/status.h" -#include "paimon/table/format/format_table.h" #include "paimon/write_context.h" namespace paimon { namespace { -/// Recognises a format table behind `table_path`, or returns null when it is not one. The scan, -/// read, write and commit entry points below all dispatch on the same question, so one place -/// answers it and they cannot disagree. Null is also the answer when no schema was found under -/// the path at all: what to say about a table that is not there is the managed path's to decide. -/// -/// The table type is read from the schema, never from `options`: `type` is structural, and one -/// call's options must not decide what kind of table this is. -Result> TryLoadFormatTable( - const std::shared_ptr& file_system, const std::string& table_path, - const std::string& branch, const std::map& options, - const std::shared_ptr& table_schema) { - if (table_schema == nullptr) { - return std::shared_ptr(); - } - // From the schema alone: see `FormatTable::Create()`. - PAIMON_ASSIGN_OR_RAISE(bool is_format_table, - TableTypeDefine::IsFormatTable(table_schema->Options())); - if (!is_format_table) { - return std::shared_ptr(); - } - // A format table has no branches, so it is refused rather than served from the table path - // under another branch's name. See `SchemaValidation::ValidateFormatTableSchema()`. - if (BranchManager::NormalizeBranch(branch) != BranchManager::DEFAULT_MAIN_BRANCH) { - return Status::Invalid( - fmt::format("a format table has no branches, so {} cannot be read on branch '{}'", - table_path, branch)); - } - Identifier identifier = CatalogUtils::IdentifierFromPath(table_path, branch); - return FormatTable::Create(file_system, table_path, identifier, - checked_pointer_cast(table_schema), - /*location_carries_paimon_metadata=*/true, options); -} - /// The options a context carries, merged into the table the load hands back: a table is the /// product of the load, so the options speak there - the way `Copy` merges them - and `NewXxx` on /// the product serves them without merging again. A caller holding a table already changes it @@ -108,13 +72,13 @@ Result> WithContextOptions( return table; } -/// The tail every `Create` ends with: the schema one of them read says what the path names. A -/// format table is handed back as the `FormatTable` it is, everything else as the `FileStoreTable` +/// The tail every `Create` ends with: the schema one of them read says what the path names, and +/// only a data table is served by it - a format table is refused, the table a catalog hands back +/// serves one through its own `NewXxx`, and everything else is handed back as the `FileStoreTable` /// a managed table is, carrying the schema for its `NewXxx` to build from. A null schema is handed -/// over as it is - see `TryLoadFormatTable`. +/// over as it is, so the managed path below decides what to say about a table that is not there. /// -/// @param file_system The file system the schema above was read through, which a format table -/// is served by - it has no schema options of its own to name another one. +/// @param file_system The file system the schema above was read through. /// @param explicit_file_system The file system the context supplied, which a managed table is /// opened through when there is one. A null one leaves the table to resolve from its /// schema's own access configuration rather than being pinned to the file system this load @@ -126,10 +90,19 @@ Result> TableOfLoadedSchema( const std::shared_ptr& schema_manager, const std::shared_ptr& latest_schema, const std::optional& identifier, const std::string& uuid, const std::map& fs_scheme_map) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, - TryLoadFormatTable(file_system, path, branch, options, latest_schema)); - if (format_table != nullptr) { - return std::shared_ptr
(std::move(format_table)); + if (latest_schema != nullptr) { + // The table type is read from the schema, never from `options`: `type` is structural, and + // one call's options must not decide what kind of table this is. See + // `FormatTable::Create()`. + PAIMON_ASSIGN_OR_RAISE(bool is_format_table, + TableTypeDefine::IsFormatTable(latest_schema->Options())); + if (is_format_table) { + return Status::Invalid(fmt::format( + "the schema under {} names a format table, which is not loaded from a path: a " + "catalog hands the table back through GetTable, and its operators come from " + "table->NewScan/NewRead/NewWrite/NewCommit on it", + path)); + } } // The branch the schema was read on is folded into the object name. Identifier table_identifier = @@ -210,24 +183,21 @@ Result> CatalogMainBranchSchemaJson( } // namespace -/// The table a scan context names: the format table the builder was handed, a system table behind -/// a `T$system` path, or the table the schema under the context's path says is there. +/// The data table a scan context names: only a data table is served by a load from the context's +/// path - a system table behind a `T$system` path and a format table are refused, served by the +/// table a catalog hands back through its own `NewXxx` instead. Result> FileStoreTableFactory::Create(const ScanContext& ctx) { - if (ctx.GetFormatTable() != nullptr) { - return std::shared_ptr
(ctx.GetFormatTable()); - } PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, CoreOptions::FromMap(ctx.GetOptions(), ctx.GetSpecificFileSystem(), {})); PAIMON_ASSIGN_OR_RAISE(std::optional system_table_path, SystemTableLoader::TryParsePath(ctx.GetPath())); if (system_table_path) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, - SystemTableLoader::LoadFromPath(tmp_options.GetFileSystem(), - ctx.GetPath(), ctx.GetOptions())); - return std::shared_ptr
(std::move(system_table)); + return Status::Invalid(fmt::format( + "the path {} names a system table, which is not loaded from a path: a catalog " + "hands the table back through GetTable, and its operators come from " + "table->NewScan/NewRead on it", + ctx.GetPath())); } - // One `TableScan` interface serves a format table too, the way Java Paimon serves both - // through one `ReadBuilder`; a format table is planned by listing directories instead. const std::string branch = BranchManager::NormalizeBranch(tmp_options.GetBranch()); return ManagedTableOf(tmp_options.GetFileSystem(), ctx.GetSpecificFileSystem(), ctx.GetPath(), branch, ctx.GetOptions(), ctx.GetSpecificTableSchema(), @@ -235,35 +205,23 @@ Result> FileStoreTableFactory::Create(const ScanContext& /*uuid=*/"", /*fs_scheme_map=*/{}); } -/// The table a read context names: the format table the builder was handed, a system table behind -/// a `T$system` path, or the table the schema under the context's path says is there. +/// The data table a read context names: only a data table is served by a load from the context's +/// path - a system table behind a `T$system` path and a format table are refused, served by the +/// table a catalog hands back through its own `NewXxx` instead. Result> FileStoreTableFactory::Create(const ReadContext& ctx) { - if (ctx.GetFormatTable() != nullptr) { - return std::shared_ptr
(ctx.GetFormatTable()); - } PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_core_options, CoreOptions::FromMap(ctx.GetOptions(), ctx.GetSpecificFileSystem(), ctx.GetFileSystemSchemeToIdentifierMap())); PAIMON_ASSIGN_OR_RAISE(std::optional system_table_path, SystemTableLoader::TryParsePath(ctx.GetPath())); if (system_table_path) { - if (PredicateUtils::ContainsFullTextMatch(ctx.GetPredicate())) { - return Status::NotImplemented( - "FullTextMatch predicate execution is not implemented yet"); - } - if (ctx.HasVectorOrFullTextSearch()) { - return Status::NotImplemented( - "VectorSearch or FullTextSearch is not supported for system tables"); - } - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, - SystemTableLoader::LoadFromPath(tmp_core_options.GetFileSystem(), - ctx.GetPath(), ctx.GetOptions())); - return std::shared_ptr
(std::move(system_table)); + return Status::Invalid(fmt::format( + "the path {} names a system table, which is not loaded from a path: a catalog " + "hands the table back through GetTable, and its operators come from " + "table->NewScan/NewRead on it", + ctx.GetPath())); } - // A format table has no manifest and a file another engine wrote carries no field ids, so how - // a row is put back together is its own; how a file is opened is not - both paths go through - // `DataFileReaderFactory`. const std::string branch = ctx.GetBranch(); // The scheme map the schema was read through, so the table's data resolves custom schemes the // same way its metadata does. @@ -273,12 +231,10 @@ Result> FileStoreTableFactory::Create(const ReadContext& ctx.GetFileSystemSchemeToIdentifierMap()); } -/// The table a write context names: the format table the builder was handed, or the table the -/// schema under the context's root path says is there. +/// The data table a write context names: only a data table is served by a load from the context's +/// root path - a format table is refused, served by the table a catalog hands back through its own +/// `NewXxx` instead. Result> FileStoreTableFactory::Create(const WriteContext& ctx) { - if (ctx.GetFormatTable() != nullptr) { - return std::shared_ptr
(ctx.GetFormatTable()); - } PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx.GetCatalog())); std::shared_ptr specific_fs = ctx.GetSpecificFileSystem(); if (specific_fs == nullptr && ctx.GetFileSystemSchemeToIdentifierMap().empty() && @@ -307,12 +263,10 @@ Result> FileStoreTableFactory::Create(const WriteContext& ctx.GetFileSystemSchemeToIdentifierMap()); } -/// The table a commit context names: the format table the builder was handed, or the table the -/// schema under the context's root path says is there. +/// The data table a commit context names: only a data table is served by a load from the context's +/// root path - a format table is refused, served by the table a catalog hands back through its own +/// `NewXxx` instead. Result> FileStoreTableFactory::Create(const CommitContext& ctx) { - if (ctx.GetFormatTable() != nullptr) { - return std::shared_ptr
(ctx.GetFormatTable()); - } PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx.GetCatalog())); // Use this table's catalog credentials unless the caller supplied a file system: a catalog // issuing per-table temporary ones only hands them out through GetTableFileSystem. diff --git a/src/paimon/core/table/file_store_table_factory.h b/src/paimon/core/table/file_store_table_factory.h index a1bb77062..e1090dc97 100644 --- a/src/paimon/core/table/file_store_table_factory.h +++ b/src/paimon/core/table/file_store_table_factory.h @@ -30,10 +30,11 @@ class ScanContext; class Table; class WriteContext; -/// Loads the table a context names, as Java Paimon's `FileStoreTableFactory` does: the format -/// table the builder was handed, the system table behind a `T$system` path, or the managed table -/// the schema under the context's path describes. The static `Create` of all four operators asks -/// the same question, so one place answers it and they cannot disagree. +/// Loads the data table a context names, as Java Paimon's `FileStoreTableFactory` does: the schema +/// under the context's path describes it. A system table behind a `T$system` path and a format +/// table are refused - a catalog hands those back through `GetTable`, and their operators come from +/// the table itself - so the static `Create` of all four operators answers the one question this +/// place asks and they cannot disagree. class FileStoreTableFactory { public: FileStoreTableFactory() = delete; diff --git a/src/paimon/core/table/format/format_table.cpp b/src/paimon/core/table/format/format_table.cpp index f14ac66d1..acb5f03e1 100644 --- a/src/paimon/core/table/format/format_table.cpp +++ b/src/paimon/core/table/format/format_table.cpp @@ -297,8 +297,8 @@ Result> FormatTable::NewWrite( PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, CopySelf(context->GetOptions())); if (context->GetCatalog() != nullptr) { return Status::Invalid( - "WithCatalog() requires a native table; use WriteContextBuilder(FormatTable) for a " - "format table"); + "WithCatalog() requires a native table; a format table takes a WriteContextBuilder " + "built from its location instead"); } if (context->IsStreamingMode()) { return Status::Invalid( diff --git a/src/paimon/core/table/format/format_table_test.cpp b/src/paimon/core/table/format/format_table_test.cpp index 84fbce351..af0686c6f 100644 --- a/src/paimon/core/table/format/format_table_test.cpp +++ b/src/paimon/core/table/format/format_table_test.cpp @@ -763,14 +763,14 @@ TEST(FormatTableTest, TestCatalogManagedPartitionsAreRejected) { ASSERT_TRUE(dynamic_table.status().IsNotImplemented()) << dynamic_table.status().ToString(); } -TEST(FormatTableTest, TestTheGenericEntryPointsValidateOptionsGivenAtTheCall) { - // An option a format table refuses has to be refused wherever it comes from. Every generic - // entry point merges what the call gave it over what the schema stored, so each runs the same - // checks over the merged result; otherwise a caller could set through a context what the - // schema would have rejected and have it silently dropped. +TEST(FormatTableTest, TestTheTableOperatorsValidateOptionsGivenAtTheCall) { + // An option a format table refuses has to be refused wherever it comes from. Every operator + // merges what the call gave it over what the schema stored, so each runs the same checks over + // the merged result; otherwise a caller could set through a context what the schema would have + // rejected and have it silently dropped. std::unique_ptr dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); - ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::shared_ptr table, + ASSERT_OK_AND_ASSIGN(std::shared_ptr table, CreateTable(dir->GetFileSystem(), dir->Str(), {"dt"})); const std::map refused = { {Options::METASTORE_PARTITIONED_TABLE, "true"}}; @@ -778,24 +778,22 @@ TEST(FormatTableTest, TestTheGenericEntryPointsValidateOptionsGivenAtTheCall) { ScanContextBuilder scan_builder(dir->Str()); scan_builder.SetOptions(refused); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), "metastore.partitioned-table"); + ASSERT_NOK_WITH_MSG(table->NewScan(std::move(scan_context)), "metastore.partitioned-table"); ReadContextBuilder read_builder(dir->Str()); read_builder.SetOptions(refused); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), "metastore.partitioned-table"); + ASSERT_NOK_WITH_MSG(table->NewRead(std::move(read_context)), "metastore.partitioned-table"); WriteContextBuilder write_builder(dir->Str(), "test-user"); write_builder.SetOptions(refused); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_context)), - "metastore.partitioned-table"); + ASSERT_NOK_WITH_MSG(table->NewWrite(std::move(write_context)), "metastore.partitioned-table"); CommitContextBuilder commit_builder(dir->Str(), "test-user"); commit_builder.SetOptions(refused); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(commit_context)), - "metastore.partitioned-table"); + ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(commit_context)), "metastore.partitioned-table"); } TEST(FormatTableTest, TestValueOnlyPartitionCannotBeNamedAfterThisTablesMetadata) { @@ -2667,19 +2665,20 @@ TEST(FormatTableTest, TestScanRejectsUnknownPartitionField) { ASSERT_TRUE(scan.status().IsInvalid()); } -TEST(FormatTableTest, TestTheGenericEntryPointsReachAFormatTable) { - // A caller holding a table path uses the interfaces it uses for every other table. Each one - // recognises a format table from its schema and dispatches to it, the way Java Paimon serves - // both kinds through one `ReadBuilder` and one `BatchWriteBuilder`. +TEST(FormatTableTest, TestAFormatTableIsWrittenCommittedAndReadBack) { + // A caller holding the table a catalog hands back drives it through the operators the table + // itself serves, with contexts built from its location the way they are for every other + // table - the C++ side of what Java serves through one `ReadBuilder` and one + // `BatchWriteBuilder` for both kinds of table. std::unique_ptr dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); - ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::shared_ptr table, + ASSERT_OK_AND_ASSIGN(std::shared_ptr table, CreateTable(dir->GetFileSystem(), dir->Str(), {"dt"})); WriteContextBuilder write_builder(dir->Str(), "test-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr write, - FileStoreWrite::Create(std::move(write_context))); + table->NewWrite(std::move(write_context))); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, MakeBatch({1, 2}, {"alice", "bob"}, "20240101", {{"dt", "20240101"}})); ASSERT_OK(write->Write(std::move(batch))); @@ -2696,26 +2695,26 @@ TEST(FormatTableTest, TestTheGenericEntryPointsReachAFormatTable) { WriteContextBuilder write_id_builder(dir->Str(), "test-user"); write_id_builder.WithWriteId(3); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_id_context, write_id_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_id_context)), + ASSERT_NOK_WITH_MSG(table->NewWrite(std::move(write_id_context)), "a write id would name nothing"); CommitContextBuilder empty_commit_builder(dir->Str(), "test-user"); empty_commit_builder.IgnoreEmptyCommit(false); ASSERT_OK_AND_ASSIGN(std::unique_ptr empty_commit_context, empty_commit_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(empty_commit_context)), + ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(empty_commit_context)), "cannot record an empty commit"); CommitContextBuilder rest_builder(dir->Str(), "test-user"); rest_builder.UseRESTCatalogCommit(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr rest_context, rest_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(rest_context)), "rest catalog"); + ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(rest_context)), "rest catalog"); CommitContextBuilder table_id_builder(dir->Str(), "test-user"); table_id_builder.WithTableId("table-uuid"); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_id_context, table_id_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(table_id_context)), + ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(table_id_context)), "a format table sends no commit table request"); auto format_catalog = std::make_shared(); @@ -2723,20 +2722,20 @@ TEST(FormatTableTest, TestTheGenericEntryPointsReachAFormatTable) { CommitContextBuilder catalog_builder(dir->Str(), "test-user"); catalog_builder.WithCatalog(format_catalog, Identifier("db", "fmt")); ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog_context, catalog_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(catalog_context)), + ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(catalog_context)), "keeps no snapshot for a catalog to take"); CommitContextBuilder conflict_builder(dir->Str(), "test-user"); conflict_builder.AppendCommitCheckConflict(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr conflict_context, conflict_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(conflict_context)), + ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(conflict_context)), "no manifests to check a concurrent commit against"); CommitContextBuilder commit_builder(dir->Str(), "test-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_context))); + table->NewCommit(std::move(commit_context))); ASSERT_OK(commit->Commit(messages)); // The same for the snapshot half of the commit interface. ASSERT_NOK_WITH_MSG(commit->Expire(), "no snapshots to expire"); @@ -2745,15 +2744,13 @@ TEST(FormatTableTest, TestTheGenericEntryPointsReachAFormatTable) { ScanContextBuilder scan_builder(dir->Str()); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, - TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); ASSERT_EQ(plan->Splits().size(), 1u); ReadContextBuilder read_builder(dir->Str()); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr read, - TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, table->NewRead(std::move(read_context))); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch read_batch, reader->NextBatch()); ASSERT_FALSE(BatchReader::IsEofBatch(read_batch)); @@ -2795,11 +2792,12 @@ TEST(FormatTableTest, TestACatalogInTheContextIsRefusedAlikeByWriteAndCommit) { ASSERT_TRUE(commit.status().IsInvalid()); } -TEST(FormatTableTest, TestTheGenericEntryPointsTakeALoadedFormatTable) { +TEST(FormatTableTest, TestALoadedFormatTableIsServedThroughItsOwnOperators) { // A table a catalog keeps the schema for, the way a rest catalog does. Nothing under its // location says that it is a format table, nor that everything below the location is data, so - // a path is not enough to reach it: the loaded table is handed to the context instead, which - // is what `FormatTable.newReadBuilder()` and `newBatchWriteBuilder()` do in Java. + // a path is not enough to reach it: the loaded table serves the operators itself, with a + // context built from its location, which is what `FormatTable.newReadBuilder()` and + // `newBatchWriteBuilder()` do in Java. std::unique_ptr schema_dir = UniqueTestDirectory::Create(); ASSERT_TRUE(schema_dir); SchemaManager schema_manager(schema_dir->GetFileSystem(), schema_dir->Str()); @@ -2826,11 +2824,11 @@ TEST(FormatTableTest, TestTheGenericEntryPointsTakeALoadedFormatTable) { // Options given at the call still win over the ones the schema stored, so a file is rolled // per row rather than one file holding all three. - WriteContextBuilder write_builder(external); + WriteContextBuilder write_builder(external->Location(), "test-user"); write_builder.SetOptions({{Options::TARGET_FILE_ROW_NUM, "1"}}); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr write, - FileStoreWrite::Create(std::move(write_context))); + external->NewWrite(std::move(write_context))); for (int32_t i = 1; i <= 3; i++) { ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, MakeBatch({i}, {"name"}, "20240101", {{"dt", "20240101"}})); @@ -2840,25 +2838,25 @@ TEST(FormatTableTest, TestTheGenericEntryPointsTakeALoadedFormatTable) { write->PrepareCommit()); ASSERT_EQ(messages.size(), 3u) << "target-file-row-num given at the call was ignored"; - CommitContextBuilder commit_builder(external); + CommitContextBuilder commit_builder(external->Location(), "test-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_context))); + external->NewCommit(std::move(commit_context))); ASSERT_OK(commit->Commit(messages)); ASSERT_OK(write->Close()); - ScanContextBuilder scan_builder(external); + ScanContextBuilder scan_builder(external->Location()); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, - TableScan::Create(std::move(scan_context))); + external->NewScan(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(PartitionList partitions, scan->ListPartitions()); ASSERT_EQ(partitions, (PartitionList{{{"dt", "20240101"}}})); - ReadContextBuilder read_builder(external); + ReadContextBuilder read_builder(external->Location()); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr read, - TableRead::Create(std::move(read_context))); + external->NewRead(std::move(read_context))); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, read->CreateReader(plan->Splits())); int64_t rows = 0; while (true) { @@ -2936,8 +2934,9 @@ TEST(FormatTableTest, TestATableServesAFormatTableThroughItsOwnEntryPoints) { TEST(FormatTableTest, TestTheTableEntryPointsRefuseWhatAFormatTableCannotHonour) { // A refusal is a promise about what was not done, so the table's own entry points must make - // the same promises the generic ones do. Each context is handed to both, and the answer has - // to come out the same whichever way in the caller takes. + // the same promises the generic ones do: a generic entry point refuses the format table the + // schema under the path names outright, while the table's operators name the setting they + // cannot honour, so a caller who reaches the table finds out exactly what is missing. std::unique_ptr dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); ASSERT_OK_AND_ASSIGN(std::shared_ptr format_table, @@ -2962,8 +2961,7 @@ TEST(FormatTableTest, TestTheTableEntryPointsRefuseWhatAFormatTableCannotHonour) return builder.Finish(); }; ASSERT_OK_AND_ASSIGN(std::unique_ptr streaming_context, streaming_scan_context()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(streaming_context)), - "nothing for a streaming scan to follow"); + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(streaming_context)), "names a format table"); ASSERT_OK_AND_ASSIGN(std::unique_ptr streaming_context_again, streaming_scan_context()); ASSERT_NOK_WITH_MSG(table->NewScan(std::move(streaming_context_again)), @@ -2979,8 +2977,7 @@ TEST(FormatTableTest, TestTheTableEntryPointsRefuseWhatAFormatTableCannotHonour) return builder.Finish(); }; ASSERT_OK_AND_ASSIGN(std::unique_ptr vector_context, vector_read_context()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(vector_context)), - "not supported for format tables"); + ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(vector_context)), "names a format table"); ASSERT_OK_AND_ASSIGN(std::unique_ptr vector_context_again, vector_read_context()); ASSERT_NOK_WITH_MSG(table->NewRead(std::move(vector_context_again)), "not supported for format tables"); @@ -2994,20 +2991,18 @@ TEST(FormatTableTest, TestTheTableEntryPointsRefuseWhatAFormatTableCannotHonour) return builder.WithCatalog(catalog, Identifier("db", "tbl")).Finish(); }; ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog_write, catalog_write_context()); - ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(catalog_write)), - "use WriteContextBuilder(FormatTable)"); + ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(catalog_write)), "names a format table"); ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog_write_again, catalog_write_context()); ASSERT_NOK_WITH_MSG(table->NewWrite(std::move(catalog_write_again)), - "use WriteContextBuilder(FormatTable)"); + "WithCatalog() requires a native table"); auto catalog_commit_context = [&dir, &catalog]() -> Result> { CommitContextBuilder builder(dir->Str(), "test-user"); return builder.WithCatalog(catalog, Identifier("db", "fmt")).Finish(); }; ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog_commit, catalog_commit_context()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(catalog_commit)), - "keeps no snapshot for a catalog to take"); + ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(catalog_commit)), "names a format table"); ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog_commit_again, catalog_commit_context()); ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(catalog_commit_again)), @@ -3020,75 +3015,50 @@ TEST(FormatTableTest, TestTheTableEntryPointsRefuseWhatAFormatTableCannotHonour) return builder.Finish(); }; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_id, write_id_context()); - ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_id)), - "a write id would name nothing"); + ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_id)), "names a format table"); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_id_again, write_id_context()); ASSERT_NOK_WITH_MSG(table->NewWrite(std::move(write_id_again)), "a write id would name nothing"); } -TEST(FormatTableTest, TestAContextBuiltFromAFormatTableRefusesASecondAnswer) { - // The table already carries its schema and the file system it was loaded through, from a - // source the context cannot see behind, so a second answer is refused rather than dropped. +TEST(FormatTableTest, TestTheGenericEntryPointsRefuseAFormatTablePath) { + // A load from a path is the data table's way in, so every generic entry point refuses the + // format table the schema under the path names rather than dispatching to it - and says where + // the table does come from, so a caller holding only the path is pointed at the catalog. std::unique_ptr dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); ASSERT_OK_AND_ASSIGN(std::shared_ptr table, CreateTable(dir->GetFileSystem(), dir->Str(), {"dt"})); - SchemaManager schema_manager(dir->GetFileSystem(), dir->Str()); - ASSERT_OK_AND_ASSIGN(std::optional> latest, - schema_manager.Latest()); - ASSERT_TRUE(latest.has_value()); - ASSERT_OK_AND_ASSIGN(std::string schema_json, (*latest)->GetJsonSchema()); - - ScanContextBuilder schema_builder(table); - schema_builder.SetTableSchema(schema_json); - ASSERT_NOK_WITH_MSG(schema_builder.Finish(), "carries its own schema"); - - ScanContextBuilder fs_builder(table); - fs_builder.WithFileSystem(dir->GetFileSystem()); - ASSERT_NOK_WITH_MSG(fs_builder.Finish(), "carries the file system"); - ReadContextBuilder branch_builder(table); - branch_builder.WithBranch("other"); - ASSERT_NOK_WITH_MSG(branch_builder.Finish(), "has no branches"); + ScanContextBuilder scan_builder(dir->Str()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), "names a format table"); - WriteContextBuilder write_fs_builder(table); - write_fs_builder.WithFileSystem(dir->GetFileSystem()); - ASSERT_NOK_WITH_MSG(write_fs_builder.Finish(), "carries the file system"); + ReadContextBuilder read_builder(dir->Str()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); + ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), "names a format table"); - auto catalog = std::make_shared(); - WriteContextBuilder write_catalog_builder(table); - ASSERT_NOK_WITH_MSG( - write_catalog_builder.WithCatalog(catalog, Identifier("db", "tbl")).Finish(), - "WithCatalog() requires a native table"); - ReadContextBuilder read_catalog_builder(table); - ASSERT_NOK_WITH_MSG(read_catalog_builder.WithCatalog(catalog, Identifier("db", "tbl")).Finish(), - "WithCatalog() cannot be used with one"); - ScanContextBuilder scan_catalog_builder(table); - ASSERT_NOK_WITH_MSG(scan_catalog_builder.WithCatalog(catalog, Identifier("db", "tbl")).Finish(), - "WithCatalog() cannot be used with one"); - catalog->SetTableSchema(latest.value()); - WriteContextBuilder path_catalog_builder(dir->Str(), "test-user"); - ASSERT_OK_AND_ASSIGN( - auto path_context, - path_catalog_builder.WithCatalog(catalog, Identifier("db", "tbl")).Finish()); - ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(path_context)), - "use WriteContextBuilder(FormatTable)"); + WriteContextBuilder write_builder(dir->Str(), "test-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); + ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_context)), "names a format table"); - CommitContextBuilder commit_fs_builder(table); - commit_fs_builder.WithFileSystem(dir->GetFileSystem()); - ASSERT_NOK_WITH_MSG(commit_fs_builder.Finish(), "carries the file system"); + CommitContextBuilder commit_builder(dir->Str(), "test-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); + ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(commit_context)), "names a format table"); - // A null table is the caller's mistake, and says so rather than complaining about a path it - // was never given. - ScanContextBuilder null_builder{std::shared_ptr()}; - ASSERT_NOK_WITH_MSG(null_builder.Finish(), "null format table"); + // The refusal points at the table's own operators, and those serve the same path-built + // contexts: this is the caller the message sends to `GetTable` and `NewXxx`. + ScanContextBuilder served_builder(dir->Str()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr served_context, served_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr served_scan, + table->NewScan(std::move(served_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, served_scan->CreatePlan()); + ASSERT_EQ(plan->Splits().size(), 0u); } -TEST(FormatTableTest, TestABranchNamedThroughOptionsIsRefusedAtEveryEntryPoint) { - // `WithBranch()` is refused on a context built from a format table, and the `branch` option - // says the same thing by another name: a format table keeps no metadata to branch, and a read - // or a write would go to its one location whatever the option named. Refusing it is the only +TEST(FormatTableTest, TestABranchNamedThroughOptionsIsRefusedByTheTableOperators) { + // The `branch` option names a branch a format table keeps no metadata for, and a read or a + // write would go to its one location whatever the option named. Refusing it is the only // answer that is not silently the wrong one - an overwrite would delete the main branch's // data while claiming to replace another branch's. std::unique_ptr dir = UniqueTestDirectory::Create(); @@ -3105,27 +3075,27 @@ TEST(FormatTableTest, TestABranchNamedThroughOptionsIsRefusedAtEveryEntryPoint) FormatTable::Create(dir->GetFileSystem(), dir->Str(), Identifier("db", "tbl"), branch), "has no branches"); - ScanContextBuilder scan_builder(table); + ScanContextBuilder scan_builder(dir->Str()); scan_builder.SetOptions(branch); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), "has no branches"); + ASSERT_NOK_WITH_MSG(table->NewScan(std::move(scan_context)), "has no branches"); - ReadContextBuilder read_builder(table); + ReadContextBuilder read_builder(dir->Str()); read_builder.SetOptions(branch); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), "has no branches"); + ASSERT_NOK_WITH_MSG(table->NewRead(std::move(read_context)), "has no branches"); - WriteContextBuilder write_builder(table); + WriteContextBuilder write_builder(dir->Str(), "test-user"); write_builder.SetOptions(branch); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_context)), "has no branches"); + ASSERT_NOK_WITH_MSG(table->NewWrite(std::move(write_context)), "has no branches"); // The commit is the one that matters most: an overwrite clears what it replaces before it // publishes anything, so the refusal has to come before the deleting does. - CommitContextBuilder commit_builder(table); + CommitContextBuilder commit_builder(dir->Str(), "test-user"); commit_builder.SetOptions(branch); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(commit_context)), "has no branches"); + ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(commit_context)), "has no branches"); ASSERT_OK_AND_ASSIGN( std::unique_ptr scan, @@ -3165,9 +3135,10 @@ TEST(FormatTableTest, TestATableCopiesItselfWithTheOptionsGivenAtTheCall) { TEST(FormatTableTest, TestAFormatTableSchemaOnABranchIsRefusedRatherThanServedFromTheTableRoot) { // A context that names a path takes its branch from `WithBranch()`, and the schema of that - // branch is what the entry point dispatches on. A format table's data is the files under the - // table path whichever branch named it, so serving one from there under another branch's name - // would answer a question about `dev` with the rows of `main`. + // branch decides what the entry point refuses. A format table's data is the files under the + // table path whichever branch named it, so the format table the branch's schema names is + // refused rather than served from there under another branch's name - the branch's table + // comes from the catalog, and its rows from that table's own operators. std::unique_ptr dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); ASSERT_OK_AND_ASSIGN(std::shared_ptr table, @@ -3186,29 +3157,30 @@ TEST(FormatTableTest, TestAFormatTableSchemaOnABranchIsRefusedRatherThanServedFr ReadContextBuilder read_builder(dir->Str()); read_builder.WithBranch("dev"); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), "has no branches"); + ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), "names a format table"); // A scan takes its branch from the option rather than from a `WithBranch()` of its own, which // is the same branch by another name and gets the same answer. ScanContextBuilder scan_builder(dir->Str()); scan_builder.SetOptions({{Options::BRANCH, "dev"}}); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), "has no branches"); + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), "names a format table"); - // The main branch is untouched by any of it. + // The main branch is untouched by any of it: the table the catalog hands back for it serves + // the scan from its own operators. ScanContextBuilder main_builder(dir->Str()); ASSERT_OK_AND_ASSIGN(std::unique_ptr main_context, main_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr main_scan, - TableScan::Create(std::move(main_context))); + table->NewScan(std::move(main_context))); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, main_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(std::vector rows, ReadAll(table, plan->Splits())); ASSERT_EQ(rows, (std::vector{"1|alice|20240101"})); } -TEST(FormatTableTest, TestTheGenericEntryPointsCarryTheContextThrough) { +TEST(FormatTableTest, TestTheTableOperatorsCarryTheContextThrough) { // A context promises that options given at the call win over the ones the schema stored, and - // that a branch and a caller-held schema are honoured. Dispatching to a format table must not - // quietly drop any of that. + // that a branch is honoured. Serving the operators off the table itself must not quietly drop + // any of that. std::unique_ptr dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); ASSERT_OK_AND_ASSIGN(std::shared_ptr table, @@ -3220,7 +3192,7 @@ TEST(FormatTableTest, TestTheGenericEntryPointsCarryTheContextThrough) { write_builder.SetOptions({{Options::TARGET_FILE_ROW_NUM, "1"}}); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr write, - FileStoreWrite::Create(std::move(write_context))); + table->NewWrite(std::move(write_context))); for (int32_t i = 0; i < 3; i++) { ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, MakeBatch({i}, {"name"}, "20240101", {})); @@ -3230,41 +3202,23 @@ TEST(FormatTableTest, TestTheGenericEntryPointsCarryTheContextThrough) { write->PrepareCommit()); ASSERT_EQ(messages.size(), 3u) << "target-file-row-num given at the call was ignored"; - // Published through the generic commit, or nothing below would see the rows: until the commit - // renames them the files sit in the hidden `_temporary` directory a scan skips. + // Published through the table's own commit, or nothing below would see the rows: until the + // commit renames them the files sit in the hidden `_temporary` directory a scan skips. CommitContextBuilder generic_commit_builder(dir->Str(), "test-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr generic_commit_context, generic_commit_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr generic_commit, - FileStoreCommit::Create(std::move(generic_commit_context))); + table->NewCommit(std::move(generic_commit_context))); ASSERT_OK(generic_commit->Commit(messages)); ASSERT_OK(write->Close()); - // A branch this table has no schema on is a managed table as far as dispatch is concerned, so - // the format branch must not answer for it. + // A branch this table has no schema on names no table at all as far as the generic entry + // points are concerned, so the format branch must not answer for it. ReadContextBuilder branch_builder(dir->Str()); branch_builder.WithBranch("nosuchbranch"); ASSERT_OK_AND_ASSIGN(std::unique_ptr branch_context, branch_builder.Finish()); ASSERT_NOK(TableRead::Create(std::move(branch_context))); - // A caller-held schema is used instead of reading one from under the path. - SchemaManager schema_manager(dir->GetFileSystem(), dir->Str()); - ASSERT_OK_AND_ASSIGN(std::optional> latest, - schema_manager.Latest()); - ASSERT_TRUE(latest.has_value()); - ASSERT_OK_AND_ASSIGN(std::string schema_json, (*latest)->GetJsonSchema()); - ScanContextBuilder seeded_builder(dir->Str()); - seeded_builder.SetTableSchema(schema_json); - ASSERT_OK_AND_ASSIGN(std::unique_ptr seeded_context, seeded_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr seeded_scan, - TableScan::Create(std::move(seeded_context))); - // Planning, not just dispatch: handing the schema over must not turn the `schema` directory - // the table keeps under its own path into data, which a plan would then read as a data file. - ASSERT_OK_AND_ASSIGN(std::shared_ptr seeded_plan, seeded_scan->CreatePlan()); - ASSERT_OK_AND_ASSIGN(std::vector seeded_rows, - ReadAll(table, seeded_plan->Splits())); - ASSERT_EQ(seeded_rows.size(), 3u); - // `type` is structural, so an option given at the call must not decide what kind of table // this is. Checked by reading the rows back rather than by counting splits: three small files // are packed into one split, so a split count says nothing about how many there are. @@ -3272,7 +3226,7 @@ TEST(FormatTableTest, TestTheGenericEntryPointsCarryTheContextThrough) { retyped_builder.SetOptions({{Options::TYPE, "table"}}); ASSERT_OK_AND_ASSIGN(std::unique_ptr retyped_context, retyped_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr retyped_scan, - TableScan::Create(std::move(retyped_context))); + table->NewScan(std::move(retyped_context))); ASSERT_OK_AND_ASSIGN(std::shared_ptr retyped_plan, retyped_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(std::vector retyped_rows, ReadAll(table, retyped_plan->Splits())); @@ -3307,7 +3261,7 @@ class CountingCache : public Cache { } // namespace -TEST(FormatTableTest, TestTheGenericReadCarriesPrefetchAndTheCacheThrough) { +TEST(FormatTableTest, TestTheReadCarriesPrefetchAndTheCacheThrough) { // A format table opens its files through the same component the managed table path opens its // own with, so what a `ReadContext` asks for about opening a file applies here too. std::unique_ptr dir = UniqueTestDirectory::Create(); @@ -3321,13 +3275,11 @@ TEST(FormatTableTest, TestTheGenericReadCarriesPrefetchAndTheCacheThrough) { ScanContextBuilder scan_builder(dir->Str()); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, - TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); - auto read_rows = [&plan](std::unique_ptr context) -> Result { - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, - TableRead::Create(std::move(context))); + auto read_rows = [&plan, &table](std::unique_ptr context) -> Result { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, table->NewRead(std::move(context))); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, read->CreateReader(plan->Splits())); int64_t rows = 0; @@ -3362,7 +3314,7 @@ TEST(FormatTableTest, TestTheGenericReadCarriesPrefetchAndTheCacheThrough) { ASSERT_GT(cache->gets, 0) << "the cache the read context carries never reached the file reader"; } -TEST(FormatTableTest, TestTheGenericEntryPointsRefuseWhatTheyCannotHonour) { +TEST(FormatTableTest, TestTheOperatorsRefuseWhatAFormatTableCannotHonour) { // Anything a context carries that a format table cannot do is a refusal naming the setting, // never a read or a write that quietly did something else. std::unique_ptr dir = UniqueTestDirectory::Create(); @@ -3377,14 +3329,14 @@ TEST(FormatTableTest, TestTheGenericEntryPointsRefuseWhatTheyCannotHonour) { read_schema_builder.SetReadSchema(std::move(projected_schema)); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_schema_context, read_schema_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_schema_context)), + ASSERT_NOK_WITH_MSG(table->NewRead(std::move(read_schema_context)), "does not take a projected read schema"); ScanContextBuilder streaming_builder(dir->Str()); streaming_builder.WithStreamingMode(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr streaming_context, streaming_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(streaming_context)), + ASSERT_NOK_WITH_MSG(table->NewScan(std::move(streaming_context)), "nothing for a streaming scan to follow"); ScanContextBuilder predicate_builder(dir->Str()); @@ -3392,13 +3344,12 @@ TEST(FormatTableTest, TestTheGenericEntryPointsRefuseWhatTheyCannotHonour) { /*field_index=*/0, /*field_name=*/"id", FieldType::INT, Literal(1))); ASSERT_OK_AND_ASSIGN(std::unique_ptr predicate_context, predicate_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(predicate_context)), - "does not take a predicate"); + ASSERT_NOK_WITH_MSG(table->NewScan(std::move(predicate_context)), "does not take a predicate"); CommitContextBuilder commit_builder(dir->Str(), "test-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_context))); + table->NewCommit(std::move(commit_context))); // A commit takes batch writes only. The rest of the snapshot half is // `TestTheGenericCommitRefusesEveryCallAboutSnapshots`. ASSERT_NOK_WITH_MSG(commit->Commit({}, /*commit_identifier=*/7), "commit identifier"); @@ -3419,12 +3370,12 @@ TEST(FormatTableTest, TestTheGenericCommitRefusesEveryCallAboutSnapshots) { // types then finds out at the call rather than from a table that did not change. std::unique_ptr dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); - ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::shared_ptr table, + ASSERT_OK_AND_ASSIGN(std::shared_ptr table, CreateTable(dir->GetFileSystem(), dir->Str(), {"dt"})); CommitContextBuilder commit_builder(dir->Str(), "test-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_context))); + table->NewCommit(std::move(commit_context))); ASSERT_NOK_WITH_MSG(commit->CommitWithProgress({}, BATCH_WRITE_COMMIT_IDENTIFIER, /*watermark=*/std::nullopt), @@ -3468,7 +3419,7 @@ TEST(FormatTableTest, TestTheGenericCommitOverwritesOnePartition) { WriteContextBuilder write_builder(dir->Str(), "test-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr write, - FileStoreWrite::Create(std::move(write_context))); + table->NewWrite(std::move(write_context))); ASSERT_OK_AND_ASSIGN(std::unique_ptr replacement, MakeBatch({3}, {"carol"}, "20240101", {{"dt", "20240101"}})); ASSERT_OK(write->Write(std::move(replacement))); @@ -3481,7 +3432,7 @@ TEST(FormatTableTest, TestTheGenericCommitOverwritesOnePartition) { CommitContextBuilder commit_builder(dir->Str(), "test-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_context))); + table->NewCommit(std::move(commit_context))); // Batch writes only, and refused before anything is cleared: an overwrite that failed on its // arguments must not have deleted the partition it named. ASSERT_NOK_WITH_MSG(commit->Overwrite({{"dt", "20240101"}}, messages, /*commit_identifier=*/7), diff --git a/src/paimon/core/table/source/table_scan_test.cpp b/src/paimon/core/table/source/table_scan_test.cpp index 435e2254f..17bf2e874 100644 --- a/src/paimon/core/table/source/table_scan_test.cpp +++ b/src/paimon/core/table/source/table_scan_test.cpp @@ -19,12 +19,15 @@ #include "paimon/table/source/table_scan.h" +#include #include #include #include #include #include "gtest/gtest.h" +#include "paimon/core/core_options.h" +#include "paimon/core/table/system/system_table.h" #include "paimon/defs.h" #include "paimon/fs/local/local_file_system.h" #include "paimon/metrics.h" @@ -164,12 +167,19 @@ TEST(TableScanTest, TestPkSchemaEvolutionScan) { TEST(TableScanTest, TestReadOptimizedPrimaryKeyStreamingScanUnsupported) { std::string path = paimon::test::GetDataDir() + "/orc/pk_table_with_alter_table.db/pk_table_with_alter_table$ro"; + // A system table is not served by the entry point a path names any more: the loader hands + // the table back the way a catalog's `GetTable` would, and the table refuses the scan. + std::map options = {{Options::FILE_FORMAT, "orc"}}; + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr table, + SystemTableLoader::LoadFromPath(core_options.GetFileSystem(), path, options)); ScanContextBuilder builder(path); builder.AddOption(Options::FILE_FORMAT, "orc"); builder.WithStreamingMode(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(context)), + ASSERT_NOK_WITH_MSG(table->NewScan(std::move(context)), "read-optimized system table does not support streaming scan for primary " "key table"); } diff --git a/src/paimon/core/table/system/system_table_test.cpp b/src/paimon/core/table/system/system_table_test.cpp index d6333475b..cfd4f5591 100644 --- a/src/paimon/core/table/system/system_table_test.cpp +++ b/src/paimon/core/table/system/system_table_test.cpp @@ -56,6 +56,7 @@ #include "paimon/scan_context.h" #include "paimon/status.h" #include "paimon/table/source/table_read.h" +#include "paimon/table/source/table_scan.h" #include "paimon/testing/mock/mock_file_batch_reader.h" #include "paimon/testing/utils/read_result_collector.h" #include "paimon/testing/utils/testharness.h" @@ -331,6 +332,21 @@ TEST(SystemTableTest, TestOperatorEntriesRefuseANullContext) { ASSERT_NOK_WITH_MSG(read_optimized.NewRead(nullptr), "read context is null pointer"); } +// The scan and read entry points serve a data table only: a `T$system` path is refused with a +// message pointing at the catalog that hands the table back, so a caller holding only the path is +// not told the table does not exist. +TEST(SystemTableTest, TestTheOperatorEntriesRefuseASystemTablePath) { + const std::string system_table_path = "/tmp/warehouse/db.db/tbl$options"; + + ScanContextBuilder scan_builder(system_table_path); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), "names a system table"); + + ReadContextBuilder read_builder(system_table_path); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); + ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), "names a system table"); +} + // The reads these views serve build a context of their own for the data table underneath, which // carries no search: a search the caller asked for is refused rather than quietly answered with // plain rows. diff --git a/src/paimon/core/table/table_test.cpp b/src/paimon/core/table/table_test.cpp index 8b7b2da45..2b6be91a4 100644 --- a/src/paimon/core/table/table_test.cpp +++ b/src/paimon/core/table/table_test.cpp @@ -36,6 +36,7 @@ #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/table_schema.h" #include "paimon/core/table/file_store_table.h" +#include "paimon/core/table/file_store_table_factory.h" #include "paimon/core/table/source/data_split_impl.h" #include "paimon/core/table/source/fallback_data_split.h" #include "paimon/defs.h" @@ -1289,22 +1290,18 @@ TEST(TableTest, TestALoadNamesTheDatabaseThePathLaysOut) { // A context that names no catalog knows the table by its directory: the directory above // the table's names the database when it carries the `.db` suffix, the way the Java - // `identifierFromPath` reads it, so the table the load serves is named for both. The - // refusal a format table open speaks names the table - and the option the context - // carries is what drives the open to it, since the schema alone opens cleanly. + // `identifierFromPath` reads it, so the table the load serves is named for both. const std::string table_dir = dir->Str() + "/my_db.db/my_table"; ASSERT_OK(fs->Mkdirs(table_dir)); SchemaManager schema_manager(fs, table_dir); - ASSERT_OK_AND_ASSIGN( - [[maybe_unused]] std::unique_ptr format_schema, - schema_manager.CreateTable(schema, /*partition_keys=*/{"dt"}, /*primary_keys=*/{}, - {{"type", "format-table"}})); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr managed_schema, + schema_manager.CreateTable(schema, /*partition_keys=*/{"dt"}, + /*primary_keys=*/{}, {{"file.format", "orc"}})); { ScanContextBuilder builder(table_dir); - builder.AddOption(Options::REALTIME_ENABLED, "true"); ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, builder.Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(ctx)), - "cannot open format table my_db.my_table"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, FileStoreTableFactory::Create(*ctx)); + ASSERT_EQ(table->FullName(), "my_db.my_table"); } // A table sitting directly under the root, with no `.db` directory above it, names no @@ -1315,13 +1312,12 @@ TEST(TableTest, TestALoadNamesTheDatabaseThePathLaysOut) { ASSERT_OK_AND_ASSIGN( [[maybe_unused]] std::unique_ptr loose_schema, loose_schema_manager.CreateTable(schema, /*partition_keys=*/{"dt"}, /*primary_keys=*/{}, - {{"type", "format-table"}})); + {{"file.format", "orc"}})); { ScanContextBuilder builder(loose_table_dir); - builder.AddOption(Options::REALTIME_ENABLED, "true"); ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, builder.Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(ctx)), - "cannot open format table loose_table"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, FileStoreTableFactory::Create(*ctx)); + ASSERT_EQ(table->FullName(), "loose_table"); } } diff --git a/test/inte/read_inte_test.cpp b/test/inte/read_inte_test.cpp index ef8e5c853..41e76b32d 100644 --- a/test/inte/read_inte_test.cpp +++ b/test/inte/read_inte_test.cpp @@ -49,6 +49,7 @@ #include "paimon/common/utils/path_util.h" #include "paimon/common/utils/read_ahead_cache.h" #include "paimon/common/utils/scope_guard.h" +#include "paimon/core/core_options.h" #include "paimon/core/io/data_file_meta.h" #include "paimon/core/manifest/file_source.h" #include "paimon/core/snapshot.h" @@ -57,6 +58,7 @@ #include "paimon/core/table/source/deletion_file.h" #include "paimon/core/table/source/fallback_data_split.h" #include "paimon/core/table/system/global_system_tables.h" +#include "paimon/core/table/system/system_table.h" #include "paimon/core/tag/tag.h" #include "paimon/core/utils/snapshot_manager.h" #include "paimon/data/decimal.h" @@ -274,6 +276,13 @@ Result> ReadSystemTable( const std::string& system_table_path, const std::map& options, bool streaming_mode = false, const std::shared_ptr& predicate = nullptr, const std::vector& read_field_names = {}, bool read_next_plan = false) { + // A system table is not served by the operator entry points a path names any more: the loader + // hands the table back, and the table serves the scan and the read itself, the way a catalog's + // `GetTable` would. + PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(options)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr table, + SystemTableLoader::LoadFromPath(core_options.GetFileSystem(), system_table_path, options)); ScanContextBuilder scan_context_builder(system_table_path); scan_context_builder.SetOptions(options).WithStreamingMode(streaming_mode); if (predicate) { @@ -282,7 +291,7 @@ Result> ReadSystemTable( PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + table->NewScan(std::move(scan_context))); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, table_scan->CreatePlan()); if (read_next_plan) { PAIMON_ASSIGN_OR_RAISE(plan, table_scan->CreatePlan()); @@ -299,7 +308,7 @@ Result> ReadSystemTable( PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + table->NewRead(std::move(read_context))); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr result, @@ -650,17 +659,20 @@ TEST(SystemTableReadInteTest, TestReadOptionsSystemTable) { ArrowSchemaRelease(&schema); std::string system_table_path = PathUtil::JoinPath(dir->Str(), "warehouse/db1.db/tbl1$options"); + // A system table comes from the catalog, and the operators from the table it hands back - + // the path alone no longer reaches it. + ASSERT_OK_AND_ASSIGN(auto table, catalog->GetTable(Identifier("db1", "tbl1$options"))); ScanContextBuilder scan_context_builder(system_table_path); scan_context_builder.SetOptions(options); ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, table->NewScan(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto plan, table_scan->CreatePlan()); ASSERT_EQ(plan->Splits().size(), 1); ReadContextBuilder read_context_builder(system_table_path); read_context_builder.SetOptions(options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, table->NewRead(std::move(read_context))); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(auto result, ReadResultCollector::CollectResult(std::move(batch_reader))); ASSERT_TRUE(result); @@ -682,17 +694,21 @@ TEST(SystemTableReadInteTest, TestReadBranchOptionsSystemTable) { std::map options = {{Options::FILE_SYSTEM, "local"}, {Options::FILE_FORMAT, "parquet"}}; std::string system_table_path = table_path + "$branch_rt$options"; + // No catalog here: the loader is the path-side half of what a catalog's `GetTable` does. + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options)); + ASSERT_OK_AND_ASSIGN(auto table, SystemTableLoader::LoadFromPath(core_options.GetFileSystem(), + system_table_path, options)); ScanContextBuilder scan_context_builder(system_table_path); scan_context_builder.SetOptions(options); ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, table->NewScan(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto plan, table_scan->CreatePlan()); ASSERT_EQ(plan->Splits().size(), 1); ReadContextBuilder read_context_builder(system_table_path); read_context_builder.SetOptions(options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, table->NewRead(std::move(read_context))); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(auto result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -1002,12 +1018,18 @@ TEST(SystemTableReadInteTest, TestReadOptimizedPrimaryKeyProjectionAndPredicateP std::shared_ptr predicate = PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"k", FieldType::INT, Literal(2)); + // A system table is not served by the operator entry points a path names any more: the loader + // hands the table back, and the table serves the scan and the read itself. + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr table, + SystemTableLoader::LoadFromPath(core_options.GetFileSystem(), table_path + "$ro", options)); ScanContextBuilder ro_scan_context_builder(table_path + "$ro"); ro_scan_context_builder.SetOptions(options).SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(std::unique_ptr ro_scan_context, ro_scan_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr ro_table_scan, - TableScan::Create(std::move(ro_scan_context))); + table->NewScan(std::move(ro_scan_context))); ASSERT_OK_AND_ASSIGN(std::shared_ptr ro_plan, ro_table_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(int64_t ro_file_count, CountDataFiles(ro_plan->Splits())); ASSERT_EQ(ro_file_count, 1); @@ -1018,7 +1040,7 @@ TEST(SystemTableReadInteTest, TestReadOptimizedPrimaryKeyProjectionAndPredicateP read_context_builder.SetOptions(options).SetPredicate(predicate).SetReadFieldNames({"k", "v"}); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + table->NewRead(std::move(read_context))); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(ro_plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, @@ -1062,11 +1084,17 @@ TEST(SystemTableReadInteTest, TestReadOptimizedSystemTableNestedProjection) { /*partition_map=*/{}, /*bucket=*/0, {})); ASSERT_OK(WriteAndFullCompact(std::move(batch), /*commit_identifier=*/0, helper.get())); + // A system table is not served by the operator entry points a path names any more: the loader + // hands the table back, and the table serves the scan and the read itself. + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr table, + SystemTableLoader::LoadFromPath(core_options.GetFileSystem(), table_path + "$ro", options)); ScanContextBuilder scan_context_builder(table_path + "$ro"); scan_context_builder.SetOptions(options); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + table->NewScan(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, table_scan->CreatePlan()); auto projected_schema = arrow::schema({ @@ -1079,7 +1107,7 @@ TEST(SystemTableReadInteTest, TestReadOptimizedSystemTableNestedProjection) { read_context_builder.SetOptions(options).SetReadSchema(std::move(c_projected_schema)); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + table->NewRead(std::move(read_context))); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp index 7111f5302..1fd0ea9ab 100644 --- a/test/inte/realtime_write_inte_test.cpp +++ b/test/inte/realtime_write_inte_test.cpp @@ -61,6 +61,7 @@ #include "paimon/core/table/sink/commit_message_impl.h" #include "paimon/core/table/source/data_split_impl.h" #include "paimon/core/table/source/realtime_split.h" +#include "paimon/core/table/system/system_table.h" #include "paimon/core/utils/snapshot_manager.h" #include "paimon/data/shredding/map_shared_shredding_schema_utils.h" #include "paimon/data/variant.h" @@ -2918,10 +2919,16 @@ TEST_F(RealtimeWriteInteTest, TestPkRealtimeReadOptimizedScanUnsupported) { ASSERT_OK_AND_ASSIGN(std::vector disk_rows, ReadRows()); ASSERT_EQ(rows, disk_rows); + // A system table is not served by the entry point a path names any more: the loader hands + // the table back, and the table refuses the scan itself. + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options_)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr table, + SystemTableLoader::LoadFromPath(core_options.GetFileSystem(), + table_path_ + "$ro", options_)); ScanContextBuilder scan_builder(table_path_ + "$ro"); scan_builder.SetOptions(options_).WithRealtimeContext(realtime_context).WithMemoryPool(pool_); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); - Result> scan = TableScan::Create(std::move(scan_context)); + Result> scan = table->NewScan(std::move(scan_context)); ASSERT_TRUE(scan.status().IsNotImplemented()) << scan.status().ToString(); ASSERT_NE(std::string::npos, scan.status().ToString().find( "PK real-time union read does not support read-optimized")); From 8da735ede295a1c7810418ead76417440e387a28 Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Wed, 28 Oct 2026 10:58:47 +0800 Subject: [PATCH 19/30] fix --- src/paimon/core/catalog/catalog.cpp | 3 - .../core/catalog/file_system_catalog.cpp | 28 +- src/paimon/core/catalog/file_system_catalog.h | 2 +- .../core/catalog/file_system_catalog_test.cpp | 76 +++- src/paimon/core/catalog/snapshot_commit.cpp | 2 - .../core/operation/internal_read_context.cpp | 14 - .../core/operation/internal_read_context.h | 7 +- src/paimon/core/schema/schema_validation.cpp | 1 - src/paimon/core/schema/table_schema.cpp | 4 - .../table/append_only_file_store_table.cpp | 3 - src/paimon/core/table/file_store.cpp | 21 +- src/paimon/core/table/file_store.h | 11 +- src/paimon/core/table/file_store_table.cpp | 130 ++++--- src/paimon/core/table/file_store_table.h | 15 + .../core/table/file_store_table_factory.cpp | 66 ---- src/paimon/core/table/format/format_table.cpp | 7 - .../core/table/format/format_table_test.cpp | 61 ---- .../table/primary_key_file_store_table.cpp | 7 - .../core/table/source/table_read_test.cpp | 10 - .../core/table/source/table_scan_test.cpp | 2 - .../table/system/audit_log_system_table.cpp | 5 - .../table/system/global_system_tables.cpp | 8 - .../table/system/in_memory_system_table.cpp | 2 - .../system/read_optimized_system_table.cpp | 5 - src/paimon/core/table/system/system_table.cpp | 6 - .../core/table/system/system_table_test.cpp | 18 - src/paimon/core/table/table_test.cpp | 335 +++++++++++------- src/paimon/rest/rest_catalog.cpp | 15 - src/paimon/rest/rest_catalog_test.cpp | 12 - test/inte/read_inte_test.cpp | 10 - test/inte/realtime_write_inte_test.cpp | 2 - 31 files changed, 394 insertions(+), 494 deletions(-) diff --git a/src/paimon/core/catalog/catalog.cpp b/src/paimon/core/catalog/catalog.cpp index b14a53eab..717606612 100644 --- a/src/paimon/core/catalog/catalog.cpp +++ b/src/paimon/core/catalog/catalog.cpp @@ -57,9 +57,6 @@ Result> Catalog::Create( std::unique_ptr rest_catalog, RestCatalog::Create(root_path, options, file_system, RestHttpClient::Config(), fs_scheme_to_identifier_map)); - // Built from the derived type so that the `enable_shared_from_this` base a - // `RestCatalog` carries is initialized: a `sys` table keeps its catalog alive through - // `weak_from_this()`, which stays empty when the pointer is widened first. return std::shared_ptr(std::move(rest_catalog)); #else return Status::NotImplemented( diff --git a/src/paimon/core/catalog/file_system_catalog.cpp b/src/paimon/core/catalog/file_system_catalog.cpp index d061acbf2..d5365d2ab 100644 --- a/src/paimon/core/catalog/file_system_catalog.cpp +++ b/src/paimon/core/catalog/file_system_catalog.cpp @@ -287,8 +287,6 @@ Result> FileSystemCatalog::LoadTableSchema( if (!supported) { return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); } - // The table built here is transient: only its schema is read and nothing outlives the - // call. It builds no rows, so the context carries no catalog. GlobalSystemTableContext context; context.fs = fs_; context.warehouse = warehouse_; @@ -316,9 +314,27 @@ Result> FileSystemCatalog::LoadTableSchema( } Result> FileSystemCatalog::GetTable(const Identifier& identifier) const { + if (CatalogUtils::IsSystemDatabase(identifier.GetDatabaseName())) { + PAIMON_ASSIGN_OR_RAISE(bool supported, GlobalSystemTableLoader::IsSupported( + identifier.GetObjectName(), catalog_options_)); + if (!supported) { + return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); + } + std::shared_ptr self = weak_from_this().lock(); + if (self == nullptr) { + return Status::Invalid( + "a global system table retains its catalog, so the catalog must be owned by " + "a std::shared_ptr before it is asked for one"); + } + GlobalSystemTableContext context; + context.catalog = std::move(self); + context.fs = fs_; + context.warehouse = warehouse_; + context.catalog_options = catalog_options_; + return GlobalSystemTableLoader::Load(identifier.GetObjectName(), context); + } PAIMON_ASSIGN_OR_RAISE(bool is_system_table, identifier.IsSystemTable()); if (is_system_table) { - // A system table reads the data table's files through this catalog's file system. return LoadSystemTable(identifier); } // The schema is read on the table's own branch: `root/branch/branch-` below the @@ -330,14 +346,10 @@ Result> FileSystemCatalog::GetTable(const Identifier& ide return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); } PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(identifier)); - // The schema's `type` says what the table is. A format table is the same directory read as - // plain data, so the schema read above is all it takes to serve one; no manifest is read. std::shared_ptr data_schema = checked_pointer_cast(*latest_schema); PAIMON_ASSIGN_OR_RAISE(bool is_format_table, TableTypeDefine::IsFormatTable(data_schema->Options())); if (is_format_table) { - // This catalog keeps a table's own metadata under the table path, so a `schema` or - // `branch` directory there is metadata rather than table content. return FormatTable::Create(fs_, table_path, identifier, data_schema, /*location_carries_paimon_metadata=*/true, /*dynamic_options=*/{}); @@ -347,7 +359,6 @@ Result> FileSystemCatalog::GetTable(const Identifier& ide // table rather than at the first read. PAIMON_RETURN_NOT_OK(CatalogUtils::CheckManagedTableType(identifier, *latest_schema, "FileSystemCatalog::GetTable")); - // The branch the schema above was read on, so the table's managers read the same one. PAIMON_ASSIGN_OR_RAISE(std::string branch, identifier.GetBranchNameOrDefault()); return FileStoreTable::Create(fs_, table_path, checked_pointer_cast(*latest_schema), identifier, /*uuid=*/"", branch, /*schema_manager=*/nullptr, @@ -375,7 +386,6 @@ Result> FileSystemCatalog::LoadSystemTable( if (branch) { dynamic_options[Options::BRANCH] = branch.value(); } - // A system table reads the files of the data table it belongs to, so its root path is that one. PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(data_identifier)); return SystemTableLoader::Load(system_table_name.value(), fs_, table_path, latest_schema.value(), dynamic_options); diff --git a/src/paimon/core/catalog/file_system_catalog.h b/src/paimon/core/catalog/file_system_catalog.h index 203e1b92e..0f3123238 100644 --- a/src/paimon/core/catalog/file_system_catalog.h +++ b/src/paimon/core/catalog/file_system_catalog.h @@ -39,7 +39,7 @@ class Identifier; class Logger; class SystemTable; -class FileSystemCatalog : public Catalog { +class FileSystemCatalog : public Catalog, public std::enable_shared_from_this { public: FileSystemCatalog(const std::shared_ptr& fs, const std::string& warehouse, const std::map& catalog_options); diff --git a/src/paimon/core/catalog/file_system_catalog_test.cpp b/src/paimon/core/catalog/file_system_catalog_test.cpp index 294b7e9b9..cc8fea8d8 100644 --- a/src/paimon/core/catalog/file_system_catalog_test.cpp +++ b/src/paimon/core/catalog/file_system_catalog_test.cpp @@ -230,9 +230,6 @@ TEST(FileSystemCatalogTest, TestOptionsSystemTableCatalog) { ASSERT_FALSE(loaded_schema->field(1)->nullable()); ASSERT_OK_AND_ASSIGN(auto system_table, catalog.GetTable(options_identifier)); - // A system table comes back as the `SystemTable` it is: it answers for itself with its own - // read-only schema and the root path of the data table it reads. Its name carries the data - // table it belongs to. ASSERT_TRUE(std::dynamic_pointer_cast(system_table) != nullptr); ASSERT_TRUE(std::dynamic_pointer_cast(system_table) == nullptr); ASSERT_EQ("tbl1$options", system_table->Name()); @@ -255,9 +252,6 @@ TEST(FileSystemCatalogTest, TestOptionsSystemTableCatalog) { "Cannot 'renameTable' for system table"); } -// A `T$system` identifier opens the read-only view it names, which serves scans and reads of -// the data table it belongs to through the same `Table` interface, and refuses the operations -// that would modify the data table. TEST(FileSystemCatalogTest, TestGetTableServesASystemTable) { std::map options; options[Options::FILE_SYSTEM] = "local"; @@ -276,7 +270,6 @@ TEST(FileSystemCatalogTest, TestGetTableServesASystemTable) { /*primary_keys=*/{}, options, /*ignore_if_exists=*/false)); ArrowSchemaRelease(&schema); - // A managed table with no `type` option still opens, as the managed table it is. ASSERT_OK_AND_ASSIGN(auto managed, catalog.GetTable(Identifier("db1", "tbl1"))); ASSERT_TRUE(std::dynamic_pointer_cast(managed) == nullptr); @@ -284,7 +277,6 @@ TEST(FileSystemCatalogTest, TestGetTableServesASystemTable) { ASSERT_TRUE(std::dynamic_pointer_cast(system_table) != nullptr); ASSERT_EQ("tbl1$options", system_table->Name()); - // The scan and the read work from a context of the caller's, like every other table. ScanContextBuilder scan_builder(system_table->GetRootPath()); scan_builder.SetOptions(options); ASSERT_OK_AND_ASSIGN(auto scan_context, scan_builder.Finish()); @@ -298,7 +290,6 @@ TEST(FileSystemCatalogTest, TestGetTableServesASystemTable) { ASSERT_OK_AND_ASSIGN(auto reader, read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(auto table_result, reader->NextBatch()); - // The options the table was created with come back, one row per key. auto& [c_array, c_schema] = table_result; std::shared_ptr batch; auto maybe_batch = arrow::ImportRecordBatch(c_array.get(), c_schema.get()); @@ -306,7 +297,6 @@ TEST(FileSystemCatalogTest, TestGetTableServesASystemTable) { batch = std::move(maybe_batch).ValueUnsafe(); ASSERT_GE(batch->num_rows(), 2); - // The operations that would modify the data table underneath are refused by name. WriteContextBuilder write_builder(system_table->GetRootPath(), "commit_user"); write_builder.SetOptions(options); ASSERT_OK_AND_ASSIGN(auto write_context, write_builder.Finish()); @@ -321,6 +311,70 @@ TEST(FileSystemCatalogTest, TestGetTableServesASystemTable) { "tbl1$options system table is read-only: committing for it is not supported"); } +TEST(FileSystemCatalogTest, TestGetTableServesTheSysDatabase) { + std::map options; + options[Options::FILE_SYSTEM] = "local"; + options[Options::FILE_FORMAT] = "orc"; + options[CatalogOptionsSystemTable::kEnabledOption] = "true"; + ASSERT_OK_AND_ASSIGN(auto core_options, CoreOptions::FromMap(options)); + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto catalog = + std::make_shared(core_options.GetFileSystem(), dir->Str(), options); + ASSERT_OK(catalog->CreateDatabase("db1", options, /*ignore_if_exists=*/false)); + + auto typed_schema = arrow::schema({arrow::field("f0", arrow::int32(), /*nullable=*/false)}); + ::ArrowSchema schema; + ASSERT_TRUE(arrow::ExportSchema(*typed_schema, &schema).ok()); + ASSERT_OK(catalog->CreateTable(Identifier("db1", "tbl1"), &schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*ignore_if_exists=*/false)); + ArrowSchemaRelease(&schema); + + { + std::map disabled = options; + disabled.erase(CatalogOptionsSystemTable::kEnabledOption); + auto disabled_catalog = + std::make_shared(core_options.GetFileSystem(), dir->Str(), disabled); + ASSERT_NOK_WITH_MSG(disabled_catalog->GetTable(Identifier("sys", "catalog_options")), + "not exist"); + } + ASSERT_OK_AND_ASSIGN(auto catalog_options, + catalog->GetTable(Identifier("sys", "catalog_options"))); + ASSERT_TRUE(std::dynamic_pointer_cast(catalog_options) != nullptr); + + ASSERT_NOK_WITH_MSG(catalog->GetTable(Identifier("sys", "nope")), "not exist"); + + ASSERT_OK_AND_ASSIGN(auto tables, catalog->GetTable(Identifier("sys", "tables"))); + ScanContextBuilder scan_builder("sys/tables"); + scan_builder.SetOptions(options); + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan, tables->NewScan(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto plan, scan->CreatePlan()); + ASSERT_EQ(plan->Splits().size(), 1); + + ReadContextBuilder read_builder("sys/tables"); + read_builder.SetOptions(options); + ASSERT_OK_AND_ASSIGN(auto read_context, read_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto read, tables->NewRead(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto reader, read->CreateReader(plan->Splits())); + ASSERT_OK_AND_ASSIGN(auto table_result, reader->NextBatch()); + + auto& [c_array, c_schema] = table_result; + std::shared_ptr batch; + auto maybe_batch = arrow::ImportRecordBatch(c_array.get(), c_schema.get()); + ASSERT_TRUE(maybe_batch.ok()) << maybe_batch.status().ToString(); + batch = std::move(maybe_batch).ValueUnsafe(); + ASSERT_GE(batch->num_rows(), 1); + + std::weak_ptr weak = catalog; + catalog.reset(); + ASSERT_FALSE(weak.expired()) << "the global system table does not retain its catalog"; + + FileSystemCatalog stack_catalog(core_options.GetFileSystem(), dir->Str(), options); + ASSERT_NOK_WITH_MSG(stack_catalog.GetTable(Identifier("sys", "tables")), + "must be owned by a std::shared_ptr"); +} + TEST(FileSystemCatalogTest, TestBranchTableLoadsBranchSchema) { std::map options; options[Options::FILE_SYSTEM] = "local"; @@ -729,8 +783,6 @@ TEST(FileSystemCatalogTest, TestGetTableServesAFormatTable) { /*ignore_if_exists=*/false)); ArrowSchemaRelease(&schema); - // The table's `type` says what it is, so the one call that opens a table hands a format - // table back as the `FormatTable` it is, through the `Table` interface the two share. ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog.GetTable(Identifier("db1", "fmt"))); std::shared_ptr format_table = std::dynamic_pointer_cast(table); ASSERT_NE(nullptr, format_table); diff --git a/src/paimon/core/catalog/snapshot_commit.cpp b/src/paimon/core/catalog/snapshot_commit.cpp index 35efe767c..91e70a851 100644 --- a/src/paimon/core/catalog/snapshot_commit.cpp +++ b/src/paimon/core/catalog/snapshot_commit.cpp @@ -38,14 +38,12 @@ Result> SnapshotCommit::Create( if (!catalog_identifier) { return Status::Invalid("a catalog commit requires a table identifier"); } - // Reuse the caller's UUID so table recreation cannot redirect this commit. return std::shared_ptr( std::make_shared(catalog, catalog_identifier.value(), table_id)); } if (use_rest_catalog_commit) { return std::shared_ptr(std::make_shared(table_id)); } - // Catalogs without version management may still supply a table ID. if (table_id && catalog == nullptr) { return Status::Invalid( "a commit which publishes the snapshot by writing it to the table directory sends no " diff --git a/src/paimon/core/operation/internal_read_context.cpp b/src/paimon/core/operation/internal_read_context.cpp index 3ee0eb755..e17cfbd4e 100644 --- a/src/paimon/core/operation/internal_read_context.cpp +++ b/src/paimon/core/operation/internal_read_context.cpp @@ -295,10 +295,6 @@ Result> InternalReadContext::CreateFromOpti const std::shared_ptr& context, const std::shared_ptr& table_schema, const std::map& options, const std::shared_ptr& file_system, std::optional path) { - // The file system the caller named, or null for the read context's own, which callers - // without a table hand over. The scheme map the context carries does not take part: a file - // system handed over wins and one missing falls to the resolving one, the same way a scan - // resolves its own. PAIMON_ASSIGN_OR_RAISE( CoreOptions core_options, CoreOptions::FromMap( @@ -328,21 +324,11 @@ Result> InternalReadContext::CreateForFallb } assert(table_schema); auto options = table_schema->Options(); - // The dynamic options the load merged into this table - and the ones its copies accumulated - - // speak for the read, and the fallback branch serves the same read: they are merged over the - // fallback's own schema options the way the load merged them over the table's, so the fallback - // reader recognises what the main one does. `type` stays the branch's own: neither a copy nor - // a fallback restyles the table. for (const auto& [key, value] : table_dynamic_options) { if (key != Options::TYPE) { options[key] = value; } } - for (const auto& [key, value] : context->GetOptions()) { - if (key != Options::TYPE) { - options[key] = value; - } - } if (branch != BranchManager::DEFAULT_MAIN_BRANCH) { options[Options::BRANCH] = branch; } else { diff --git a/src/paimon/core/operation/internal_read_context.h b/src/paimon/core/operation/internal_read_context.h index 9e5d91ce0..a710ca15e 100644 --- a/src/paimon/core/operation/internal_read_context.h +++ b/src/paimon/core/operation/internal_read_context.h @@ -79,8 +79,11 @@ class InternalReadContext { /// The read context of a fallback branch, whose schema is not the table's own: it is read /// through a manager of that branch, and the read serves the options that schema was loaded /// with, the branch it was read from among them. A branch the context carries is not honoured - /// here. The C++ reading of the Java `FileStoreTableFactory#createOtherBranchTable`, which - /// loads the other branch's table the same way for the fallback reader to serve. + /// here, and neither are options the context adds on its own: the fallback reader serves the + /// table the way the main reader does, and a load merges the context's options into the + /// dynamic options handed in here. The C++ reading of the Java + /// `FileStoreTableFactory#createOtherBranchTable`, which loads the other branch's table the + /// same way for the fallback reader to serve. /// @param file_system The file system the table was opened with, in place of one resolved from /// the context. /// @param root_path The root path the table was opened with, in place of the one the context diff --git a/src/paimon/core/schema/schema_validation.cpp b/src/paimon/core/schema/schema_validation.cpp index c799c24f8..9fcb5f370 100644 --- a/src/paimon/core/schema/schema_validation.cpp +++ b/src/paimon/core/schema/schema_validation.cpp @@ -443,7 +443,6 @@ Status SchemaValidation::CheckImmutableOptions( return Status::Invalid(fmt::format("Change '{}' is not supported yet.", key)); } if (key == Options::IGNORE_DELETE) { - // An unset option is one that never ignored deletes, the same default they parse to. PAIMON_ASSIGN_OR_RAISE(bool old_ignore_delete, OptionsUtils::GetValueFromMap(schema_options, key, false)); PAIMON_ASSIGN_OR_RAISE(bool new_ignore_delete, diff --git a/src/paimon/core/schema/table_schema.cpp b/src/paimon/core/schema/table_schema.cpp index e6151d4c7..e8a3b7b65 100644 --- a/src/paimon/core/schema/table_schema.cpp +++ b/src/paimon/core/schema/table_schema.cpp @@ -203,16 +203,12 @@ Result> TableSchema::MergeOptions( auto merged = std::unique_ptr( new TableSchema(version_, id_, fields_, highest_field_id_, partition_keys_, primary_keys_, options, comment_, time_millis_)); - // The derived members follow the merged options: a copy the options restyle must serve the - // bucket keys and the bucket count it carries, not the ones the base schema carried. PAIMON_RETURN_NOT_OK(merged->DeriveBucketInfo()); return merged; } Status TableSchema::DeriveBucketInfo() { PAIMON_ASSIGN_OR_RAISE(std::vector keys, TrimmedPrimaryKeys()); - - // Try to validate bucket keys PAIMON_ASSIGN_OR_RAISE(std::vector bucket_keys, OriginalBucketKeys()); if (bucket_keys.empty()) { bucket_keys = keys; diff --git a/src/paimon/core/table/append_only_file_store_table.cpp b/src/paimon/core/table/append_only_file_store_table.cpp index fb504ee4c..713c9160b 100644 --- a/src/paimon/core/table/append_only_file_store_table.cpp +++ b/src/paimon/core/table/append_only_file_store_table.cpp @@ -58,7 +58,6 @@ Result> AppendOnlyFileStoreTable::CreateWrite( const std::shared_ptr& path_factory, const std::shared_ptr& snapshot_manager, const std::shared_ptr& io_manager, bool ignore_previous_files) const { - // append table const std::shared_ptr& schema = store_->GetSchema(); const bool realtime_deduplicate = !options.GetRealtimeDeduplicateKeyFields().empty(); bool need_dv_maintainer_factory = options.DeletionVectorsEnabled(); @@ -177,8 +176,6 @@ CommitScanner::ScanSupplier AppendOnlyFileStoreTable::NewCommitScanSupplier( const std::shared_ptr& manifest_file, const std::shared_ptr& arrow_schema, const CoreOptions& options, const std::shared_ptr& executor, const std::shared_ptr& pool) const { - // The schema manager the table carries, which the scans this supplier hands out read the - // schema through. const std::shared_ptr& schema_manager = GetSchemaManager(); const std::shared_ptr& table_schema = store_->GetSchema(); return [snapshot_manager, schema_manager, manifest_list, manifest_file, table_schema, diff --git a/src/paimon/core/table/file_store.cpp b/src/paimon/core/table/file_store.cpp index fe8b092c7..649a7f980 100644 --- a/src/paimon/core/table/file_store.cpp +++ b/src/paimon/core/table/file_store.cpp @@ -49,10 +49,6 @@ namespace paimon { namespace { -/// Whether an option key takes part in file-system resolution: the backend `file-system` picks, -/// an `fs.` access key carries, or a native S3 key names - `s3.` and its `s3a.` spelling, the -/// prefixes the S3 file system consumes alongside `fs.s3.`/`fs.s3a.`. The scheme-map keys a -/// context carries are the same set. bool IsFileSystemResolutionKey(const std::string& key) { return key == Options::FILE_SYSTEM || key.rfind("fs.", 0) == 0 || key.rfind("s3.", 0) == 0 || key.rfind("s3a.", 0) == 0; @@ -69,10 +65,6 @@ Result> FileStore::Create( auto store = std::shared_ptr(new FileStore( file_system, root_path, schema, branch, schema_manager, snapshot_manager, fs_scheme_map)); if (schema != nullptr) { - // The options the schema was loaded with, on the branch this store serves: what reads - // through them - whose cache keys are scoped by the branch - reads the same one the - // managers point at. The scheme map is the one the schema was read through, so a scheme - // it mapped resolves here the way it resolved there. std::map options = schema->Options(); if (BranchManager::IsMainBranch(store->branch_)) { options.erase(Options::BRANCH); @@ -83,10 +75,6 @@ Result> FileStore::Create( CoreOptions::FromMap(options, file_system, fs_scheme_map)); store->core_options_ = std::move(core_options); } - // The file system the table is served through: the one the options resolve - a schema - // carrying access configuration names its own, which the file system the caller opened the - // table through wins over - and without a schema, the one handed over, whose absence leaves - // the default local one. if (store->core_options_.has_value()) { store->file_system_ = store->core_options_->GetFileSystem(); } else if (file_system != nullptr) { @@ -95,10 +83,6 @@ Result> FileStore::Create( PAIMON_ASSIGN_OR_RAISE(CoreOptions default_options, CoreOptions::FromMap({})); store->file_system_ = default_options.GetFileSystem(); } - // The managers read through the same file system: the metadata belongs to the data, and - // both were written under the options the schema carries. A manager handed over already - // points at `root_path` on `branch`; a missing one is built here, so the operators always - // read through the managers the store carries. if (store->schema_manager_ == nullptr) { store->schema_manager_ = std::make_shared(store->file_system_, root_path, store->branch_); @@ -161,14 +145,15 @@ FileStore::FileStore(const std::shared_ptr& file_system, const std:: snapshot_manager_(snapshot_manager) {} Result> FileStore::NewPathFactory( - const std::shared_ptr& arrow_schema, const CoreOptions& options, + const std::shared_ptr& arrow_schema, + const std::vector& partition_keys, const CoreOptions& options, const std::string& file_format_identifier, const std::shared_ptr& memory_pool) const { PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, options.CreateExternalPaths()); PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, options.CreateGlobalIndexExternalPath()); return FileStorePathFactory::Create( - root_path_, arrow_schema, schema_->PartitionKeys(), options.GetPartitionDefaultName(), + root_path_, arrow_schema, partition_keys, options.GetPartitionDefaultName(), file_format_identifier, options.DataFilePrefix(), options.LegacyPartitionNameEnabled(), external_paths, global_index_external_path, options.IndexFileInDataFileDir(), memory_pool); } diff --git a/src/paimon/core/table/file_store.h b/src/paimon/core/table/file_store.h index 339dd3810..1dbdee7ef 100644 --- a/src/paimon/core/table/file_store.h +++ b/src/paimon/core/table/file_store.h @@ -23,6 +23,7 @@ #include #include #include +#include #include "paimon/catalog/identifier.h" #include "paimon/core/core_options.h" @@ -141,13 +142,15 @@ class FileStore { } /// Builds the path factory an operator names its metadata paths through, over this store's - /// root path and the partition keys of its schema. No schema change can alter the partition - /// keys, so an operator serving a schema read after the table was loaded still names the same - /// partitions. The caller names the file format identifier: a write names the format of its + /// root path. The caller names the partition keys of the schema the arrow schema was converted + /// from: an operator serving the table's own schema names this store's partition keys, while + /// a read of a fallback branch names the keys of that branch's schema, which is not this + /// store's. The caller names the file format identifier: a write names the format of its /// level-0 files, which the options may set per level, while a scan, a read and a commit name /// the table's format. Result> NewPathFactory( - const std::shared_ptr& arrow_schema, const CoreOptions& options, + const std::shared_ptr& arrow_schema, + const std::vector& partition_keys, const CoreOptions& options, const std::string& file_format_identifier, const std::shared_ptr& memory_pool) const; diff --git a/src/paimon/core/table/file_store_table.cpp b/src/paimon/core/table/file_store_table.cpp index 8d48d71c6..e93c8ae6d 100644 --- a/src/paimon/core/table/file_store_table.cpp +++ b/src/paimon/core/table/file_store_table.cpp @@ -33,6 +33,7 @@ #include "paimon/common/types/data_field.h" #include "paimon/common/utils/binary_row_partition_computer.h" #include "paimon/common/utils/options_utils.h" +#include "paimon/common/utils/path_util.h" #include "paimon/common/utils/sensitive_config_utils.h" #include "paimon/common/utils/string_utils.h" #include "paimon/core/catalog/catalog_utils.h" @@ -109,10 +110,6 @@ Result> FileStoreTable::Create( const std::shared_ptr& snapshot_manager, const std::map& fs_scheme_map, const std::map& dynamic_options) { - // The kind the schema's primary keys name, which no schema change can add or drop, so the kind - // the table is built as is the kind every schema its root path ever held describes. A caller - // without a schema gets the append-only kind, whose operators report that none was found; a - // schema that is not a table schema - a system table's read-only one - is refused, not cast. std::shared_ptr table_schema; if (schema != nullptr) { table_schema = std::dynamic_pointer_cast(schema); @@ -121,7 +118,6 @@ Result> FileStoreTable::Create( fmt::format("{} is not a managed table schema", identifier.ToString())); } } - // The store parses the schema's options, so one that does not parse is refused here. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr store, FileStore::Create(file_system, root_path, table_schema, branch, schema_manager, snapshot_manager, fs_scheme_map)); @@ -141,29 +137,14 @@ Result> FileStoreTable::Copy( if (loaded_schema == nullptr) { return Status::Invalid(fmt::format("not found latest schema of {}", FullName())); } - // `type` is structural and read from the schema alone, the promise `Copy` makes: an option - // given at the call must not restyle the table. `branch` is the same promise - the copy serves - // the branch this table does, and a caller that means to move has `SwitchToBranch`. Both are - // dropped before the merge, so the copy's options never speak of a branch it does not serve. std::map overridable = dynamic_options; overridable.erase(Options::TYPE); overridable.erase(Options::BRANCH); - // The physical layout the data on the path was written by is as immovable as the style or the - // branch: the copy serves the same data. PAIMON_RETURN_NOT_OK(SchemaValidation::CheckImmutableOptions(*loaded_schema, overridable)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merged_schema, loaded_schema->MergeOptions(overridable)); - // The managers are handed over, not rebuilt, when the copy resolves the same file system this - // table does: its operators then read the schema cache this table already warmed. The copy - // resolves from the file system the caller opened this table through, not the one it resolved - // to, so a file-system option it was given speaks; when one does change the resolution, fresh - // managers are built over it instead. The scheme map travels the same way. const bool fs_resolution_may_change = FileStore::FileSystemResolutionMayChange(*loaded_schema, overridable); - // The dynamic options the copy was given join the ones this table was loaded with - which a - // copy of a copy carries the same way - so a read's fallback reader serves what the copy's own - // reader does however many copies deep. The branch was dropped above, so none accumulates, and - // an option given again replaces the one carried, as the merge above does. std::map accumulated = dynamic_options_; for (const auto& [key, value] : overridable) { accumulated.insert_or_assign(key, value); @@ -179,31 +160,23 @@ Result> FileStoreTable::Copy( Result> FileStoreTable::SwitchToBranch(const std::string& branch) const { const std::string target_branch = BranchManager::NormalizeBranch(branch); if (target_branch == CurrentBranch()) { - // The table already serves this branch, so a copy of it is the same table to the caller. return FileStoreTable::Create(store_->GetSpecifiedFileSystem(), root_path_, schema_, identifier_, CatalogUuid().value_or(""), CurrentBranch(), GetSchemaManager(), GetSnapshotManager(), store_->GetFsSchemeMap(), dynamic_options_); } - // The schema of the branch switched to, read from its own directory. auto schema_manager = std::make_shared(file_system_, root_path_, target_branch); PAIMON_ASSIGN_OR_RAISE(std::optional> latest, schema_manager->Latest()); if (!latest) { return Status::Invalid(fmt::format("branch {} does not exist", target_branch)); } - // A branch other than main is named in the options the schema carries - the way a read context - // names it - so what reads through them serves the branch the managers point at. std::map overridable; if (target_branch != BranchManager::DEFAULT_MAIN_BRANCH) { overridable[Options::BRANCH] = target_branch; } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr branch_schema, (*latest)->MergeOptions(overridable)); - // The branch's schema may name another file system than this table's options resolved - a - // recreation that changed the access configuration - and a manager kept through that - // disagreement would read the branch's metadata through the backend this table left, so the - // branch builds its own. The schema is in hand already, so only the cache is lost. const std::shared_ptr branch_schema_manager = (store_->GetSpecifiedFileSystem() == nullptr && (!store_->GetFsSchemeMap().empty() || @@ -211,8 +184,6 @@ Result> FileStoreTable::SwitchToBranch(const std::string& branch_schema->Options(), std::static_pointer_cast(schema_)->Options()))) ? nullptr : schema_manager; - // The identifier names the branch switched to, as Java's `newCatalogEnvironment` rebuilds it, - // so `Name()` and `FullName()` answer for the table served rather than the one left behind. PAIMON_ASSIGN_OR_RAISE(Identifier branch_identifier, CatalogUtils::BranchIdentifier(identifier_, target_branch)); return FileStoreTable::Create(store_->GetSpecifiedFileSystem(), root_path_, @@ -246,7 +217,6 @@ Status FileStoreTable::CheckOptionsAgreeWithTable( for (const auto& [key, value] : context_options) { auto table_option = table_options.find(key); if (table_option != table_options.end() && table_option->second != value) { - // A value may be a credential and the error travels into logs, so it is redacted. return Status::Invalid( fmt::format("option '{}' is '{}' in the context but '{}' on the table", key, SensitiveConfigUtils::RedactValue(key, value), @@ -256,6 +226,59 @@ Status FileStoreTable::CheckOptionsAgreeWithTable( return Status::OK(); } +Status FileStoreTable::CheckContextNamesThisTable( + const std::string& kind, const std::string& context_root_path, + const std::optional& context_identifier, + const std::optional& context_table_id) const { + if (!root_path_.empty()) { + PAIMON_ASSIGN_OR_RAISE(std::string context_path, + PathUtil::NormalizePath(context_root_path)); + PAIMON_ASSIGN_OR_RAISE(std::string table_path, PathUtil::NormalizePath(root_path_)); + if (context_path != table_path) { + return Status::Invalid(fmt::format( + "the {} context names root path '{}' but this table serves '{}': the store below " + "reads and writes this table's files, so a context naming another path would " + "leave this table's metadata there and publish it as the other's", + kind, context_root_path, root_path_)); + } + } + if (context_identifier) { + PAIMON_ASSIGN_OR_RAISE(std::optional context_branch, + context_identifier.value().GetBranchName()); + if (context_branch && context_branch.value() != CurrentBranch()) { + return Status::Invalid(fmt::format( + "the {} context names branch '{}' of '{}' but this table serves branch '{}', " + "which is the one a {} of this table writes and publishes", + kind, context_branch.value(), context_identifier.value().ToString(), + CurrentBranch(), kind)); + } + PAIMON_ASSIGN_OR_RAISE( + Identifier context_table, + CatalogUtils::BranchIdentifier(context_identifier.value(), CurrentBranch())); + PAIMON_ASSIGN_OR_RAISE(Identifier served_table, + CatalogUtils::BranchIdentifier(identifier_, CurrentBranch())); + const bool names_this_table = + (served_table.GetDatabaseName() == Identifier::kUnknownDatabase || + context_table.GetDatabaseName() == served_table.GetDatabaseName()) && + context_table.GetObjectName() == served_table.GetObjectName(); + if (!names_this_table) { + return Status::Invalid(fmt::format( + "the {} context names table '{}' but this table is '{}': the store below serves " + "this table's files, so a context naming another table would write this table's " + "metadata and publish it as the other's", + kind, context_identifier.value().ToString(), identifier_.ToString())); + } + } + if (context_table_id && context_table_id != CatalogUuid()) { + return Status::Invalid(fmt::format( + "the {} context carries table id '{}' but this table's is '{}': the id tells a " + "recreated table from the one the changes were prepared for, so one this table does " + "not carry would publish this table's metadata as the recreation's", + kind, context_table_id.value(), CatalogUuid().value_or("none"))); + } + return Status::OK(); +} + std::unique_ptr FileStoreTable::WithTablePath(std::unique_ptr&& scan, const std::string& root_path) { scan->WithTablePath(root_path); @@ -265,14 +288,18 @@ std::unique_ptr FileStoreTable::WithTablePath(std::unique_ptr> FileStoreTable::NewWrite( std::unique_ptr ctx) const { PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "write")); + PAIMON_RETURN_NOT_OK(CheckContextNamesThisTable("write", ctx->GetRootPath(), + ctx->GetIdentifier(), + /*context_table_id=*/std::nullopt)); + if (AsVersionManaged(ctx->GetCatalog()) != nullptr) { + PAIMON_RETURN_NOT_OK(BranchManager::CheckCatalogAddressableBranch(CurrentBranch())); + } PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog())); std::shared_ptr specific_fs = ctx->GetSpecificFileSystem(); - // A catalog write names its table; the file system comes from the table's own credentials. if (specific_fs == nullptr && ctx->GetFileSystemSchemeToIdentifierMap().empty() && ctx->GetCatalog() != nullptr && !ctx->GetIdentifier()) { return Status::Invalid("a catalog write requires a table identifier"); } - // The write goes to the branch the table serves, not the context's own. PAIMON_ASSIGN_OR_RAISE(std::optional catalog_identifier, CatalogUtils::BranchIdentifier(ctx->GetIdentifier(), CurrentBranch())); const std::shared_ptr& schema = store_->GetSchema(); @@ -294,7 +321,7 @@ Result> FileStoreTable::NewWrite( PAIMON_ASSIGN_OR_RAISE( std::shared_ptr file_store_path_factory, - store_->NewPathFactory(arrow_schema, options, + store_->NewPathFactory(arrow_schema, schema->PartitionKeys(), options, options.GetWriteFileFormat(/*level=*/0)->Identifier(), ctx->GetMemoryPool())); PAIMON_ASSIGN_OR_RAISE( @@ -330,16 +357,13 @@ Result> FileStoreTable::NewScan( PAIMON_ASSIGN_OR_RAISE(bool read_optimized, OptionsUtils::GetValueFromMap(context->GetOptions(), kReadOptimizedScanOption, false)); - // The branch in the store's options is the table's, so one the context carries is not honoured. CoreOptions core_options = store_->GetOptions(); - // The cache is a runtime setting of this scan, not of the table. core_options.WithCache(context->GetCache()); PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*table_schema, core_options)); PAIMON_RETURN_NOT_OK( RealtimeTableScan::ValidateScan(*table_schema, core_options, *context, read_optimized)); - // validate options if (core_options.GetBucket() == -1) { if (!table_schema->PrimaryKeys().empty()) { return Status::NotImplemented(fmt::format( @@ -351,17 +375,17 @@ Result> FileStoreTable::NewScan( fmt::format("do not support bucket={} in scan process", core_options.GetBucket())); } - // validate schema and scan filter auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); if (context->GetScanFilters() && context->GetScanFilters()->GetPredicate()) { PAIMON_RETURN_NOT_OK(PredicateValidator::ValidatePredicateWithSchema( *arrow_schema, context->GetScanFilters()->GetPredicate(), /*validate_field_idx=*/false)); } - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr path_factory, - store_->NewPathFactory(arrow_schema, core_options, - core_options.GetFileFormat()->Identifier(), - context->GetMemoryPool())); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr path_factory, + store_->NewPathFactory(arrow_schema, table_schema->PartitionKeys(), core_options, + core_options.GetFileFormat()->Identifier(), + context->GetMemoryPool())); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr manifest_list, @@ -440,11 +464,8 @@ Result> FileStoreTable::CreateTableRead( auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr path_factory, - store_->NewPathFactory(arrow_schema, core_options, + store_->NewPathFactory(arrow_schema, table_schema->PartitionKeys(), core_options, core_options.GetFileFormat()->Identifier(), memory_pool)); - // The reader is selected by the schema the read serves, not by the table's kind: an - // append-only table can have a primary-key fallback branch and vice versa, so a reader typed - // after the main table would fail the fallback's splits. if (internal_context->GetPrimaryKeys().empty()) { return std::make_unique(path_factory, internal_context, memory_pool, executor); @@ -461,7 +482,6 @@ Result> FileStoreTable::NewRead( return Status::Invalid("not found latest schema"); } PAIMON_RETURN_NOT_OK(CheckOptionsAgreeWithTable(*loaded_schema, read_context->GetOptions())); - // The read reads through the branch the table serves, not the context's own. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr internal_context, InternalReadContext::CreateForTable(read_context, loaded_schema, store_->GetOptions(), root_path_)); @@ -477,7 +497,6 @@ Result> FileStoreTable::NewRead( return std::move(table_read); } - // The fallback branch is not the table's, so its schema is read through its own manager. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr fallback_context, InternalReadContext::CreateForFallbackBranch( read_context, /*branch=*/scan_fallback_branch.value(), file_system_, @@ -544,17 +563,18 @@ Result> NewSnapshotCommit( Result> FileStoreTable::NewCommit( std::unique_ptr ctx) const { PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "commit")); + PAIMON_RETURN_NOT_OK(CheckContextNamesThisTable("commit", ctx->GetRootPath(), + ctx->GetIdentifier(), ctx->GetTableId())); + if (ctx->GetCatalog() != nullptr || ctx->UseRESTCatalogCommit()) { + PAIMON_RETURN_NOT_OK(BranchManager::CheckCatalogAddressableBranch(CurrentBranch())); + } PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog())); std::shared_ptr specific_fs = ctx->GetSpecificFileSystem(); - // A catalog commit names its table; the file system comes from the table's own credentials. if (specific_fs == nullptr && ctx->GetCatalog() != nullptr && !ctx->GetIdentifier()) { return Status::Invalid("a catalog commit requires a table identifier"); } - // A commit commits to the root path and branch the table was loaded with, not the context's. PAIMON_ASSIGN_OR_RAISE(std::optional catalog_identifier, CatalogUtils::BranchIdentifier(ctx->GetIdentifier(), CurrentBranch())); - // The id a snapshot records the catalog's current schema by, read through the loader below - // rather than trusted from a schema this commit will not re-read. FileStoreCommitImpl::SchemaIdLoader schema_id_loader; if (ctx->GetCatalog() != nullptr) { if (!catalog_identifier) { @@ -588,8 +608,6 @@ Result> FileStoreTable::NewCommit( assert(options.GetFileFormat()); PAIMON_RETURN_NOT_OK(FileStoreCommitImpl::ValidateCommitOptions(options)); PAIMON_ASSIGN_OR_RAISE(bool is_object_store, FileSystem::IsObjectStore(root_path_)); - // Catalog commits do not require atomic rename on the object store. The inte-test marker is - // not a table option, so it is read from the context the commit was handed. if (is_object_store && !ctx->UseRESTCatalogCommit() && AsVersionManaged(ctx->GetCatalog()) == nullptr && ctx->GetOptions().find("enable-object-store-commit-in-inte-test") == @@ -602,8 +620,8 @@ Result> FileStoreTable::NewCommit( store_->NewPartitionComputer(arrow_schema, options, ctx->GetMemoryPool())); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr path_factory, - store_->NewPathFactory(arrow_schema, options, options.GetFileFormat()->Identifier(), - ctx->GetMemoryPool())); + store_->NewPathFactory(arrow_schema, schema->PartitionKeys(), options, + options.GetFileFormat()->Identifier(), ctx->GetMemoryPool())); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr snapshot_manager, diff --git a/src/paimon/core/table/file_store_table.h b/src/paimon/core/table/file_store_table.h index 04b088ab8..66e40ad59 100644 --- a/src/paimon/core/table/file_store_table.h +++ b/src/paimon/core/table/file_store_table.h @@ -134,6 +134,21 @@ class FileStoreTable : public Table { static Status CheckOptionsAgreeWithTable( const TableSchema& table_schema, const std::map& context_options); + /// Checks the identity a context carries - the path it names, the identifier it gives a + /// catalog, the table id it records - against this table's own. The store below serves this + /// table's files, so a context naming another table would write this table's metadata and + /// publish it as the other's; a field that contradicts the table is refused by name, the way + /// `CheckOptionsAgreeWithTable` refuses a contradicting option. A field the context does not + /// carry is no contradiction: a write or a commit from a bare path names no catalog object. + /// An identifier that names a branch this table does not serve names a branch of another + /// table, as the operator serves the branch the table was loaded with alone. + /// @param kind The operator the context was handed to, which the error names. + /// @param context_table_id The id `CommitContext` records for a catalog commit; a write + /// context carries none, so `NewWrite` passes null. + Status CheckContextNamesThisTable(const std::string& kind, const std::string& context_root_path, + const std::optional& context_identifier, + const std::optional& context_table_id) const; + /// Builds the read `NewRead` hands back, from the internal context it read the schema into. The /// path factory comes from the store. The reader is picked by the schema the read serves - the /// primary keys it carries - not by the table's own kind. diff --git a/src/paimon/core/table/file_store_table_factory.cpp b/src/paimon/core/table/file_store_table_factory.cpp index 5eca88fcf..c0c5e7152 100644 --- a/src/paimon/core/table/file_store_table_factory.cpp +++ b/src/paimon/core/table/file_store_table_factory.cpp @@ -51,10 +51,6 @@ namespace paimon { namespace { -/// The options a context carries, merged into the table the load hands back: a table is the -/// product of the load, so the options speak there - the way `Copy` merges them - and `NewXxx` on -/// the product serves them without merging again. A caller holding a table already changes it -/// through `Copy` itself, and an option that contradicts the table that way is refused. Result> WithContextOptions( const std::shared_ptr
& table, const std::shared_ptr& schema, const std::map& options) { @@ -64,7 +60,6 @@ Result> WithContextOptions( const std::map& schema_options = schema->Options(); for (const auto& [key, value] : options) { auto schema_option = schema_options.find(key); - // A key the schema does not carry, or carries differently: the merge has something to do. if (schema_option == schema_options.end() || schema_option->second != value) { return table->Copy(options); } @@ -72,17 +67,6 @@ Result> WithContextOptions( return table; } -/// The tail every `Create` ends with: the schema one of them read says what the path names, and -/// only a data table is served by it - a format table is refused, the table a catalog hands back -/// serves one through its own `NewXxx`, and everything else is handed back as the `FileStoreTable` -/// a managed table is, carrying the schema for its `NewXxx` to build from. A null schema is handed -/// over as it is, so the managed path below decides what to say about a table that is not there. -/// -/// @param file_system The file system the schema above was read through. -/// @param explicit_file_system The file system the context supplied, which a managed table is -/// opened through when there is one. A null one leaves the table to resolve from its -/// schema's own access configuration rather than being pinned to the file system this load -/// happened to read the schema through. Result> TableOfLoadedSchema( const std::shared_ptr& file_system, const std::shared_ptr& explicit_file_system, const std::string& path, @@ -91,9 +75,6 @@ Result> TableOfLoadedSchema( const std::shared_ptr& latest_schema, const std::optional& identifier, const std::string& uuid, const std::map& fs_scheme_map) { if (latest_schema != nullptr) { - // The table type is read from the schema, never from `options`: `type` is structural, and - // one call's options must not decide what kind of table this is. See - // `FormatTable::Create()`. PAIMON_ASSIGN_OR_RAISE(bool is_format_table, TableTypeDefine::IsFormatTable(latest_schema->Options())); if (is_format_table) { @@ -104,15 +85,8 @@ Result> TableOfLoadedSchema( path)); } } - // The branch the schema was read on is folded into the object name. Identifier table_identifier = identifier.value_or(CatalogUtils::IdentifierFromPath(path, branch)); - // The manager was built over the file system the context's own options resolve, while the - // store resolves the schema's: a schema carrying access configuration names a backend the - // context's options never did, and a manager kept through that disagreement would read the - // table's metadata through the wrong one. The schema is in hand already, so dropping the - // manager costs its cache alone. A context that supplied a file system wins on both sides and - // keeps it; a scheme map only the store's resolution takes part in is the same disagreement. std::shared_ptr manager = schema_manager; if (manager != nullptr && latest_schema != nullptr && explicit_file_system == nullptr && (!fs_scheme_map.empty() || @@ -127,15 +101,6 @@ Result> TableOfLoadedSchema( return WithContextOptions(table, latest_schema, options); } -/// The managed table a context names behind a path: the schema is read through a manager of the -/// context's branch - handed over instead when the caller already read it - and what the schema -/// says is there decides the table `TableOfLoadedSchema` hands back. -/// -/// @param schema_json The schema of the table, when the caller already read it, or null when it -/// has to be read here. It is honoured on the main branch only; a branch publishes its own -/// under `branch/branch-`, which is where the read below goes. -/// @param explicit_file_system The file system the context supplied, which the managed table the -/// schema describes is opened through - see `TableOfLoadedSchema`. Result> ManagedTableOf( const std::shared_ptr& file_system, const std::shared_ptr& explicit_file_system, const std::string& path, @@ -145,8 +110,6 @@ Result> ManagedTableOf( auto schema_manager = std::make_shared(file_system, path, branch); std::shared_ptr latest_schema; if (branch == BranchManager::DEFAULT_MAIN_BRANCH && schema_json) { - // Handing the schema over only saves reading it. A context always names a table path and - // this library writes the metadata under it, so `schema` and `branch` stay metadata. PAIMON_ASSIGN_OR_RAISE(latest_schema, TableSchema::CreateFromJson(schema_json.value())); } else { PAIMON_ASSIGN_OR_RAISE(std::optional> latest, @@ -159,10 +122,6 @@ Result> ManagedTableOf( schema_manager, latest_schema, identifier, uuid, fs_scheme_map); } -/// The schema a catalog may hold for the table, read from it on the main branch only and null -/// otherwise: a branch publishes its own under `branch/branch-`, which is where -/// `ManagedTableOf` reads instead, so a catalog answering for no branch schema does not fail the -/// `kind` operation to one. `kind` also names the operator the messages speak of. Result> CatalogMainBranchSchemaJson( const std::shared_ptr& catalog, const std::optional& catalog_identifier, const std::string& branch, const char* kind) { @@ -183,9 +142,6 @@ Result> CatalogMainBranchSchemaJson( } // namespace -/// The data table a scan context names: only a data table is served by a load from the context's -/// path - a system table behind a `T$system` path and a format table are refused, served by the -/// table a catalog hands back through its own `NewXxx` instead. Result> FileStoreTableFactory::Create(const ScanContext& ctx) { PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, CoreOptions::FromMap(ctx.GetOptions(), ctx.GetSpecificFileSystem(), {})); @@ -205,9 +161,6 @@ Result> FileStoreTableFactory::Create(const ScanContext& /*uuid=*/"", /*fs_scheme_map=*/{}); } -/// The data table a read context names: only a data table is served by a load from the context's -/// path - a system table behind a `T$system` path and a format table are refused, served by the -/// table a catalog hands back through its own `NewXxx` instead. Result> FileStoreTableFactory::Create(const ReadContext& ctx) { PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_core_options, CoreOptions::FromMap(ctx.GetOptions(), ctx.GetSpecificFileSystem(), @@ -223,17 +176,12 @@ Result> FileStoreTableFactory::Create(const ReadContext& } const std::string branch = ctx.GetBranch(); - // The scheme map the schema was read through, so the table's data resolves custom schemes the - // same way its metadata does. return ManagedTableOf(tmp_core_options.GetFileSystem(), ctx.GetSpecificFileSystem(), ctx.GetPath(), branch, ctx.GetOptions(), ctx.GetSpecificTableSchema(), ctx.GetIdentifier(), /*uuid=*/"", ctx.GetFileSystemSchemeToIdentifierMap()); } -/// The data table a write context names: only a data table is served by a load from the context's -/// root path - a format table is refused, served by the table a catalog hands back through its own -/// `NewXxx` instead. Result> FileStoreTableFactory::Create(const WriteContext& ctx) { PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx.GetCatalog())); std::shared_ptr specific_fs = ctx.GetSpecificFileSystem(); @@ -242,8 +190,6 @@ Result> FileStoreTableFactory::Create(const WriteContext& if (!ctx.GetIdentifier()) { return Status::Invalid("a catalog write requires a table identifier"); } - // A catalog issuing per-table temporary credentials only hands them out through - // GetTableFileSystem, so the write uses this table's. PAIMON_ASSIGN_OR_RAISE(specific_fs, ctx.GetCatalog()->GetTableFileSystem(ctx.GetIdentifier().value())); } @@ -253,8 +199,6 @@ Result> FileStoreTableFactory::Create(const WriteContext& std::string branch = ctx.GetBranch(); PAIMON_ASSIGN_OR_RAISE(std::optional catalog_identifier, CatalogUtils::BranchIdentifier(ctx.GetIdentifier(), branch)); - // Only the main branch may have a current schema the catalog alone holds: a branch publishes - // its own under `branch/branch-`, which is where the load below reads it instead. PAIMON_ASSIGN_OR_RAISE( std::optional catalog_table_schema, CatalogMainBranchSchemaJson(ctx.GetCatalog(), catalog_identifier, branch, "write")); @@ -263,13 +207,8 @@ Result> FileStoreTableFactory::Create(const WriteContext& ctx.GetFileSystemSchemeToIdentifierMap()); } -/// The data table a commit context names: only a data table is served by a load from the context's -/// root path - a format table is refused, served by the table a catalog hands back through its own -/// `NewXxx` instead. Result> FileStoreTableFactory::Create(const CommitContext& ctx) { PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx.GetCatalog())); - // Use this table's catalog credentials unless the caller supplied a file system: a catalog - // issuing per-table temporary ones only hands them out through GetTableFileSystem. std::shared_ptr specific_fs = ctx.GetSpecificFileSystem(); if (specific_fs == nullptr && ctx.GetCatalog() != nullptr) { if (!ctx.GetIdentifier()) { @@ -281,19 +220,14 @@ Result> FileStoreTableFactory::Create(const CommitContext PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, CoreOptions::FromMap(ctx.GetOptions(), specific_fs)); const std::string& root_path = ctx.GetRootPath(); - // The branch the context was built for, already validated by `CommitContextBuilder`. PAIMON_ASSIGN_OR_RAISE(std::string branch, BranchManager::ResolveBranch( ctx.GetIdentifier(), ctx.GetOptions(), /*explicit_branch=*/std::nullopt, "commit")); PAIMON_ASSIGN_OR_RAISE(std::optional catalog_identifier, CatalogUtils::BranchIdentifier(ctx.GetIdentifier(), branch)); - // Only the main branch may have a current schema the catalog alone holds: a branch publishes - // its own under `branch/branch-`, which is where the load below reads it instead. PAIMON_ASSIGN_OR_RAISE( std::optional catalog_table_schema, CatalogMainBranchSchemaJson(ctx.GetCatalog(), catalog_identifier, branch, "commit")); - // The managed path below reads the commit's branch, so this reads the same one: the two must - // not dispatch on different schemas. return ManagedTableOf(tmp_options.GetFileSystem(), specific_fs, root_path, branch, ctx.GetOptions(), catalog_table_schema, ctx.GetIdentifier(), ctx.GetTableId().value_or(""), /*fs_scheme_map=*/{}); diff --git a/src/paimon/core/table/format/format_table.cpp b/src/paimon/core/table/format/format_table.cpp index acb5f03e1..846842cb7 100644 --- a/src/paimon/core/table/format/format_table.cpp +++ b/src/paimon/core/table/format/format_table.cpp @@ -224,17 +224,12 @@ Result> FormatTable::GetArrowSchema() const { return schema_->GetArrowSchema(); } -// The refusals below use two codes. `Invalid` is a setting that clashes with what a format table -// is, and will never work: no snapshots, no buckets, no real-time store, no manifests, no -// catalog-side version. `NotImplemented` is a capability this code lacks today and could gain. Result> FormatTable::NewScan( std::unique_ptr context) const { std::shared_ptr shared_context = std::move(context); PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(shared_context, "scan")); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, CopySelf(shared_context->GetOptions())); - // Anything a `ScanContext` carries that a format table cannot honour is refused rather than - // silently dropped. if (shared_context->IsStreamingMode()) { return Status::Invalid( "a format table has no snapshots, so there is nothing for a streaming scan to follow"); @@ -313,8 +308,6 @@ Result> FormatTable::NewWrite( "a format table write takes the table's own columns; a write schema naming a " "subset of them is not supported yet"); } - // A write id prefixes a postpone-bucket writer's files so that one compaction reader can - // put them back in order. A format table has no buckets, so it would identify nothing. if (context->GetWriteId().has_value()) { return Status::Invalid("a format table has no buckets, so a write id would name nothing"); } diff --git a/src/paimon/core/table/format/format_table_test.cpp b/src/paimon/core/table/format/format_table_test.cpp index af0686c6f..c7456d005 100644 --- a/src/paimon/core/table/format/format_table_test.cpp +++ b/src/paimon/core/table/format/format_table_test.cpp @@ -764,10 +764,6 @@ TEST(FormatTableTest, TestCatalogManagedPartitionsAreRejected) { } TEST(FormatTableTest, TestTheTableOperatorsValidateOptionsGivenAtTheCall) { - // An option a format table refuses has to be refused wherever it comes from. Every operator - // merges what the call gave it over what the schema stored, so each runs the same checks over - // the merged result; otherwise a caller could set through a context what the schema would have - // rejected and have it silently dropped. std::unique_ptr dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); ASSERT_OK_AND_ASSIGN(std::shared_ptr table, @@ -2666,10 +2662,6 @@ TEST(FormatTableTest, TestScanRejectsUnknownPartitionField) { } TEST(FormatTableTest, TestAFormatTableIsWrittenCommittedAndReadBack) { - // A caller holding the table a catalog hands back drives it through the operators the table - // itself serves, with contexts built from its location the way they are for every other - // table - the C++ side of what Java serves through one `ReadBuilder` and one - // `BatchWriteBuilder` for both kinds of table. std::unique_ptr dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); ASSERT_OK_AND_ASSIGN(std::shared_ptr table, @@ -2764,10 +2756,6 @@ TEST(FormatTableTest, TestAFormatTableIsWrittenCommittedAndReadBack) { } TEST(FormatTableTest, TestACatalogInTheContextIsRefusedAlikeByWriteAndCommit) { - // `WithCatalog()` asks for a catalog-side version of the commit, which a format table has no - // way to produce. Both halves of the write path answer with the same code, so a caller reads - // one kind of failure out of them rather than "cannot be done" from one and "not yet" from - // the other. std::unique_ptr dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); ASSERT_OK_AND_ASSIGN(std::shared_ptr table, @@ -2793,11 +2781,6 @@ TEST(FormatTableTest, TestACatalogInTheContextIsRefusedAlikeByWriteAndCommit) { } TEST(FormatTableTest, TestALoadedFormatTableIsServedThroughItsOwnOperators) { - // A table a catalog keeps the schema for, the way a rest catalog does. Nothing under its - // location says that it is a format table, nor that everything below the location is data, so - // a path is not enough to reach it: the loaded table serves the operators itself, with a - // context built from its location, which is what `FormatTable.newReadBuilder()` and - // `newBatchWriteBuilder()` do in Java. std::unique_ptr schema_dir = UniqueTestDirectory::Create(); ASSERT_TRUE(schema_dir); SchemaManager schema_manager(schema_dir->GetFileSystem(), schema_dir->Str()); @@ -2873,11 +2856,6 @@ TEST(FormatTableTest, TestALoadedFormatTableIsServedThroughItsOwnOperators) { } TEST(FormatTableTest, TestATableServesAFormatTableThroughItsOwnEntryPoints) { - // `NewScan()` and its kin take the table they were called on as the one to serve, the way Java - // Paimon's `Table` serves both kinds through one `newReadBuilder()`, so a caller holding a - // format table as the `Table` a catalog hands out reaches it through them as well. The - // contexts name a path only and carry no format table, so it is the table itself that - // answers. std::unique_ptr dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); std::shared_ptr
table; @@ -2918,8 +2896,6 @@ TEST(FormatTableTest, TestATableServesAFormatTableThroughItsOwnEntryPoints) { } ASSERT_OK_AND_ASSIGN(std::shared_ptr record_batch, ImportBatch(read_batch)); - // `_VALUE_KIND` first, then the table's own columns, exactly as the generic entry points - // give them. auto ids = checked_pointer_cast(record_batch->column(1)); auto names = checked_pointer_cast(record_batch->column(2)); auto dts = checked_pointer_cast(record_batch->column(3)); @@ -2933,18 +2909,12 @@ TEST(FormatTableTest, TestATableServesAFormatTableThroughItsOwnEntryPoints) { } TEST(FormatTableTest, TestTheTableEntryPointsRefuseWhatAFormatTableCannotHonour) { - // A refusal is a promise about what was not done, so the table's own entry points must make - // the same promises the generic ones do: a generic entry point refuses the format table the - // schema under the path names outright, while the table's operators name the setting they - // cannot honour, so a caller who reaches the table finds out exactly what is missing. std::unique_ptr dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); ASSERT_OK_AND_ASSIGN(std::shared_ptr format_table, CreateTable(dir->GetFileSystem(), dir->Str(), {"dt"})); std::shared_ptr
table = format_table; - // A context that never was is refused first of all, and by name, exactly as the generic - // entry points refuse one. ASSERT_NOK_WITH_MSG(TableScan::Create(nullptr), "scan context is null pointer"); ASSERT_NOK_WITH_MSG(table->NewScan(nullptr), "scan context is null pointer"); ASSERT_NOK_WITH_MSG(TableRead::Create(nullptr), "read context is null pointer"); @@ -2954,7 +2924,6 @@ TEST(FormatTableTest, TestTheTableEntryPointsRefuseWhatAFormatTableCannotHonour) ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(nullptr), "commit context is null pointer"); ASSERT_NOK_WITH_MSG(table->NewCommit(nullptr), "commit context is null pointer"); - // A streaming scan follows snapshots, which a format table keeps none of. auto streaming_scan_context = [&dir]() -> Result> { ScanContextBuilder builder(dir->Str()); builder.WithStreamingMode(true); @@ -2967,7 +2936,6 @@ TEST(FormatTableTest, TestTheTableEntryPointsRefuseWhatAFormatTableCannotHonour) ASSERT_NOK_WITH_MSG(table->NewScan(std::move(streaming_context_again)), "nothing for a streaming scan to follow"); - // Vector search needs an index a format table never builds. auto vector_read_context = [&dir]() -> Result> { ReadContextBuilder builder(dir->Str()); builder.SetVectorSearch(std::make_shared( @@ -2982,7 +2950,6 @@ TEST(FormatTableTest, TestTheTableEntryPointsRefuseWhatAFormatTableCannotHonour) ASSERT_NOK_WITH_MSG(table->NewRead(std::move(vector_context_again)), "not supported for format tables"); - // A catalog takes the snapshot a native table commits, and a format table has none to give. auto catalog = std::make_shared(); catalog->SetTableSchema(format_table->LatestSchema()); @@ -3008,7 +2975,6 @@ TEST(FormatTableTest, TestTheTableEntryPointsRefuseWhatAFormatTableCannotHonour) ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(catalog_commit_again)), "keeps no snapshot for a catalog to take"); - // A write id prefixes a postpone-bucket writer's files; a format table has no buckets. auto write_id_context = [&dir]() -> Result> { WriteContextBuilder builder(dir->Str(), "test-user"); builder.WithWriteId(3); @@ -3022,9 +2988,6 @@ TEST(FormatTableTest, TestTheTableEntryPointsRefuseWhatAFormatTableCannotHonour) } TEST(FormatTableTest, TestTheGenericEntryPointsRefuseAFormatTablePath) { - // A load from a path is the data table's way in, so every generic entry point refuses the - // format table the schema under the path names rather than dispatching to it - and says where - // the table does come from, so a caller holding only the path is pointed at the catalog. std::unique_ptr dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); ASSERT_OK_AND_ASSIGN(std::shared_ptr table, @@ -3046,8 +3009,6 @@ TEST(FormatTableTest, TestTheGenericEntryPointsRefuseAFormatTablePath) { ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(commit_context)), "names a format table"); - // The refusal points at the table's own operators, and those serve the same path-built - // contexts: this is the caller the message sends to `GetTable` and `NewXxx`. ScanContextBuilder served_builder(dir->Str()); ASSERT_OK_AND_ASSIGN(std::unique_ptr served_context, served_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr served_scan, @@ -3057,10 +3018,6 @@ TEST(FormatTableTest, TestTheGenericEntryPointsRefuseAFormatTablePath) { } TEST(FormatTableTest, TestABranchNamedThroughOptionsIsRefusedByTheTableOperators) { - // The `branch` option names a branch a format table keeps no metadata for, and a read or a - // write would go to its one location whatever the option named. Refusing it is the only - // answer that is not silently the wrong one - an overwrite would delete the main branch's - // data while claiming to replace another branch's. std::unique_ptr dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); ASSERT_OK_AND_ASSIGN(std::shared_ptr table, @@ -3117,8 +3074,6 @@ TEST(FormatTableTest, TestATableCopiesItselfWithTheOptionsGivenAtTheCall) { ASSERT_OK_AND_ASSIGN(std::shared_ptr table, CreateTable(dir->GetFileSystem(), dir->Str(), {"dt"})); - // The copy overlays what the call gives on the options the schema stored, the precedence - // `Copy()` promises: a probe option lands, `type` - structural - does not. ASSERT_OK_AND_ASSIGN(std::shared_ptr
copied, table->Copy({{Options::TYPE, "table"}, {"probe", "on-top"}})); std::shared_ptr copied_table = std::static_pointer_cast(copied); @@ -3128,17 +3083,10 @@ TEST(FormatTableTest, TestATableCopiesItselfWithTheOptionsGivenAtTheCall) { EXPECT_EQ(copied_table->Location(), table->Location()); EXPECT_EQ(copied_table->GetFileSystem(), table->GetFileSystem()); - // The refusal `Copy(table, ...)` reports is the one the table itself reports: the copy goes - // through the same entry point, not a shortcut around its checks. ASSERT_NOK_WITH_MSG(table->Copy({{Options::BRANCH, "dev"}}), "has no branches"); } TEST(FormatTableTest, TestAFormatTableSchemaOnABranchIsRefusedRatherThanServedFromTheTableRoot) { - // A context that names a path takes its branch from `WithBranch()`, and the schema of that - // branch decides what the entry point refuses. A format table's data is the files under the - // table path whichever branch named it, so the format table the branch's schema names is - // refused rather than served from there under another branch's name - the branch's table - // comes from the catalog, and its rows from that table's own operators. std::unique_ptr dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); ASSERT_OK_AND_ASSIGN(std::shared_ptr table, @@ -3166,8 +3114,6 @@ TEST(FormatTableTest, TestAFormatTableSchemaOnABranchIsRefusedRatherThanServedFr ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), "names a format table"); - // The main branch is untouched by any of it: the table the catalog hands back for it serves - // the scan from its own operators. ScanContextBuilder main_builder(dir->Str()); ASSERT_OK_AND_ASSIGN(std::unique_ptr main_context, main_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr main_scan, @@ -3178,9 +3124,6 @@ TEST(FormatTableTest, TestAFormatTableSchemaOnABranchIsRefusedRatherThanServedFr } TEST(FormatTableTest, TestTheTableOperatorsCarryTheContextThrough) { - // A context promises that options given at the call win over the ones the schema stored, and - // that a branch is honoured. Serving the operators off the table itself must not quietly drop - // any of that. std::unique_ptr dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); ASSERT_OK_AND_ASSIGN(std::shared_ptr table, @@ -3202,8 +3145,6 @@ TEST(FormatTableTest, TestTheTableOperatorsCarryTheContextThrough) { write->PrepareCommit()); ASSERT_EQ(messages.size(), 3u) << "target-file-row-num given at the call was ignored"; - // Published through the table's own commit, or nothing below would see the rows: until the - // commit renames them the files sit in the hidden `_temporary` directory a scan skips. CommitContextBuilder generic_commit_builder(dir->Str(), "test-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr generic_commit_context, generic_commit_builder.Finish()); @@ -3212,8 +3153,6 @@ TEST(FormatTableTest, TestTheTableOperatorsCarryTheContextThrough) { ASSERT_OK(generic_commit->Commit(messages)); ASSERT_OK(write->Close()); - // A branch this table has no schema on names no table at all as far as the generic entry - // points are concerned, so the format branch must not answer for it. ReadContextBuilder branch_builder(dir->Str()); branch_builder.WithBranch("nosuchbranch"); ASSERT_OK_AND_ASSIGN(std::unique_ptr branch_context, branch_builder.Finish()); diff --git a/src/paimon/core/table/primary_key_file_store_table.cpp b/src/paimon/core/table/primary_key_file_store_table.cpp index 9a5875c9f..90ed47a30 100644 --- a/src/paimon/core/table/primary_key_file_store_table.cpp +++ b/src/paimon/core/table/primary_key_file_store_table.cpp @@ -68,9 +68,6 @@ class MergeFunctionWrapper; namespace { -/// Whether a schema older than `current_schema_id` carried a primary key b-tree index, which a -/// write has to keep maintaining: the index files an earlier schema wrote are still there to be -/// read, and dropping the definition now would leave them orphaned. Result HasHistoricalPrimaryKeyBTreeDefinition( const std::shared_ptr& schema_manager, int64_t current_schema_id) { if (current_schema_id <= TableSchema::FIRST_SCHEMA_ID) { @@ -106,7 +103,6 @@ Result> PrimaryKeyFileStoreTable::CreateWrite( const std::shared_ptr& path_factory, const std::shared_ptr& snapshot_manager, const std::shared_ptr& io_manager, bool ignore_previous_files) const { - // pk table const std::shared_ptr& schema = store_->GetSchema(); if (ctx.GetRealtimeContext()) { PAIMON_RETURN_NOT_OK(PrimaryKeyTableUtils::ValidateRealtimeOptions(options, *schema)); @@ -142,7 +138,6 @@ Result> PrimaryKeyFileStoreTable::CreateWrite( PrimaryKeyTableUtils::CreateMergeFunction(arrow_schema, primary_keys, options, ctx.GetMemoryPool())); if (options.NeedLookup() && options.GetMergeEngine() != MergeEngine::FIRST_ROW) { - // don't wrap first row, it is already OK merge_function = std::make_unique(std::move(merge_function)); } std::shared_ptr> merge_function_wrapper = @@ -261,8 +256,6 @@ CommitScanner::ScanSupplier PrimaryKeyFileStoreTable::NewCommitScanSupplier( const std::shared_ptr& manifest_file, const std::shared_ptr& arrow_schema, const CoreOptions& options, const std::shared_ptr& executor, const std::shared_ptr& pool) const { - // The schema manager the table carries, which the scans this supplier hands out read the - // schema through. const std::shared_ptr& schema_manager = GetSchemaManager(); const std::shared_ptr& table_schema = store_->GetSchema(); return [snapshot_manager, schema_manager, manifest_list, manifest_file, table_schema, diff --git a/src/paimon/core/table/source/table_read_test.cpp b/src/paimon/core/table/source/table_read_test.cpp index 3e79c537d..bd608bd74 100644 --- a/src/paimon/core/table/source/table_read_test.cpp +++ b/src/paimon/core/table/source/table_read_test.cpp @@ -262,9 +262,6 @@ TEST(TableReadTest, TestCreateAppendOnlyTableRead) { TEST(TableReadTest, TestMergeOptions) { std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; { - // The options the schema carries stand in the table the load builds, and a setting the - // context adds on top - the table was loaded with neither of these keys - is served as - // given. ReadContextBuilder context_builder(path); context_builder.SetReadFieldNames({"f0", "f1", "f2", "f3"}); context_builder.AddOption("read.batch-size", "2"); @@ -287,10 +284,6 @@ TEST(TableReadTest, TestMergeOptions) { ASSERT_EQ(expected_options, core_options.ToMap()); } { - // An option the schema also carries speaks at the load: the table the context is - // handed back is built with the context's value over the schema's, and the read serves - // the merged options. A caller holding a table already - a load that is over - is the - // one that gets a refusal instead, which `TableTest` covers. ReadContextBuilder context_builder(path); context_builder.SetReadFieldNames({"f0", "f1", "f2", "f3"}); context_builder.AddOption("bucket", "10"); @@ -334,9 +327,6 @@ TEST(TableReadTest, TestDependencyReadWithoutCatalogUsesReadOptions) { expected_options[read_via_option] = explicit_header ? "read-view" : "table-view"; ASSERT_EQ(dependency_context->root_path, "warehouse"); ASSERT_EQ(dependency_context->options, expected_options); - // The load merged the read options into the table it built - a table is the product of - // the load - so the schema the read serves carries the explicit header over the one the - // schema was written with, the same value the dependency context sees. ASSERT_EQ(internal_context->GetTableSchema()->Options(), expected_options); } } diff --git a/src/paimon/core/table/source/table_scan_test.cpp b/src/paimon/core/table/source/table_scan_test.cpp index 17bf2e874..4fd0f9873 100644 --- a/src/paimon/core/table/source/table_scan_test.cpp +++ b/src/paimon/core/table/source/table_scan_test.cpp @@ -167,8 +167,6 @@ TEST(TableScanTest, TestPkSchemaEvolutionScan) { TEST(TableScanTest, TestReadOptimizedPrimaryKeyStreamingScanUnsupported) { std::string path = paimon::test::GetDataDir() + "/orc/pk_table_with_alter_table.db/pk_table_with_alter_table$ro"; - // A system table is not served by the entry point a path names any more: the loader hands - // the table back the way a catalog's `GetTable` would, and the table refuses the scan. std::map options = {{Options::FILE_FORMAT, "orc"}}; ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options)); ASSERT_OK_AND_ASSIGN( diff --git a/src/paimon/core/table/system/audit_log_system_table.cpp b/src/paimon/core/table/system/audit_log_system_table.cpp index 0bf82ead3..671081e54 100644 --- a/src/paimon/core/table/system/audit_log_system_table.cpp +++ b/src/paimon/core/table/system/audit_log_system_table.cpp @@ -400,9 +400,6 @@ Result> AuditLogSystemTable::NewScan( .WithStreamingMode(context->IsStreamingMode()) .WithMemoryPool(context->GetMemoryPool()) .WithExecutor(context->GetExecutor()) - // The view's own file system when the context names none, rather than the default backend. - // A `ScanContext` carries no scheme-to-identifier map, so unlike `CreateDataReadContext` - // below there is no map here to resolve through instead. .WithFileSystem(context->GetSpecificFileSystem() != nullptr ? context->GetSpecificFileSystem() : GetFileSystem()) @@ -465,8 +462,6 @@ Result> AuditLogSystemTable::CreateDataReadContext( PAIMON_ASSIGN_OR_RAISE(std::shared_ptr base_read_schema, BaseReadSchema()); PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(read_options)); ReadContextBuilder builder(table_path_); - // The view's own file system, but only when the context names neither one of its own nor a - // scheme map to resolve through - a file system and a scheme map cannot both be named. std::shared_ptr data_table_fs = context->GetSpecificFileSystem(); if (data_table_fs == nullptr && context->GetFileSystemSchemeToIdentifierMap().empty()) { data_table_fs = GetFileSystem(); diff --git a/src/paimon/core/table/system/global_system_tables.cpp b/src/paimon/core/table/system/global_system_tables.cpp index 92f10c49d..050964d62 100644 --- a/src/paimon/core/table/system/global_system_tables.cpp +++ b/src/paimon/core/table/system/global_system_tables.cpp @@ -126,8 +126,6 @@ Result IsEnabled(const GlobalSystemTableRegistryEntry& entry, CatalogOptionsSystemTable::kEnabledOption, false); } -// Whether a table reads its rows by enumerating what the catalog holds, and so cannot be built -// without one. bool RequiresCatalog(const std::string& table_name) { std::string normalized = StringUtils::ToLowerCase(table_name); for (const auto& entry : GlobalSystemTableRegistry()) { @@ -146,7 +144,6 @@ struct CatalogTableInfo { // Match Java CatalogUtils::listAllTables: tolerate databases or tables removed concurrently, but // propagate all other catalog and schema errors instead of returning incomplete system-table rows. -// `table_name` names the table asking, for the refusal a missing catalog reports. Result> LoadAllDataTables( const std::shared_ptr& catalog, const std::string& table_name) { if (catalog == nullptr) { @@ -306,8 +303,6 @@ Result GlobalSystemTableLoader::IsSupported( Result> GlobalSystemTableLoader::Load( const std::string& table_name, const GlobalSystemTableContext& context) { - // Refused here rather than at the first read: a table handed back to a caller that can never - // build its rows is a failure the caller finds out about late and far from its cause. if (context.catalog == nullptr && RequiresCatalog(table_name)) { return Status::NotImplemented("global system table requires catalog context: ", table_name); } @@ -325,9 +320,6 @@ Result> GlobalSystemTableLoader::LoadForSchema( } PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, entry.factory(context)); - // The table is handed out through the `Table` interface, which serves the schema - // the base holds, so its own read-only schema is adopted before it leaves the - // loader. PAIMON_RETURN_NOT_OK(system_table->AdoptOwnSchema()); return system_table; } diff --git a/src/paimon/core/table/system/in_memory_system_table.cpp b/src/paimon/core/table/system/in_memory_system_table.cpp index 87fdd1d29..a2c69195a 100644 --- a/src/paimon/core/table/system/in_memory_system_table.cpp +++ b/src/paimon/core/table/system/in_memory_system_table.cpp @@ -119,8 +119,6 @@ Result> InMemorySystemTable::NewRead( std::unique_ptr ctx) const { const std::shared_ptr context = std::move(ctx); PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "read")); - // The rows this view serves are its own, read straight out of memory: there is no search - // over them, so a search the caller asked for would be quietly answered with plain rows. PAIMON_RETURN_NOT_OK(CheckNoSearch(context)); return std::make_unique( checked_pointer_cast(shared_from_this()), diff --git a/src/paimon/core/table/system/read_optimized_system_table.cpp b/src/paimon/core/table/system/read_optimized_system_table.cpp index 2c4123f2f..89f0455b0 100644 --- a/src/paimon/core/table/system/read_optimized_system_table.cpp +++ b/src/paimon/core/table/system/read_optimized_system_table.cpp @@ -69,9 +69,6 @@ Result> ReadOptimizedSystemTable::NewScan( .WithStreamingMode(context->IsStreamingMode()) .WithMemoryPool(context->GetMemoryPool()) .WithExecutor(context->GetExecutor()) - // The view's own file system when the context names none, rather than the default backend. - // A `ScanContext` carries no scheme-to-identifier map, so unlike `CreateDataReadContext` - // below there is no map here to resolve through instead. .WithFileSystem(context->GetSpecificFileSystem() != nullptr ? context->GetSpecificFileSystem() : GetFileSystem()) @@ -106,8 +103,6 @@ Result> ReadOptimizedSystemTable::CreateDataReadCon branch = branch_iter->second; } ReadContextBuilder builder(table_path_); - // The view's own file system, but only when the context names neither one of its own nor a - // scheme map to resolve through - a file system and a scheme map cannot both be named. std::shared_ptr data_table_fs = context->GetSpecificFileSystem(); if (data_table_fs == nullptr && context->GetFileSystemSchemeToIdentifierMap().empty()) { data_table_fs = GetFileSystem(); diff --git a/src/paimon/core/table/system/system_table.cpp b/src/paimon/core/table/system/system_table.cpp index 7a8146534..872f6e502 100644 --- a/src/paimon/core/table/system/system_table.cpp +++ b/src/paimon/core/table/system/system_table.cpp @@ -198,9 +198,6 @@ Status SystemTable::CheckNoSearch(const std::shared_ptr& context) { return Status::NotImplemented( "VectorSearch or FullTextSearch does not support system-table reads yet"); } - // A `FullTextMatch` predicate is not a TopK request, so the check above does not see it. The - // factory refuses one for a system table with the same message. An in-memory view serves rows - // of its own and never evaluates a predicate, so without this it answers with all of them. if (PredicateUtils::ContainsFullTextMatch(context->GetPredicate())) { return Status::NotImplemented("FullTextMatch predicate execution is not implemented yet"); } @@ -241,8 +238,6 @@ Result> SystemTableLoader::Load( if (factory) { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, factory.value()(fs, table_path, table_schema, dynamic_options)); - // The table is handed out through the `Table` interface, which serves the schema the - // base holds, so its own read-only schema is adopted before it leaves the loader. PAIMON_RETURN_NOT_OK(system_table->AdoptOwnSchema()); return system_table; } @@ -295,7 +290,6 @@ Result> SystemTableLoader::LoadFromPath( // The warehouse is the grandparent of the sys/
path context.warehouse = PathUtil::GetParentDirPath(PathUtil::GetParentDirPath(path)); context.catalog_options = dynamic_options; - // Note: context.catalog is intentionally left empty here. // Global tables loaded from path do not have a Catalog reference and // cannot enumerate databases/tables. Only tables that don't require // catalog enumeration (e.g. catalog_options) will work in this path. diff --git a/src/paimon/core/table/system/system_table_test.cpp b/src/paimon/core/table/system/system_table_test.cpp index cfd4f5591..9ea5f3389 100644 --- a/src/paimon/core/table/system/system_table_test.cpp +++ b/src/paimon/core/table/system/system_table_test.cpp @@ -227,8 +227,6 @@ TEST(SystemTableTest, TestReadOptimizedSystemTableRegistration) { SystemTableLoader::Load(ReadOptimizedSystemTable::kName, /*fs=*/nullptr, "/tmp/table", table_schema, /*dynamic_options=*/{})); - // The name carries the data table it belongs to, the way Java Paimon's - // `ReadOptimizedTable.name()` does. ASSERT_EQ(system_table->Name(), "table$ro"); ASSERT_OK_AND_ASSIGN(std::shared_ptr arrow_schema, system_table->ArrowSchema()); @@ -238,8 +236,6 @@ TEST(SystemTableTest, TestReadOptimizedSystemTableRegistration) { } TEST(SystemTableTest, TestCopyIsRefused) { - // A system table is a read-only view over the data table underneath it, and the schema it - // exposes is not one a copy reconfigures. The data table is copied through its own `Table`. std::map options = {{Options::FILE_SYSTEM, "local"}, {Options::FILE_FORMAT, "orc"}}; ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, @@ -311,9 +307,6 @@ TEST(SystemTableTest, TestGlobalSystemTableWithoutCatalogReturnsNotImplemented) "global system table requires catalog context: tables"); } -// A system table reached through `NewXxx` directly - one a catalog handed out - is checked the -// way the static factories check the contexts they take: a null context is refused rather than -// dereferenced. TEST(SystemTableTest, TestOperatorEntriesRefuseANullContext) { std::map options = {{Options::FILE_SYSTEM, "local"}, {Options::FILE_FORMAT, "orc"}}; @@ -332,9 +325,6 @@ TEST(SystemTableTest, TestOperatorEntriesRefuseANullContext) { ASSERT_NOK_WITH_MSG(read_optimized.NewRead(nullptr), "read context is null pointer"); } -// The scan and read entry points serve a data table only: a `T$system` path is refused with a -// message pointing at the catalog that hands the table back, so a caller holding only the path is -// not told the table does not exist. TEST(SystemTableTest, TestTheOperatorEntriesRefuseASystemTablePath) { const std::string system_table_path = "/tmp/warehouse/db.db/tbl$options"; @@ -347,9 +337,6 @@ TEST(SystemTableTest, TestTheOperatorEntriesRefuseASystemTablePath) { ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), "names a system table"); } -// The reads these views serve build a context of their own for the data table underneath, which -// carries no search: a search the caller asked for is refused rather than quietly answered with -// plain rows. TEST(SystemTableTest, TestNewReadRefusesASearch) { std::map options = {{Options::FILE_SYSTEM, "local"}, {Options::FILE_FORMAT, "orc"}}; @@ -382,9 +369,6 @@ TEST(SystemTableTest, TestNewReadRefusesASearch) { "VectorSearch or FullTextSearch does not support system-table reads yet"); } -// `SetPredicate` carries a search too, as a `FullTextMatch` leaf. An in-memory view reads rows -// of its own and never evaluates a predicate, so without the check it would answer with all of -// them where `TableRead::Create` refuses the same request. TEST(SystemTableTest, TestNewReadRefusesAFullTextMatchPredicate) { std::map options = {{Options::FILE_SYSTEM, "local"}, {Options::FILE_FORMAT, "orc"}}; @@ -447,8 +431,6 @@ TEST(SystemTableTest, TestGlobalPartitionsUsesTheFileSystemOfEachTable) { std::unique_ptr dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); - // Held by a `shared_ptr` because a global system table keeps shared ownership of the catalog - // it reads its rows through. std::shared_ptr catalog = std::make_shared(core_options.GetFileSystem(), dir->Str(), options); diff --git a/src/paimon/core/table/table_test.cpp b/src/paimon/core/table/table_test.cpp index 2b6be91a4..4f321f17b 100644 --- a/src/paimon/core/table/table_test.cpp +++ b/src/paimon/core/table/table_test.cpp @@ -62,8 +62,6 @@ namespace paimon::test { namespace { -// Creates a managed table directory and a `Table` over it, holding the schema written to -// `branch`'s directory under `identifier` with `options`. Result> CreateManagedTable( const std::shared_ptr& file_system, const std::string& path, const Identifier& identifier, const std::string& branch = "main", @@ -80,7 +78,6 @@ Result> CreateManagedTable( /*fs_scheme_map=*/{}, /*dynamic_options=*/{}); } -/// Writes two rows through the table's own write and commit, so a scan below has data to find. void WriteAndCommitTwoRows(const std::shared_ptr
& table, const std::string& root_path) { WriteContextBuilder write_builder(root_path, "commit-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); @@ -108,8 +105,6 @@ void WriteAndCommitTwoRows(const std::shared_ptr
& table, const std::strin ASSERT_OK(write->Close()); } -/// Writes one `id`/`v` row through the table's own write and commit, so a scan below has data -/// to find. void WriteAndCommitRow(const std::shared_ptr
& table, const std::string& root_path, int32_t id, int32_t v) { WriteContextBuilder write_builder(root_path, "commit-user"); @@ -157,14 +152,12 @@ TEST(TableTest, TestTableCarriesManagersOfItsBranch) { /*branch=*/"dev")); auto file_store_table = std::dynamic_pointer_cast(table); ASSERT_TRUE(file_store_table != nullptr); - // The branch the table was built for is the one its managers point at. EXPECT_EQ(file_store_table->CurrentBranch(), "dev"); ASSERT_TRUE(file_store_table->GetSchemaManager() != nullptr); ASSERT_TRUE(file_store_table->GetSnapshotManager() != nullptr); EXPECT_EQ(file_store_table->GetSnapshotManager()->Branch(), "dev"); EXPECT_EQ(file_store_table->GetSnapshotManager()->RootPath(), path); EXPECT_EQ(file_store_table->GetSnapshotManager()->Fs(), fs); - // A copy serves the same branch through the same managers, whose cache it shares. ASSERT_OK_AND_ASSIGN(std::shared_ptr
copy, table->Copy({})); auto file_store_copy = std::dynamic_pointer_cast(copy); ASSERT_TRUE(file_store_copy != nullptr); @@ -182,10 +175,6 @@ TEST(TableTest, TestCopyKeepsTheBranchTheTableServes) { CreateManagedTable(dir->GetFileSystem(), dir->Str(), identifier, /*branch=*/"main", {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); - // A copy handed a `branch` option is the table it was copied from: the branch is the - // table's own - `SwitchToBranch` is the way to change it - so the option moves neither the - // copy nor its options, and an operator asked for the branch the copy serves - `main` - is - // served rather than refused by options claiming a `dev` the copy does not serve. ASSERT_OK_AND_ASSIGN(std::shared_ptr
copy, table->Copy({{Options::BRANCH, "dev"}})); auto file_store_copy = std::dynamic_pointer_cast(copy); ASSERT_TRUE(file_store_copy != nullptr); @@ -198,8 +187,6 @@ TEST(TableTest, TestCopyKeepsTheBranchTheTableServes) { file_store_copy->NewScan(std::move(ctx))); } - // The same the other way round: a copy of a branch table serves that branch still, so - // naming the branch it serves - `dev` - is agreed with rather than refused. ASSERT_OK_AND_ASSIGN(std::shared_ptr
branch_table, CreateManagedTable(dir->GetFileSystem(), dir->Str() + "/branch_tbl", identifier, /*branch=*/"dev", @@ -225,7 +212,6 @@ TEST(TableTest, TestACopyHandsItsDynamicOptionsToTheFallbackReader) { const std::string path = dir->Str() + "/tbl"; const Identifier identifier("db", "tbl"); - // A primary-key table whose reads fall back to the `fb` branch, and that branch's schema. SchemaManager schema_manager(fs, path, "main"); auto schema = arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false)}); ASSERT_OK_AND_ASSIGN( @@ -248,11 +234,6 @@ TEST(TableTest, TestACopyHandsItsDynamicOptionsToTheFallbackReader) { identifier, /*uuid=*/"", /*branch=*/"main", /*schema_manager=*/nullptr, /*snapshot_manager=*/nullptr, /*fs_scheme_map=*/{}, /*dynamic_options=*/{})); - // The copy turns the sequence-number column on, and a read projects it through a context - // that carries no options of its own. The main reader serves the copy's merged options, so - // it knows the column; the fallback serves the `fb` branch's schema, whose options do not - // name it, so the copy's own dynamic options are what let the fallback reader know it too - - // without them the read as a whole fails to build. ASSERT_OK_AND_ASSIGN(std::shared_ptr
copy, table->Copy({{Options::KEY_VALUE_SEQUENCE_NUMBER_ENABLED, "true"}})); ReadContextBuilder read_builder(path); @@ -269,7 +250,6 @@ TEST(TableTest, TestACopyOfACopyServesTheLatestValueOfARepeatedDynamicOption) { const std::string path = dir->Str() + "/tbl"; const Identifier identifier("db", "tbl"); - // A primary-key table whose reads fall back to the `fb` branch, and that branch's schema. SchemaManager schema_manager(fs, path, "main"); auto schema = arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false)}); ASSERT_OK_AND_ASSIGN( @@ -292,11 +272,6 @@ TEST(TableTest, TestACopyOfACopyServesTheLatestValueOfARepeatedDynamicOption) { identifier, /*uuid=*/"", /*branch=*/"main", /*schema_manager=*/nullptr, /*snapshot_manager=*/nullptr, /*fs_scheme_map=*/{}, /*dynamic_options=*/{})); - // The option is turned off and then on again by the copy the read serves, so the copy's - // schema carries the latest value. The fallback branch's schema does not carry the option - // at all, so what lets its reader know the column is the copy's accumulated dynamic - // options - which must speak the same latest value the schema does, not the older one a - // first copy left, or the fallback reader fails to build while the main one succeeds. ASSERT_OK_AND_ASSIGN(std::shared_ptr
off, table->Copy({{Options::KEY_VALUE_SEQUENCE_NUMBER_ENABLED, "false"}})); ASSERT_OK_AND_ASSIGN(std::shared_ptr
copy, @@ -315,9 +290,6 @@ TEST(TableTest, TestAFallbackReaderServesItsBranchsOwnSchema) { const std::string path = dir->Str() + "/tbl"; const Identifier identifier("db", "tbl"); - // The main branch is an append-only table whose reads fall back to the `fb` branch, and - // that branch is laid out as a primary-key table: a second write of the key the first - // wrote is a merge, not a duplicate. SchemaManager main_schema_manager(fs, path, "main"); auto row_schema = arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false), arrow::field("v", arrow::int32())}); @@ -351,11 +323,6 @@ TEST(TableTest, TestAFallbackReaderServesItsBranchsOwnSchema) { WriteAndCommitRow(fallback, path, /*id=*/1, /*v=*/10); WriteAndCommitRow(fallback, path, /*id=*/1, /*v=*/20); - // The read of the main table carries its fallback reader, which reads the `fb` branch's - // data through the `fb` branch's own schema - so it is the primary-key read that schema's - // layout asks for, not the append-only one the main table's kind would pick. Its splits - // are the `fb` branch's own, scanned through a table that serves `fb` and read as - // fallback ones, which is how a fallback reader is asked for its rows. ReadContextBuilder read_builder(path); read_builder.SetReadFieldNames({"id", "v"}); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); @@ -376,10 +343,6 @@ TEST(TableTest, TestAFallbackReaderServesItsBranchsOwnSchema) { ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, read->CreateReader(fallback_splits)); - // The one merged row the `fb` branch's primary-key layout leaves behind - at the value - // the second write carried - is what the fallback reader answers with; the append-only - // reader the main table's kind would have picked answers with both rows, or refuses the - // split outright. ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, batch_reader->NextBatch()); ASSERT_OK_AND_ASSIGN(std::shared_ptr array, ReadResultCollector::GetArray(std::move(batch))); @@ -401,6 +364,129 @@ TEST(TableTest, TestAFallbackReaderServesItsBranchsOwnSchema) { EXPECT_TRUE(BatchReader::IsEofBatch(eof_batch)); } +TEST(TableTest, TestAFallbackReaderServesItsBranchsOwnPartitionKeys) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::shared_ptr fs = dir->GetFileSystem(); + const std::string path = dir->Str() + "/tbl"; + const Identifier identifier("db", "tbl"); + + SchemaManager main_schema_manager(fs, path, "main"); + auto partitioned_schema = + arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false), + arrow::field("v", arrow::int32()), arrow::field("dt", arrow::utf8())}); + ASSERT_OK_AND_ASSIGN(std::unique_ptr main_created, + main_schema_manager.CreateTable(partitioned_schema, + /*partition_keys=*/{"dt"}, + /*primary_keys=*/{}, + {{Options::SCAN_FALLBACK_BRANCH, "fb"}})); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
main, + FileStoreTable::Create( + fs, path, + checked_pointer_cast(std::shared_ptr(std::move(main_created))), + identifier, /*uuid=*/"", /*branch=*/"main", /*schema_manager=*/nullptr, + /*snapshot_manager=*/nullptr, /*fs_scheme_map=*/{}, /*dynamic_options=*/{})); + + SchemaManager fallback_schema_manager(fs, path, "fb"); + auto row_schema = arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false), + arrow::field("v", arrow::int32())}); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr fallback_created, + fallback_schema_manager.CreateTable(row_schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
fallback, + FileStoreTable::Create( + fs, path, + checked_pointer_cast(std::shared_ptr(std::move(fallback_created))), + identifier, /*uuid=*/"", /*branch=*/"fb", /*schema_manager=*/nullptr, + /*snapshot_manager=*/nullptr, /*fs_scheme_map=*/{}, /*dynamic_options=*/{})); + WriteAndCommitRow(fallback, path, /*id=*/1, /*v=*/20); + + ReadContextBuilder read_builder(path); + read_builder.SetReadFieldNames({"id", "v"}); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, main->NewRead(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, + ScanContextBuilder(path).Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr fallback_scan, + fallback->NewScan(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr fallback_plan, fallback_scan->CreatePlan()); + ASSERT_FALSE(fallback_plan->Splits().empty()); + std::vector> fallback_splits; + for (const std::shared_ptr& split : fallback_plan->Splits()) { + auto data_split = std::dynamic_pointer_cast(split); + ASSERT_TRUE(data_split != nullptr); + fallback_splits.push_back( + std::make_shared(data_split, /*is_fallback=*/true)); + } + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, + read->CreateReader(fallback_splits)); + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, batch_reader->NextBatch()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr array, + ReadResultCollector::GetArray(std::move(batch))); + ASSERT_TRUE(array); + ASSERT_EQ(array->length(), 1); + auto struct_array = std::dynamic_pointer_cast(array); + ASSERT_TRUE(struct_array != nullptr); + // Every batch reader leads with `_VALUE_KIND`, ahead of the fields the read named. + EXPECT_EQ(struct_array->num_fields(), 3); + auto value_kind_array = std::dynamic_pointer_cast(struct_array->field(0)); + auto id_array = std::dynamic_pointer_cast(struct_array->field(1)); + auto v_array = std::dynamic_pointer_cast(struct_array->field(2)); + ASSERT_TRUE(value_kind_array != nullptr); + ASSERT_TRUE(id_array != nullptr); + ASSERT_TRUE(v_array != nullptr); + EXPECT_EQ(value_kind_array->Value(0), 0); + EXPECT_EQ(id_array->Value(0), 1); + EXPECT_EQ(v_array->Value(0), 20); + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch eof_batch, batch_reader->NextBatch()); + EXPECT_TRUE(BatchReader::IsEofBatch(eof_batch)); +} + +TEST(TableTest, TestAFallbackReaderIgnoresReadContextOptionsAbsentFromTheTable) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::shared_ptr fs = dir->GetFileSystem(); + + auto make_table = + [&](const std::string& path, + const std::map& options) -> Result> { + SchemaManager schema_manager(fs, path, "main"); + auto schema = arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false)}); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr created, + schema_manager.CreateTable(schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options)); + if (options.count(Options::SCAN_FALLBACK_BRANCH) != 0) { + SchemaManager fallback_schema_manager(fs, path, "fb"); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr fallback_created, + fallback_schema_manager.CreateTable(schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, {})); + } + return FileStoreTable::Create( + fs, path, + checked_pointer_cast(std::shared_ptr(std::move(created))), + Identifier("db", "tbl"), /*uuid=*/"", /*branch=*/"main", /*schema_manager=*/nullptr, + /*snapshot_manager=*/nullptr, /*fs_scheme_map=*/{}, /*dynamic_options=*/{}); + }; + ASSERT_OK_AND_ASSIGN(std::shared_ptr
plain, make_table(dir->Str() + "/plain", {})); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
with_fallback, + make_table(dir->Str() + "/fallback", {{Options::SCAN_FALLBACK_BRANCH, "fb"}})); + + for (const std::shared_ptr
& table : {plain, with_fallback}) { + SCOPED_TRACE(table->GetRootPath()); + ReadContextBuilder read_builder(table->GetRootPath()); + read_builder.AddOption(Options::FILE_FORMAT, "bogus"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, + table->NewRead(std::move(read_context))); + } +} + TEST(TableTest, TestACopyThatMovesAnS3OptionRebuildsItsManagers) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); @@ -415,10 +501,6 @@ TEST(TableTest, TestACopyThatMovesAnS3OptionRebuildsItsManagers) { auto file_store_table = std::dynamic_pointer_cast(table); ASSERT_TRUE(file_store_table != nullptr); - // A native `s3.` option names the file system the S3 backend resolves from, the way an - // `fs.` one does: a copy that moves it re-resolves, so the managers are rebuilt rather - // than inherited - otherwise the copy would read its metadata through the old endpoint - // while its data goes through the new one. ASSERT_OK_AND_ASSIGN(std::shared_ptr
moved, table->Copy({{"s3.endpoint", "http://new"}})); auto moved_file_store_table = std::dynamic_pointer_cast(moved); @@ -426,8 +508,6 @@ TEST(TableTest, TestACopyThatMovesAnS3OptionRebuildsItsManagers) { EXPECT_NE(moved_file_store_table->GetSchemaManager(), file_store_table->GetSchemaManager()); EXPECT_NE(moved_file_store_table->GetSnapshotManager(), file_store_table->GetSnapshotManager()); - // A copy whose `s3.` option says what the schema already says resolves the same file - // system, so the managers are the ones this table already holds. ASSERT_OK_AND_ASSIGN(std::shared_ptr
kept, table->Copy({{"s3.endpoint", "http://old"}})); auto kept_file_store_table = std::dynamic_pointer_cast(kept); ASSERT_TRUE(kept_file_store_table != nullptr); @@ -442,14 +522,12 @@ TEST(TableTest, TestSwitchToBranch) { const std::string path = dir->Str() + "/tbl"; const Identifier identifier(Identifier::kUnknownDatabase, "tbl"); ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, CreateManagedTable(fs, path, identifier)); - // A branch of its own, with a schema of its own. SchemaManager dev_schema_manager(fs, path, "dev"); ASSERT_OK_AND_ASSIGN( [[maybe_unused]] std::unique_ptr dev_schema, dev_schema_manager.CreateTable(arrow::schema({arrow::field("c0", arrow::int32())}), /*partition_keys=*/{}, /*primary_keys=*/{}, {})); - // Switching to a branch serves that branch's schema and managers. auto main_file_store_table = std::dynamic_pointer_cast(table); ASSERT_TRUE(main_file_store_table != nullptr); ASSERT_OK_AND_ASSIGN(std::shared_ptr
dev_table, @@ -457,8 +535,6 @@ TEST(TableTest, TestSwitchToBranch) { auto dev_file_store_table = std::dynamic_pointer_cast(dev_table); ASSERT_TRUE(dev_file_store_table != nullptr); EXPECT_EQ("dev", dev_file_store_table->CurrentBranch()); - // The identifier names the branch switched to, so `Name()` and `FullName()` answer for the - // table served, and a caller that reloads through `FullName()` lands on the same branch. EXPECT_EQ("tbl$branch_dev", dev_table->Name()); EXPECT_EQ("tbl$branch_dev", dev_table->FullName()); ASSERT_EQ((std::vector{"c0"}), dev_table->LatestSchema()->FieldNames()); @@ -469,7 +545,6 @@ TEST(TableTest, TestSwitchToBranch) { ASSERT_TRUE(dev_file_store_table->GetSnapshotManager() != nullptr); EXPECT_EQ("dev", dev_file_store_table->GetSnapshotManager()->Branch()); - // Switching to the branch already served hands back the same table, managers shared. ASSERT_OK_AND_ASSIGN(std::shared_ptr
same_table, dev_file_store_table->SwitchToBranch("dev")); auto same_file_store_table = std::dynamic_pointer_cast(same_table); @@ -479,13 +554,11 @@ TEST(TableTest, TestSwitchToBranch) { EXPECT_EQ(same_file_store_table->GetSnapshotManager(), dev_file_store_table->GetSnapshotManager()); - // Switching back serves the main branch's schema again. ASSERT_OK_AND_ASSIGN(std::shared_ptr
main_again, dev_file_store_table->SwitchToBranch("main")); auto main_again_file_store_table = std::dynamic_pointer_cast(main_again); ASSERT_TRUE(main_again_file_store_table != nullptr); EXPECT_EQ("main", main_again_file_store_table->CurrentBranch()); - // Switching back leaves no branch in the name either, the way the main branch is addressed. EXPECT_EQ("tbl", main_again->Name()); EXPECT_EQ("tbl", main_again->FullName()); ASSERT_EQ((std::vector{"id"}), main_again->LatestSchema()->FieldNames()); @@ -493,7 +566,6 @@ TEST(TableTest, TestSwitchToBranch) { ASSERT_TRUE(main_again_schema != nullptr); EXPECT_EQ(0, main_again_schema->Options().count(Options::BRANCH)); - // A branch with no schema does not exist. ASSERT_NOK_WITH_MSG(main_file_store_table->SwitchToBranch("missing"), "branch"); } @@ -506,8 +578,6 @@ TEST(TableTest, TestNewWriteServesTheBranchTheTableWasLoadedWith) { CreateManagedTable(fs, dir->Str(), identifier, /*branch=*/"dev", {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); - // A context that names no branch is the one a caller hands straight to the table's write, - // which serves the branch the table was loaded with rather than the context's own main. WriteContextBuilder write_builder(dir->Str(), "commit-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr write, @@ -528,8 +598,6 @@ TEST(TableTest, TestNewWriteServesTheBranchTheTableWasLoadedWith) { write->PrepareCommit()); ASSERT_OK(write->Close()); - // The files land under the branch's own directory, which a write of the main branch - the - // branch the context alone would name - would never touch. std::vector branch_entries; ASSERT_OK(fs->ListDir(BranchManager::BranchPath(dir->Str(), "dev"), &branch_entries)); EXPECT_FALSE(branch_entries.empty()); @@ -544,8 +612,6 @@ TEST(TableTest, TestOperatorsServeTheBranchTheTableWasLoadedWith) { CreateManagedTable(dir->GetFileSystem(), dir->Str(), identifier, /*branch=*/"dev", {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); - // The write and the commit below carry no branch, and both serve the branch the table was - // loaded with, so the data they publish lands on it. WriteContextBuilder write_builder(dir->Str(), "commit-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr write, @@ -571,15 +637,12 @@ TEST(TableTest, TestOperatorsServeTheBranchTheTableWasLoadedWith) { table->NewCommit(std::move(commit_context))); ASSERT_OK(commit->Commit(messages)); - // The scan finds what they wrote, reading through the table's own managers on its branch. ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_ctx, ScanContextBuilder(dir->Str()).Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(scan_ctx))); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); EXPECT_FALSE(plan->Splits().empty()); - // A branch the scan context carries does not win either: the table's own does, so the data - // it finds is still the branch's. ScanContextBuilder main_scan_builder(dir->Str()); main_scan_builder.AddOption(Options::BRANCH, "main"); ASSERT_OK_AND_ASSIGN(std::unique_ptr main_scan_ctx, main_scan_builder.Finish()); @@ -687,8 +750,6 @@ TEST(TableTest, TestMainBranchAliasResolvesToTheMainBranch) { {Identifier("db", "tbl"), Identifier("db", "tbl$branch_main"), Identifier("db", "tbl$branch_MAIN")}) { SCOPED_TRACE(main_identifier.ToString()); - // A catalog that loaded the schema itself builds the table through the factory, which a - // rest catalog is shaped like. The alias resolves to the main branch as well. ASSERT_OK_AND_ASSIGN(std::string branch, main_identifier.GetBranchNameOrDefault()); ASSERT_OK_AND_ASSIGN(std::shared_ptr table, FileStoreTable::Create(fs, dir->Str(), schema, main_identifier, @@ -713,7 +774,6 @@ TEST(TableTest, TestCurrentBranch) { ASSERT_TRUE(main_file_store_table != nullptr); EXPECT_EQ(main_file_store_table->CurrentBranch(), "main"); - // The branch a table was created on is the one it carries. ASSERT_OK_AND_ASSIGN(std::shared_ptr
branch_table, CreateManagedTable(dir->GetFileSystem(), dir->Str() + "/branch_tbl", Identifier("db", "tbl", "dev"), /*branch=*/"dev")); @@ -732,9 +792,6 @@ TEST(TableTest, TestCatalogOpenedTableServesThePerTableFileSystem) { auto catalog = std::make_shared(); catalog->SetTableSchema(table->LatestSchema()); - // A catalog that hands out no per-table file system opens the table through the one its - // schema's options resolve - never through none, the default local one when they name no - // other - and one that does opens it with the file system issued for that table. ASSERT_OK_AND_ASSIGN(std::shared_ptr
opened, catalog->GetTable(identifier)); EXPECT_NE(opened->GetFileSystem(), nullptr); @@ -746,8 +803,6 @@ TEST(TableTest, TestCatalogOpenedTableServesThePerTableFileSystem) { } TEST(TableTest, TestGetTableServesEachKindOfTableAsWhatItIs) { - // The table's `type` option says what it is, so the one call that opens a table hands each - // kind back as what it is, through the `Table` interface the two kinds share. auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); auto fs = dir->GetFileSystem(); @@ -763,7 +818,6 @@ TEST(TableTest, TestGetTableServesEachKindOfTableAsWhatItIs) { format_schema_manager.CreateTable(schema, /*partition_keys=*/{"dt"}, /*primary_keys=*/{}, {{"type", "format-table"}, {"file.format", "parquet"}})); - // A format table comes back as the `FormatTable` it is. ASSERT_OK_AND_ASSIGN(std::shared_ptr
opened, catalog.GetTable(Identifier("db", "format"))); ASSERT_NE(nullptr, std::dynamic_pointer_cast(opened)); @@ -831,8 +885,6 @@ TEST(TableTest, TestNewScanServesTheSchemaTheTableWasLoadedWith) { ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, CreateManagedTable(fs, dir->Str(), identifier)); - // With the schema file gone the table still serves a scan: it carries the schema it was - // loaded with, which is what a scan is built from. SchemaManager schema_manager(fs, dir->Str()); ASSERT_OK(fs->Delete(schema_manager.ToSchemaPath(0), /*recursive=*/false)); ASSERT_OK_AND_ASSIGN(std::unique_ptr held_ctx, @@ -840,8 +892,6 @@ TEST(TableTest, TestNewScanServesTheSchemaTheTableWasLoadedWith) { ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr scan, table->NewScan(std::move(held_ctx))); - // A context without a table still reads the schema from the directory, which no longer has - // one. ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, ScanContextBuilder(dir->Str()).Finish()); ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(ctx)), "not found latest schema"); } @@ -855,8 +905,6 @@ TEST(TableTest, TestNewScanServesTheFileSystemTheTableWasOpenedWith) { CreateManagedTable(fs, dir->Str(), identifier, /*branch=*/"main", {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); - // A file system is only observable once a scan has data to read, so two rows are written and - // committed through the table's own write and commit. WriteContextBuilder write_builder(dir->Str(), "commit-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr write, @@ -882,8 +930,6 @@ TEST(TableTest, TestNewScanServesTheFileSystemTheTableWasOpenedWith) { ASSERT_OK(commit->Commit(messages)); ASSERT_OK(write->Close()); - // The table was opened with the local file system, so a scan serves its data through that - // one even when the context hands over a mock that can read nothing. ASSERT_OK_AND_ASSIGN( std::unique_ptr held_ctx, ScanContextBuilder(dir->Str()).WithFileSystem(std::make_shared()).Finish()); @@ -891,8 +937,6 @@ TEST(TableTest, TestNewScanServesTheFileSystemTheTableWasOpenedWith) { ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); ASSERT_FALSE(plan->Splits().empty()); - // A context without a table resolves the file system itself, which is the mock and reads no - // schema at all: the file system counterpart of the schema case above. ASSERT_OK_AND_ASSIGN( std::unique_ptr ctx, ScanContextBuilder(dir->Str()).WithFileSystem(std::make_shared()).Finish()); @@ -908,8 +952,6 @@ TEST(TableTest, TestNewScanServesTheRootPathTheTableWasOpenedWith) { CreateManagedTable(fs, dir->Str(), identifier, /*branch=*/"main", {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); - // Two rows are written and committed through the table's own write and commit, so the scan - // below has data to find. WriteContextBuilder write_builder(dir->Str(), "commit-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr write, @@ -935,16 +977,12 @@ TEST(TableTest, TestNewScanServesTheRootPathTheTableWasOpenedWith) { ASSERT_OK(commit->Commit(messages)); ASSERT_OK(write->Close()); - // The table was opened with this directory as its root path, so a scan serves its data from - // there even when the context names a different one. ASSERT_OK_AND_ASSIGN(std::unique_ptr held_ctx, ScanContextBuilder(dir->Str() + "/not-the-table").Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(held_ctx))); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); ASSERT_FALSE(plan->Splits().empty()); - // A context without a table reads the schema from the directory it names, which is not a - // table at all: the root path counterpart of the schema case above. ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, ScanContextBuilder(dir->Str() + "/not-the-table").Finish()); ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(ctx)), "not found latest schema"); @@ -1004,6 +1042,89 @@ TEST(TableTest, TestNewCommitCreatesAFileStoreCommit) { table->NewCommit(std::move(ctx))); } +TEST(TableTest, TestNewWriteAndCommitRefuseAContextThatNamesAnotherTable) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto other_dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(other_dir); + const std::shared_ptr fs = dir->GetFileSystem(); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + CreateManagedTable(fs, dir->Str(), identifier)); + + { + WriteContextBuilder write_builder(other_dir->Str(), "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, write_builder.Finish()); + ASSERT_NOK_WITH_MSG(table->NewWrite(std::move(ctx)), "names root path"); + } + { + CommitContextBuilder commit_builder(other_dir->Str(), "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, commit_builder.Finish()); + ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(ctx)), "names root path"); + } + + auto catalog = std::make_shared(); + catalog->SetFileSystem(fs); + catalog->SetTableSchema(table->LatestSchema()); + { + WriteContextBuilder write_builder(dir->Str(), "commit-user"); + write_builder.WithCatalog(catalog, Identifier("db", "other")); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, write_builder.Finish()); + ASSERT_NOK_WITH_MSG(table->NewWrite(std::move(ctx)), "names table"); + } + { + CommitContextBuilder commit_builder(dir->Str(), "commit-user"); + commit_builder.WithCatalog(catalog, Identifier("db", "other")); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, commit_builder.Finish()); + ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(ctx)), "names table"); + } + { + CommitContextBuilder commit_builder(dir->Str(), "commit-user"); + commit_builder.WithCatalog(catalog, Identifier("db", "tbl$branch_dev")); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, commit_builder.Finish()); + ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(ctx)), "names branch"); + } + { + CommitContextBuilder commit_builder(dir->Str(), "commit-user"); + commit_builder.UseRESTCatalogCommit(true).WithTableId("another-table"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, commit_builder.Finish()); + ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(ctx)), "table id"); + } + + CommitContextBuilder commit_builder(dir->Str(), "commit-user"); + commit_builder.WithCatalog(catalog, identifier); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, commit_builder.Finish()); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr commit, + table->NewCommit(std::move(ctx))); +} + +TEST(TableTest, TestNewCommitRefusesABranchACatalogCannotAddress) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::shared_ptr fs = dir->GetFileSystem(); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + CreateManagedTable(fs, dir->Str(), identifier, /*branch=*/"dev$x")); + + CommitContextBuilder fs_builder(dir->Str(), "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr fs_ctx, fs_builder.Finish()); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr fs_commit, + table->NewCommit(std::move(fs_ctx))); + + CommitContextBuilder rest_builder(dir->Str(), "commit-user"); + rest_builder.UseRESTCatalogCommit(true); + ASSERT_OK_AND_ASSIGN(std::unique_ptr rest_ctx, rest_builder.Finish()); + ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(rest_ctx)), "cannot contain"); + + auto catalog = std::make_shared(); + catalog->SetFileSystem(fs); + catalog->SetTableSchema(table->LatestSchema()); + WriteContextBuilder write_builder(dir->Str(), "commit-user"); + write_builder.WithCatalog(catalog, identifier); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_ctx, write_builder.Finish()); + ASSERT_NOK_WITH_MSG(table->NewWrite(std::move(write_ctx)), "cannot contain"); +} + TEST(TableTest, TestNewScanReadWriteCommitRejectANullContext) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); @@ -1028,9 +1149,6 @@ TEST(TableTest, TestOperatorsRefuseAnOptionThatContradictsTheTable) { CreateManagedTable(dir->GetFileSystem(), dir->Str(), identifier, /*branch=*/"main", {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); - // The table was loaded with `bucket` = 1, so every operator refuses a context that names - // another value for it: the table serves the options it was loaded with, and a caller that - // means to change them copies the table. { ScanContextBuilder scan_builder(dir->Str()); scan_builder.AddOption(Options::BUCKET, "2"); @@ -1060,7 +1178,6 @@ TEST(TableTest, TestOperatorsRefuseAnOptionThatContradictsTheTable) { "option 'bucket' is '2' in the context but '1' on the table"); } - // The copy made with the option in question agrees with the context, so it serves it. ASSERT_OK_AND_ASSIGN(std::shared_ptr
copied, table->Copy({{Options::BUCKET, "2"}})); ScanContextBuilder agreeing_scan_builder(dir->Str()); agreeing_scan_builder.AddOption(Options::BUCKET, "2"); @@ -1079,10 +1196,6 @@ TEST(TableTest, TestOperatorsIgnoreARuntimeKeyTheTableWasNotLoadedWith) { {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); WriteAndCommitTwoRows(table, dir->Str()); - // `scan.mode` and `scan.snapshot-id` name no option the table was loaded with, so they name - // a runtime setting the scan does not take: the scan serves the table's own options and - // reads its latest snapshot - a merged context would have gone looking for snapshot 999, - // which was never committed. ScanContextBuilder scan_builder(dir->Str()); scan_builder.AddOption(Options::SCAN_MODE, "from-snapshot") .AddOption(Options::SCAN_SNAPSHOT_ID, "999"); @@ -1092,8 +1205,6 @@ TEST(TableTest, TestOperatorsIgnoreARuntimeKeyTheTableWasNotLoadedWith) { ASSERT_FALSE(plan->Splits().empty()); ASSERT_EQ(plan->SnapshotId(), std::optional(1)); - // A read takes a runtime key the same way: the key names nothing the table carries, so the - // read is served rather than refused. ReadContextBuilder read_builder(dir->Str()); read_builder.AddOption(Options::SCAN_MODE, "from-snapshot") .AddOption(Options::SCAN_SNAPSHOT_ID, "999"); @@ -1101,9 +1212,6 @@ TEST(TableTest, TestOperatorsIgnoreARuntimeKeyTheTableWasNotLoadedWith) { ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr read, table->NewRead(std::move(read_ctx))); - // The runtime setting a caller means to serve is the copy's: `Table::Copy` derives the table - // whose options carry it, and that table's scan reads the snapshot it names - which is not - // there, as this table only ever committed snapshot 1. ASSERT_OK_AND_ASSIGN( std::shared_ptr
copied, table->Copy({{Options::SCAN_MODE, "from-snapshot"}, {Options::SCAN_SNAPSHOT_ID, "999"}})); @@ -1117,9 +1225,6 @@ TEST(TableTest, TestOperatorsIgnoreARuntimeKeyTheTableWasNotLoadedWith) { TEST(TableTest, TestCreateFailsWhenTheSchemaOptionsDoNotParse) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); - // The schema itself carries no validation, so a schema written with options no - // `CoreOptions` accepts surfaces when the store parses them - at the load, not at the first - // operator built on the table. ASSERT_OK_AND_ASSIGN( std::unique_ptr created, TableSchema::Create( @@ -1146,10 +1251,6 @@ TEST(TableTest, TestTheCacheAScanContextCarriesReachesTheScan) { {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); WriteAndCommitTwoRows(table, dir->Str()); - // The cache is a runtime setting the scan serves from the context rather than from the - // table's own options, so a scan built with one reads its manifests through it. A scan reads - // the manifest list of a snapshot on its executor's threads, so the counter the cache keeps - // is guarded rather than a plain field. auto cache = std::make_shared( std::map{{CacheKind::MANIFEST, 64 * 1024 * 1024}, {CacheKind::SNAPSHOT_LIVE_MANIFEST, 64 * 1024 * 1024}}); @@ -1172,8 +1273,6 @@ TEST(TableTest, TestCopyServesTheTableWithTheGivenOptionsOnTop) { CreateManagedTable(fs, dir->Str(), identifier, /*branch=*/"main", {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); - // Two rows are written and committed through the table's own write and commit, so the copy - // below has data to find. WriteContextBuilder write_builder(dir->Str(), "commit-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr write, @@ -1199,16 +1298,12 @@ TEST(TableTest, TestCopyServesTheTableWithTheGivenOptionsOnTop) { ASSERT_OK(commit->Commit(messages)); ASSERT_OK(write->Close()); - // The copy overlays what the call gives on the options the schema stored; the schema itself - // and the identity the table was opened with are copied as they are. ASSERT_OK_AND_ASSIGN(std::shared_ptr
copied, table->Copy({{Options::BUCKET, "2"}, {Options::TYPE, "format-table"}})); std::shared_ptr copied_schema = std::dynamic_pointer_cast(copied->LatestSchema()); ASSERT_TRUE(copied_schema != nullptr); EXPECT_EQ(copied_schema->Options().at(Options::BUCKET), "2"); - // `type` is structural and read from the schema alone, so the copy carries none of it: the - // option given at the call does not restyle a managed table. EXPECT_EQ(copied_schema->Options().count(Options::TYPE), 0); EXPECT_EQ(copied_schema->Options().at(Options::BUCKET_KEY), "id"); EXPECT_EQ(copied_schema->Id(), @@ -1216,12 +1311,10 @@ TEST(TableTest, TestCopyServesTheTableWithTheGivenOptionsOnTop) { EXPECT_EQ(copied->FullName(), table->FullName()); EXPECT_EQ(copied->GetRootPath(), table->GetRootPath()); EXPECT_EQ(copied->GetFileSystem(), table->GetFileSystem()); - // The original keeps the options it was loaded with. EXPECT_EQ( std::dynamic_pointer_cast(table->LatestSchema())->Options().at(Options::BUCKET), "1"); - // The copy is a table like the one it came from, and serves its data as one. ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_ctx, ScanContextBuilder(dir->Str()).Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, copied->NewScan(std::move(scan_ctx))); @@ -1238,14 +1331,6 @@ TEST(TableTest, TestCopyRefusesThePhysicalOptionsTheDataWasLaidOutWith) { /*branch=*/"main", {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); - // The data on the path was hashed by the `bucket-key` it was written with: a copy that - // renamed the key would prune by a hash its rows never matched, dropping them silently, so - // the change is refused - at `Copy` and at the load a context drives alike, which merges - // through `Copy` - the way the Java `checkImmutability` refuses it. The same goes for the - // other options that name the physical layout, which a key the schema does not carry - // changes as much as one it carries differently. `merge-engine`, `bucket`, - // `row-tracking.enabled` and `data-evolution.enabled` are the options the Java immutable - // set holds that stay servable here. ASSERT_NOK_WITH_MSG(table->Copy({{Options::BUCKET_KEY, "v"}}), "Change 'bucket-key' is not supported yet."); ASSERT_NOK_WITH_MSG(table->Copy({{Options::BUCKET_FUNCTION_TYPE, "hash"}}), @@ -1260,9 +1345,6 @@ TEST(TableTest, TestCopyRefusesThePhysicalOptionsTheDataWasLaidOutWith) { "Change 'bucket-key' is not supported yet."); } - // An option that says what the schema already says moves nothing, and a bucket count is - // the physical option a rescale legitimately rewrites, so a copy may move it. An - // `ignore-delete` may only be turned on, never back off. ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::shared_ptr
unchanged, table->Copy({{Options::BUCKET_KEY, "id"}, {Options::BUCKET, "1"}})); ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::shared_ptr
rescaled, @@ -1288,9 +1370,6 @@ TEST(TableTest, TestALoadNamesTheDatabaseThePathLaysOut) { auto schema = arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false), arrow::field("dt", arrow::utf8())}); - // A context that names no catalog knows the table by its directory: the directory above - // the table's names the database when it carries the `.db` suffix, the way the Java - // `identifierFromPath` reads it, so the table the load serves is named for both. const std::string table_dir = dir->Str() + "/my_db.db/my_table"; ASSERT_OK(fs->Mkdirs(table_dir)); SchemaManager schema_manager(fs, table_dir); @@ -1304,8 +1383,6 @@ TEST(TableTest, TestALoadNamesTheDatabaseThePathLaysOut) { ASSERT_EQ(table->FullName(), "my_db.my_table"); } - // A table sitting directly under the root, with no `.db` directory above it, names no - // database: the table is known by its directory alone, the way it was before. const std::string loose_table_dir = dir->Str() + "/loose_table"; ASSERT_OK(fs->Mkdirs(loose_table_dir)); SchemaManager loose_schema_manager(fs, loose_table_dir); diff --git a/src/paimon/rest/rest_catalog.cpp b/src/paimon/rest/rest_catalog.cpp index 167f3530c..ccc10351b 100644 --- a/src/paimon/rest/rest_catalog.cpp +++ b/src/paimon/rest/rest_catalog.cpp @@ -384,8 +384,6 @@ Result> RestCatalog::LoadTableSchema(const Identifier& i if (!supported) { return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); } - // Only the schema is read here and the table does not outlive the call, so it needs no - // catalog and this one hands over no ownership of itself. GlobalSystemTableContext context; context.fs = fs_; context.warehouse = warehouse_; @@ -428,8 +426,6 @@ Result> RestCatalog::LoadSystemTable( if (branch) { dynamic_options[Options::BRANCH] = branch.value(); } - // The system table reads the data table's own files, so it is served by that table's own - // credentials - with data tokens the only file system that can open them. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table_file_system, GetTableFileSystem(identifier)); return SystemTableLoader::Load(system_table_name.value(), table_file_system, data_table_path, @@ -437,9 +433,6 @@ Result> RestCatalog::LoadSystemTable( } Result> RestCatalog::GetTable(const Identifier& identifier) const { - // The global system tables of the "sys" database expose the catalog itself rather than any - // table the server stands behind, so they are served locally and never reach the managed-table - // load below, whose schema cast a read-only system table schema would not survive. if (CatalogUtils::IsSystemDatabase(identifier.GetDatabaseName())) { PAIMON_ASSIGN_OR_RAISE(bool supported, GlobalSystemTableLoader::IsSupported(identifier.GetObjectName(), @@ -447,9 +440,6 @@ Result> RestCatalog::GetTable(const Identifier& identifie if (!supported) { return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); } - // A global system table builds its rows lazily through this catalog, so the table handed - // back has to keep it alive or the pointer dangles at the first `BuildRows`. A catalog - // not owned by a `std::shared_ptr` cannot be kept, so it is refused by name. std::shared_ptr self = weak_from_this().lock(); if (self == nullptr) { return Status::Invalid( @@ -474,15 +464,11 @@ Result> RestCatalog::GetTable(const Identifier& identifie std::string table_path; PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, LoadTableSchema(identifier, &uuid, &table_path)); - // The schema the same response carried says what the table is: a format table is handed back - // as the `FormatTable` it is rather than refused as the managed table it is not. std::shared_ptr data_schema = std::dynamic_pointer_cast(schema); if (data_schema != nullptr) { PAIMON_ASSIGN_OR_RAISE(bool is_format_table, TableTypeDefine::IsFormatTable(data_schema->Options())); if (is_format_table) { - // A rest catalog holds the schema itself, so everything below the location is data. - // The catalog-wide file system, as a format table reads no metadata of its own. return FormatTable::Create(fs_, table_path, identifier, data_schema, /*location_carries_paimon_metadata=*/false, /*dynamic_options=*/{}); @@ -494,7 +480,6 @@ Result> RestCatalog::GetTable(const Identifier& identifie // tokens binds to this table. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table_file_system, GetTableFileSystem(identifier)); - // The branch the schema and path above were read on, which the table's managers then serve. PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); return FileStoreTable::Create( table_file_system, table_path, schema, identifier, uuid, diff --git a/src/paimon/rest/rest_catalog_test.cpp b/src/paimon/rest/rest_catalog_test.cpp index 6a8a202e7..78e1eb529 100644 --- a/src/paimon/rest/rest_catalog_test.cpp +++ b/src/paimon/rest/rest_catalog_test.cpp @@ -1070,8 +1070,6 @@ TEST_F(RestCatalogTest, SystemTableSchema) { requests_before = state_->get_table_requests; } ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog->GetTable(system_identifier)); - // A system table comes back as the read-only `SystemTable` it is, naming itself after the - // data table it belongs to. std::shared_ptr system_table = std::dynamic_pointer_cast(table); ASSERT_NE(nullptr, system_table); ASSERT_EQ("t1$options", system_table->Name()); @@ -1293,7 +1291,6 @@ TEST_F(RestCatalogTest, FormatTableIsLoadedInOneRequest) { // directory named "schema" there is a partition value, not metadata to skip. ASSERT_FALSE(table->LocationCarriesPaimonMetadata()); - // A managed table comes back through the same call, as the managed table it is. ASSERT_OK(CreateSampleTable(catalog.get(), Identifier("db1", "t1"))); ASSERT_OK_AND_ASSIGN(std::shared_ptr
managed, catalog->GetTable(Identifier("db1", "t1"))); @@ -1545,9 +1542,6 @@ TEST_F(RestCatalogTest, SystemDatabase) { } TEST_F(RestCatalogTest, GlobalSystemTableRetainsItsCatalog) { - // A global system table builds its rows lazily through the catalog that served it, so the - // table handed back keeps that catalog alive: reading it after the caller let its own - // reference go still answers, rather than reading through a catalog already destroyed. ASSERT_OK_AND_ASSIGN(std::unique_ptr created, CreateRestCatalog()); std::shared_ptr catalog = std::move(created); ASSERT_OK(catalog->CreateDatabase("db1", {}, /*ignore_if_exists=*/false)); @@ -1559,8 +1553,6 @@ TEST_F(RestCatalogTest, GlobalSystemTableRetainsItsCatalog) { catalog.reset(); ASSERT_FALSE(weak.expired()) << "the global system table does not retain its catalog"; - // The rows are built through the retained catalog, so they still answer once the caller's - // own reference is gone: the one table the catalog stands behind shows up in them. ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, ScanContextBuilder("sys/tables").Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(scan_context))); @@ -1579,10 +1571,6 @@ TEST_F(RestCatalogTest, GlobalSystemTableRetainsItsCatalog) { } TEST_F(RestCatalogTest, GlobalSystemTableWorksThroughThePublicCatalogFactory) { - // `Catalog::Create()` is the factory a caller outside paimon uses, and it hands back a - // `Catalog` rather than a `RestCatalog`. A global system table keeps its catalog alive - // through that pointer, so this path has to work too - a catalog whose shared ownership - // was not set up would fail here with the refusal naming the missing ownership. ASSERT_OK_AND_ASSIGN(std::shared_ptr catalog, Catalog::Create(kWarehouse, options_)); ASSERT_OK(catalog->CreateDatabase("db1", {}, /*ignore_if_exists=*/false)); ASSERT_OK(CreateSampleTable(catalog.get(), Identifier("db1", "t1"))); diff --git a/test/inte/read_inte_test.cpp b/test/inte/read_inte_test.cpp index 41e76b32d..5aa2f22c3 100644 --- a/test/inte/read_inte_test.cpp +++ b/test/inte/read_inte_test.cpp @@ -276,9 +276,6 @@ Result> ReadSystemTable( const std::string& system_table_path, const std::map& options, bool streaming_mode = false, const std::shared_ptr& predicate = nullptr, const std::vector& read_field_names = {}, bool read_next_plan = false) { - // A system table is not served by the operator entry points a path names any more: the loader - // hands the table back, and the table serves the scan and the read itself, the way a catalog's - // `GetTable` would. PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(options)); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr table, @@ -659,8 +656,6 @@ TEST(SystemTableReadInteTest, TestReadOptionsSystemTable) { ArrowSchemaRelease(&schema); std::string system_table_path = PathUtil::JoinPath(dir->Str(), "warehouse/db1.db/tbl1$options"); - // A system table comes from the catalog, and the operators from the table it hands back - - // the path alone no longer reaches it. ASSERT_OK_AND_ASSIGN(auto table, catalog->GetTable(Identifier("db1", "tbl1$options"))); ScanContextBuilder scan_context_builder(system_table_path); scan_context_builder.SetOptions(options); @@ -694,7 +689,6 @@ TEST(SystemTableReadInteTest, TestReadBranchOptionsSystemTable) { std::map options = {{Options::FILE_SYSTEM, "local"}, {Options::FILE_FORMAT, "parquet"}}; std::string system_table_path = table_path + "$branch_rt$options"; - // No catalog here: the loader is the path-side half of what a catalog's `GetTable` does. ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options)); ASSERT_OK_AND_ASSIGN(auto table, SystemTableLoader::LoadFromPath(core_options.GetFileSystem(), system_table_path, options)); @@ -1018,8 +1012,6 @@ TEST(SystemTableReadInteTest, TestReadOptimizedPrimaryKeyProjectionAndPredicateP std::shared_ptr predicate = PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"k", FieldType::INT, Literal(2)); - // A system table is not served by the operator entry points a path names any more: the loader - // hands the table back, and the table serves the scan and the read itself. ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options)); ASSERT_OK_AND_ASSIGN( std::shared_ptr table, @@ -1084,8 +1076,6 @@ TEST(SystemTableReadInteTest, TestReadOptimizedSystemTableNestedProjection) { /*partition_map=*/{}, /*bucket=*/0, {})); ASSERT_OK(WriteAndFullCompact(std::move(batch), /*commit_identifier=*/0, helper.get())); - // A system table is not served by the operator entry points a path names any more: the loader - // hands the table back, and the table serves the scan and the read itself. ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options)); ASSERT_OK_AND_ASSIGN( std::shared_ptr table, diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp index 1fd0ea9ab..7c4110257 100644 --- a/test/inte/realtime_write_inte_test.cpp +++ b/test/inte/realtime_write_inte_test.cpp @@ -2919,8 +2919,6 @@ TEST_F(RealtimeWriteInteTest, TestPkRealtimeReadOptimizedScanUnsupported) { ASSERT_OK_AND_ASSIGN(std::vector disk_rows, ReadRows()); ASSERT_EQ(rows, disk_rows); - // A system table is not served by the entry point a path names any more: the loader hands - // the table back, and the table refuses the scan itself. ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options_)); ASSERT_OK_AND_ASSIGN(std::shared_ptr table, SystemTableLoader::LoadFromPath(core_options.GetFileSystem(), From b07ebda990224b356cf6e328a0f42509be6e69c6 Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Wed, 28 Oct 2026 13:18:34 +0800 Subject: [PATCH 20/30] fix --- src/paimon/core/table/file_store.cpp | 45 ------------ src/paimon/core/table/file_store.h | 23 ++---- src/paimon/core/table/file_store_table.cpp | 23 +++--- src/paimon/core/table/file_store_table.h | 15 ++-- .../core/table/file_store_table_factory.cpp | 24 +++---- src/paimon/core/table/table_test.cpp | 72 +++++++++++++++---- 6 files changed, 91 insertions(+), 111 deletions(-) diff --git a/src/paimon/core/table/file_store.cpp b/src/paimon/core/table/file_store.cpp index 649a7f980..b48f8cb8c 100644 --- a/src/paimon/core/table/file_store.cpp +++ b/src/paimon/core/table/file_store.cpp @@ -47,15 +47,6 @@ namespace paimon { -namespace { - -bool IsFileSystemResolutionKey(const std::string& key) { - return key == Options::FILE_SYSTEM || key.rfind("fs.", 0) == 0 || key.rfind("s3.", 0) == 0 || - key.rfind("s3a.", 0) == 0; -} - -} // namespace - Result> FileStore::Create( const std::shared_ptr& file_system, const std::string& root_path, const std::shared_ptr& schema, const std::string& branch, @@ -94,42 +85,6 @@ Result> FileStore::Create( return store; } -bool FileStore::FileSystemResolutionMayChange( - const TableSchema& schema, const std::map& dynamic_options) { - const std::map& schema_options = schema.Options(); - for (const auto& [key, value] : dynamic_options) { - if (!IsFileSystemResolutionKey(key)) { - continue; - } - auto schema_option = schema_options.find(key); - if (schema_option == schema_options.end() || schema_option->second != value) { - return true; - } - } - return false; -} - -bool FileStore::FileSystemResolutionAgrees(const std::map& left, - const std::map& right) { - for (const auto& [key, value] : left) { - if (IsFileSystemResolutionKey(key)) { - auto right_option = right.find(key); - if (right_option == right.end() || right_option->second != value) { - return false; - } - } - } - for (const auto& [key, value] : right) { - if (IsFileSystemResolutionKey(key)) { - auto left_option = left.find(key); - if (left_option == left.end() || left_option->second != value) { - return false; - } - } - } - return true; -} - FileStore::FileStore(const std::shared_ptr& file_system, const std::string& root_path, const std::shared_ptr& schema, const std::string& branch, const std::shared_ptr& schema_manager, diff --git a/src/paimon/core/table/file_store.h b/src/paimon/core/table/file_store.h index 1dbdee7ef..7acc5ca38 100644 --- a/src/paimon/core/table/file_store.h +++ b/src/paimon/core/table/file_store.h @@ -66,10 +66,13 @@ class FileStore { /// the paths below are built for, and `GetOptions()` parses its options. Null when the /// caller had none, which the operators then report. /// @param branch The branch the store serves, normalized here. - /// @param schema_manager A schema manager to carry instead of building one: the one a context - /// dispatch read the schema through, whose cache the operators then serve. - /// @param snapshot_manager A snapshot manager to carry instead of building one. Null builds one - /// over `root_path` on `branch`. + /// @param schema_manager A schema manager to carry instead of building one: the one a table + /// asked for the branch it already serves hands over, whose resolution this store's + /// options therefore name as well. Null - what a load, a copy and a branch switch pass, + /// each resolving through a schema of its own - builds one over the resolved file + /// system, so no manager is ever left reading through a backend its table moved off. + /// @param snapshot_manager A snapshot manager to carry instead of building one, as above. Null + /// builds one over `root_path` on `branch`. /// @param fs_scheme_map The scheme-to-identifier map the caller's contexts resolve custom /// schemes through, kept so a copy resolves as this store does. static Result> Create( @@ -79,18 +82,6 @@ class FileStore { const std::shared_ptr& snapshot_manager, const std::map& fs_scheme_map); - /// Returns whether a copy's dynamic options change the file system its store resolves to. When - /// they do, the copy must not inherit the managers of the table it was copied from: they were - /// built over that table's resolution and would keep reading metadata through the backend the - /// copy left. - static bool FileSystemResolutionMayChange( - const TableSchema& schema, const std::map& dynamic_options); - - /// Returns whether two options maps resolve to the same file system. Every resolution key has - /// to agree, including a key missing on both sides. - static bool FileSystemResolutionAgrees(const std::map& left, - const std::map& right); - const std::string& CurrentBranch() const { return branch_; } diff --git a/src/paimon/core/table/file_store_table.cpp b/src/paimon/core/table/file_store_table.cpp index e93c8ae6d..725c42984 100644 --- a/src/paimon/core/table/file_store_table.cpp +++ b/src/paimon/core/table/file_store_table.cpp @@ -143,17 +143,16 @@ Result> FileStoreTable::Copy( PAIMON_RETURN_NOT_OK(SchemaValidation::CheckImmutableOptions(*loaded_schema, overridable)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merged_schema, loaded_schema->MergeOptions(overridable)); - const bool fs_resolution_may_change = - FileStore::FileSystemResolutionMayChange(*loaded_schema, overridable); std::map accumulated = dynamic_options_; for (const auto& [key, value] : overridable) { accumulated.insert_or_assign(key, value); } + // A copy resolves its file system from its own merged options, so it takes over no manager of + // the table it was copied from: the store builds both over the resolution the copy names. return FileStoreTable::Create(store_->GetSpecifiedFileSystem(), root_path_, std::shared_ptr(std::move(merged_schema)), identifier_, CatalogUuid().value_or(""), CurrentBranch(), - fs_resolution_may_change ? nullptr : GetSchemaManager(), - fs_resolution_may_change ? nullptr : GetSnapshotManager(), + /*schema_manager=*/nullptr, /*snapshot_manager=*/nullptr, store_->GetFsSchemeMap(), accumulated); } @@ -165,9 +164,12 @@ Result> FileStoreTable::SwitchToBranch(const std::string& GetSchemaManager(), GetSnapshotManager(), store_->GetFsSchemeMap(), dynamic_options_); } - auto schema_manager = std::make_shared(file_system_, root_path_, target_branch); + // The branch is read through the file system this table serves, and then served through the + // one its own schema resolves: the store builds the managers over that resolution, so none of + // this table's is carried over. + SchemaManager schema_manager(file_system_, root_path_, target_branch); PAIMON_ASSIGN_OR_RAISE(std::optional> latest, - schema_manager->Latest()); + schema_manager.Latest()); if (!latest) { return Status::Invalid(fmt::format("branch {} does not exist", target_branch)); } @@ -177,19 +179,12 @@ Result> FileStoreTable::SwitchToBranch(const std::string& } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr branch_schema, (*latest)->MergeOptions(overridable)); - const std::shared_ptr branch_schema_manager = - (store_->GetSpecifiedFileSystem() == nullptr && - (!store_->GetFsSchemeMap().empty() || - !FileStore::FileSystemResolutionAgrees( - branch_schema->Options(), std::static_pointer_cast(schema_)->Options()))) - ? nullptr - : schema_manager; PAIMON_ASSIGN_OR_RAISE(Identifier branch_identifier, CatalogUtils::BranchIdentifier(identifier_, target_branch)); return FileStoreTable::Create(store_->GetSpecifiedFileSystem(), root_path_, std::shared_ptr(std::move(branch_schema)), branch_identifier, CatalogUuid().value_or(""), target_branch, - branch_schema_manager, /*snapshot_manager=*/nullptr, + /*schema_manager=*/nullptr, /*snapshot_manager=*/nullptr, store_->GetFsSchemeMap(), /*dynamic_options=*/{}); } diff --git a/src/paimon/core/table/file_store_table.h b/src/paimon/core/table/file_store_table.h index 66e40ad59..20f811a5c 100644 --- a/src/paimon/core/table/file_store_table.h +++ b/src/paimon/core/table/file_store_table.h @@ -66,8 +66,8 @@ class FileStoreTable : public Table { /// @param branch The branch the table serves, which the managers below read and write on. /// @param schema_manager A schema manager to carry instead of building one. See /// `FileStore::Create`. - /// @param snapshot_manager A snapshot manager to carry instead of building one. `Copy` hands - /// its table's over so both share one. See `FileStore::Create`. + /// @param snapshot_manager A snapshot manager to carry instead of building one. See + /// `FileStore::Create`. /// @param fs_scheme_map The scheme-to-identifier map the caller's contexts resolve custom /// schemes through, which the store's options resolve through as well. `Copy` hands its /// table's map over. @@ -102,14 +102,17 @@ class FileStoreTable : public Table { return identifier_.GetFullName(); } - /// Returns a copy that serves `dynamic_options`. The file system, root path and identifier - /// stay as they are: they name this table, and an option does not rename it. + /// Returns a copy that serves `dynamic_options`. The root path and the identifier stay as they + /// are: they name this table, and an option does not rename it. The copy resolves the file + /// system the caller opened this table through, if any, together with its own merged options, + /// and builds its managers over that resolution rather than taking this table's over. Result> Copy( const std::map& dynamic_options) const override; /// Returns the table of `branch`, as Java Paimon's `switchToBranch` does. The branch switched - /// to is served as it is: its schema is read from its own directory, so this table's dynamic - /// options are not carried over. Switching to the branch this table already serves hands back a + /// to is served as it is: its schema is read from its own directory and its managers are built + /// over the file system that schema resolves, so neither this table's dynamic options nor its + /// managers are carried over. Switching to the branch this table already serves hands back a /// copy of it with the schema and the managers shared. A branch with no schema does not exist. Result> SwitchToBranch(const std::string& branch) const; diff --git a/src/paimon/core/table/file_store_table_factory.cpp b/src/paimon/core/table/file_store_table_factory.cpp index c0c5e7152..4e068d0a6 100644 --- a/src/paimon/core/table/file_store_table_factory.cpp +++ b/src/paimon/core/table/file_store_table_factory.cpp @@ -35,7 +35,6 @@ #include "paimon/core/options/table_type.h" #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/table_schema.h" -#include "paimon/core/table/file_store.h" #include "paimon/core/table/file_store_table.h" #include "paimon/core/table/system/system_table.h" #include "paimon/core/utils/branch_manager.h" @@ -68,10 +67,8 @@ Result> WithContextOptions( } Result> TableOfLoadedSchema( - const std::shared_ptr& file_system, const std::shared_ptr& explicit_file_system, const std::string& path, const std::string& branch, const std::map& options, - const std::shared_ptr& schema_manager, const std::shared_ptr& latest_schema, const std::optional& identifier, const std::string& uuid, const std::map& fs_scheme_map) { if (latest_schema != nullptr) { @@ -87,17 +84,14 @@ Result> TableOfLoadedSchema( } Identifier table_identifier = identifier.value_or(CatalogUtils::IdentifierFromPath(path, branch)); - std::shared_ptr manager = schema_manager; - if (manager != nullptr && latest_schema != nullptr && explicit_file_system == nullptr && - (!fs_scheme_map.empty() || - !FileStore::FileSystemResolutionAgrees(latest_schema->Options(), options))) { - manager = nullptr; - } + // The schema was read through a file system the context's options resolved, while the table is + // served through the one the schema's own options resolve: the store builds its managers over + // that resolution rather than taking over the reader's. PAIMON_ASSIGN_OR_RAISE( std::shared_ptr table, FileStoreTable::Create(explicit_file_system, path, latest_schema, table_identifier, uuid, - branch, manager, /*snapshot_manager=*/nullptr, fs_scheme_map, - /*dynamic_options=*/{})); + branch, /*schema_manager=*/nullptr, /*snapshot_manager=*/nullptr, + fs_scheme_map, /*dynamic_options=*/{})); return WithContextOptions(table, latest_schema, options); } @@ -107,19 +101,19 @@ Result> ManagedTableOf( const std::string& branch, const std::map& options, const std::optional& schema_json, const std::optional& identifier, const std::string& uuid, const std::map& fs_scheme_map) { - auto schema_manager = std::make_shared(file_system, path, branch); + SchemaManager schema_manager(file_system, path, branch); std::shared_ptr latest_schema; if (branch == BranchManager::DEFAULT_MAIN_BRANCH && schema_json) { PAIMON_ASSIGN_OR_RAISE(latest_schema, TableSchema::CreateFromJson(schema_json.value())); } else { PAIMON_ASSIGN_OR_RAISE(std::optional> latest, - schema_manager->Latest()); + schema_manager.Latest()); if (latest) { latest_schema = *latest; } } - return TableOfLoadedSchema(file_system, explicit_file_system, path, branch, options, - schema_manager, latest_schema, identifier, uuid, fs_scheme_map); + return TableOfLoadedSchema(explicit_file_system, path, branch, options, latest_schema, + identifier, uuid, fs_scheme_map); } Result> CatalogMainBranchSchemaJson( diff --git a/src/paimon/core/table/table_test.cpp b/src/paimon/core/table/table_test.cpp index 4f321f17b..e2261518d 100644 --- a/src/paimon/core/table/table_test.cpp +++ b/src/paimon/core/table/table_test.cpp @@ -162,8 +162,15 @@ TEST(TableTest, TestTableCarriesManagersOfItsBranch) { auto file_store_copy = std::dynamic_pointer_cast(copy); ASSERT_TRUE(file_store_copy != nullptr); EXPECT_EQ(file_store_copy->CurrentBranch(), "dev"); - EXPECT_EQ(file_store_copy->GetSchemaManager(), file_store_table->GetSchemaManager()); - EXPECT_EQ(file_store_copy->GetSnapshotManager(), file_store_table->GetSnapshotManager()); + // A copy builds its managers over its own resolution rather than taking this table's over, and + // so serves the same branch through managers of its own. + ASSERT_TRUE(file_store_copy->GetSchemaManager() != nullptr); + ASSERT_TRUE(file_store_copy->GetSnapshotManager() != nullptr); + EXPECT_NE(file_store_copy->GetSchemaManager(), file_store_table->GetSchemaManager()); + EXPECT_NE(file_store_copy->GetSnapshotManager(), file_store_table->GetSnapshotManager()); + EXPECT_EQ(file_store_copy->GetSnapshotManager()->Branch(), "dev"); + EXPECT_EQ(file_store_copy->GetSnapshotManager()->RootPath(), path); + EXPECT_EQ(file_store_copy->GetSnapshotManager()->Fs(), fs); } TEST(TableTest, TestCopyKeepsTheBranchTheTableServes) { @@ -487,7 +494,7 @@ TEST(TableTest, TestAFallbackReaderIgnoresReadContextOptionsAbsentFromTheTable) } } -TEST(TableTest, TestACopyThatMovesAnS3OptionRebuildsItsManagers) { +TEST(TableTest, TestACopyServesItsS3OptionThroughManagersOfItsOwn) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); const std::shared_ptr fs = dir->GetFileSystem(); @@ -501,18 +508,22 @@ TEST(TableTest, TestACopyThatMovesAnS3OptionRebuildsItsManagers) { auto file_store_table = std::dynamic_pointer_cast(table); ASSERT_TRUE(file_store_table != nullptr); - ASSERT_OK_AND_ASSIGN(std::shared_ptr
moved, - table->Copy({{"s3.endpoint", "http://new"}})); - auto moved_file_store_table = std::dynamic_pointer_cast(moved); - ASSERT_TRUE(moved_file_store_table != nullptr); - EXPECT_NE(moved_file_store_table->GetSchemaManager(), file_store_table->GetSchemaManager()); - EXPECT_NE(moved_file_store_table->GetSnapshotManager(), file_store_table->GetSnapshotManager()); - - ASSERT_OK_AND_ASSIGN(std::shared_ptr
kept, table->Copy({{"s3.endpoint", "http://old"}})); - auto kept_file_store_table = std::dynamic_pointer_cast(kept); - ASSERT_TRUE(kept_file_store_table != nullptr); - EXPECT_EQ(kept_file_store_table->GetSchemaManager(), file_store_table->GetSchemaManager()); - EXPECT_EQ(kept_file_store_table->GetSnapshotManager(), file_store_table->GetSnapshotManager()); + // An option that moves the file system the schema named and one that leaves it where it was + // both come back as a copy of their own: a copy never reads metadata through the managers of + // the table it was copied from, whichever resolution it ends up serving. + for (const std::string& endpoint : {std::string("http://new"), std::string("http://old")}) { + SCOPED_TRACE(endpoint); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
copy, table->Copy({{"s3.endpoint", endpoint}})); + auto copy_file_store_table = std::dynamic_pointer_cast(copy); + ASSERT_TRUE(copy_file_store_table != nullptr); + EXPECT_NE(copy_file_store_table->GetSchemaManager(), file_store_table->GetSchemaManager()); + EXPECT_NE(copy_file_store_table->GetSnapshotManager(), + file_store_table->GetSnapshotManager()); + EXPECT_EQ("main", copy_file_store_table->CurrentBranch()); + auto copy_schema = std::dynamic_pointer_cast(copy->LatestSchema()); + ASSERT_TRUE(copy_schema != nullptr); + EXPECT_EQ(endpoint, copy_schema->Options().at("s3.endpoint")); + } } TEST(TableTest, TestSwitchToBranch) { @@ -569,6 +580,37 @@ TEST(TableTest, TestSwitchToBranch) { ASSERT_NOK_WITH_MSG(main_file_store_table->SwitchToBranch("missing"), "branch"); } +TEST(TableTest, TestSwitchToBranchFromATableThatHasNoSchemaOfItsOwn) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::string path = dir->Str() + "/tbl"; + SchemaManager dev_schema_manager(dir->GetFileSystem(), path, "dev"); + ASSERT_OK_AND_ASSIGN( + [[maybe_unused]] std::unique_ptr dev_schema, + dev_schema_manager.CreateTable(arrow::schema({arrow::field("c0", arrow::int32())}), + /*partition_keys=*/{}, /*primary_keys=*/{}, {})); + + // A table opened over a path its own branch holds no schema under, and opened without a file + // system of its own: its operators report that no schema was found, while a branch that has one + // is read through the default file system the store falls back to and served on its own. + ASSERT_OK_AND_ASSIGN( + std::shared_ptr table, + FileStoreTable::Create(/*file_system=*/nullptr, path, /*schema=*/nullptr, + Identifier("db", "tbl"), /*uuid=*/"", + /*branch=*/BranchManager::DEFAULT_MAIN_BRANCH, + /*schema_manager=*/nullptr, /*snapshot_manager=*/nullptr, + /*fs_scheme_map=*/{}, /*dynamic_options=*/{})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
dev_table, table->SwitchToBranch("dev")); + auto dev_file_store_table = std::dynamic_pointer_cast(dev_table); + ASSERT_TRUE(dev_file_store_table != nullptr); + EXPECT_EQ("dev", dev_file_store_table->CurrentBranch()); + EXPECT_EQ("tbl$branch_dev", dev_table->Name()); + ASSERT_EQ((std::vector{"c0"}), dev_table->LatestSchema()->FieldNames()); + ASSERT_TRUE(dev_file_store_table->GetSchemaManager() != nullptr); + ASSERT_TRUE(dev_file_store_table->GetSnapshotManager() != nullptr); + EXPECT_EQ("dev", dev_file_store_table->GetSnapshotManager()->Branch()); +} + TEST(TableTest, TestNewWriteServesTheBranchTheTableWasLoadedWith) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); From 337811eb4c8868060b697e290b4e382f559d9e20 Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Wed, 28 Oct 2026 14:10:07 +0800 Subject: [PATCH 21/30] fix --- src/paimon/core/catalog/snapshot_commit.cpp | 22 +++++++- src/paimon/core/catalog/snapshot_commit.h | 9 ++- src/paimon/core/table/file_store_table.cpp | 61 ++------------------- 3 files changed, 30 insertions(+), 62 deletions(-) diff --git a/src/paimon/core/catalog/snapshot_commit.cpp b/src/paimon/core/catalog/snapshot_commit.cpp index 91e70a851..5d1ec7b86 100644 --- a/src/paimon/core/catalog/snapshot_commit.cpp +++ b/src/paimon/core/catalog/snapshot_commit.cpp @@ -20,10 +20,14 @@ #include #include +#include +#include "fmt/format.h" #include "paimon/core/catalog/catalog_snapshot_commit.h" #include "paimon/core/catalog/renaming_snapshot_commit.h" #include "paimon/core/catalog/version_managed_catalog.h" +#include "paimon/core/core_options.h" +#include "paimon/defs.h" #include "paimon/result.h" #include "paimon/status.h" @@ -31,17 +35,29 @@ namespace paimon { Result> SnapshotCommit::Create( const std::shared_ptr& catalog, const std::optional& catalog_identifier, - const std::optional& table_id, bool use_rest_catalog_commit, + const CoreOptions& options, const std::optional& table_id, + const std::string& commit_user, bool use_rest_catalog_commit, const std::shared_ptr& file_system, const std::shared_ptr& snapshot_manager) { if (AsVersionManaged(catalog) != nullptr) { if (!catalog_identifier) { return Status::Invalid("a catalog commit requires a table identifier"); } - return std::shared_ptr( - std::make_shared(catalog, catalog_identifier.value(), table_id)); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr lock, + AsVersionManaged(catalog) + ->CreateCommitLock(catalog_identifier.value(), table_id, commit_user, options)); + // Reuse the caller's UUID so table recreation cannot redirect this commit. + return std::shared_ptr(std::make_shared( + catalog, catalog_identifier.value(), table_id, std::move(lock))); } if (use_rest_catalog_commit) { + if (options.RestCommitAcquireLock()) { + return Status::Invalid(fmt::format( + "'{}' needs a commit through a REST catalog with WithCatalog(); a commit table " + "request built for the caller to send cannot hold a commit lease", + Options::REST_COMMIT_ACQUIRE_LOCK)); + } return std::shared_ptr(std::make_shared(table_id)); } if (table_id && catalog == nullptr) { diff --git a/src/paimon/core/catalog/snapshot_commit.h b/src/paimon/core/catalog/snapshot_commit.h index 58b8bbe9d..2062b6c2f 100644 --- a/src/paimon/core/catalog/snapshot_commit.h +++ b/src/paimon/core/catalog/snapshot_commit.h @@ -31,6 +31,7 @@ namespace paimon { class Catalog; +class CoreOptions; class FileSystem; class SnapshotManager; @@ -54,17 +55,19 @@ class SnapshotCommit { /// @param catalog The catalog the commit was handed, or null for one that has none. /// @param catalog_identifier The identifier the catalog knows the table by, on the branch the /// commit commits to. A version-managing catalog needs one and is refused without it. + /// @param options Table options merged with the writer's dynamic options. /// @param table_id The catalog's UUID for the table, captured before the changes were /// prepared; null when there is none. + /// @param commit_user The commit user of the snapshots published under a commit lock. /// @param use_rest_catalog_commit Whether the caller asks for the request rather than for the /// catalog to send it. /// @param file_system The file system the snapshot file is renamed through. /// @param snapshot_manager The manager that names the snapshot file renamed. static Result> Create( const std::shared_ptr& catalog, - const std::optional& catalog_identifier, - const std::optional& table_id, bool use_rest_catalog_commit, - const std::shared_ptr& file_system, + const std::optional& catalog_identifier, const CoreOptions& options, + const std::optional& table_id, const std::string& commit_user, + bool use_rest_catalog_commit, const std::shared_ptr& file_system, const std::shared_ptr& snapshot_manager); /// @param base_snapshot_uuid Base snapshot UUID; null for an absent or legacy snapshot. diff --git a/src/paimon/core/table/file_store_table.cpp b/src/paimon/core/table/file_store_table.cpp index 725c42984..f2d9f2546 100644 --- a/src/paimon/core/table/file_store_table.cpp +++ b/src/paimon/core/table/file_store_table.cpp @@ -37,8 +37,6 @@ #include "paimon/common/utils/sensitive_config_utils.h" #include "paimon/common/utils/string_utils.h" #include "paimon/core/catalog/catalog_utils.h" -#include "paimon/core/catalog/commit_lock.h" -#include "paimon/core/catalog/renaming_snapshot_commit.h" #include "paimon/core/catalog/snapshot_commit.h" #include "paimon/core/catalog/version_managed_catalog.h" #include "paimon/core/core_options.h" @@ -505,56 +503,6 @@ Result> FileStoreTable::NewRead( std::move(fallback_table_read), arrow_pool); } -Result> TableRead::Create(std::unique_ptr ctx) { - PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "read")); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, FileStoreTableFactory::Create(*ctx)); - return table->NewRead(std::move(ctx)); -} - -namespace { - -bool IsCatalogCommit(const CommitContext& ctx) { - return AsVersionManaged(ctx.GetCatalog()) != nullptr; -} - -Result> NewSnapshotCommit( - const CommitContext& ctx, const std::optional& catalog_identifier, - const CoreOptions& options, const std::shared_ptr& snapshot_manager) { - if (IsCatalogCommit(ctx)) { - if (!catalog_identifier) { - return Status::Invalid("a catalog commit requires a table identifier"); - } - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr lock, - AsVersionManaged(ctx.GetCatalog()) - ->CreateCommitLock(catalog_identifier.value(), ctx.GetTableId(), - ctx.GetCommitUser(), options)); - // Reuse the caller's UUID so table recreation cannot redirect this commit. - return std::shared_ptr(std::make_shared( - ctx.GetCatalog(), catalog_identifier.value(), ctx.GetTableId(), std::move(lock))); - } - if (ctx.UseRESTCatalogCommit()) { - if (options.RestCommitAcquireLock()) { - return Status::Invalid(fmt::format( - "'{}' needs a commit through a REST catalog with WithCatalog(); a commit table " - "request built for the caller to send cannot hold a commit lease", - Options::REST_COMMIT_ACQUIRE_LOCK)); - } - return std::shared_ptr( - std::make_shared(ctx.GetTableId())); - } - // Catalogs without version management may still supply a table ID. - if (ctx.GetTableId() && ctx.GetCatalog() == nullptr) { - return Status::Invalid( - "a commit which publishes the snapshot by writing it to the table directory sends no " - "commit table request, so there is nowhere to put a table id; commit through a " - "catalog with WithCatalog() or ask for the request with UseRESTCatalogCommit()"); - } - return std::shared_ptr( - std::make_shared(options.GetFileSystem(), snapshot_manager)); -} - -} // namespace - Result> FileStoreTable::NewCommit( std::unique_ptr ctx) const { PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "commit")); @@ -642,10 +590,11 @@ Result> FileStoreTable::NewCommit( NewCommitScanSupplier(snapshot_manager, manifest_list, manifest_file, arrow_schema, options, ctx->GetExecutor(), ctx->GetMemoryPool()); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr snapshot_commit, - SnapshotCommit::Create(ctx->GetCatalog(), catalog_identifier, options, - ctx->GetTableId(), ctx->UseRESTCatalogCommit(), - options.GetFileSystem(), snapshot_manager)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr snapshot_commit, + SnapshotCommit::Create(ctx->GetCatalog(), catalog_identifier, options, ctx->GetTableId(), + ctx->GetCommitUser(), ctx->UseRESTCatalogCommit(), + options.GetFileSystem(), snapshot_manager)); return std::make_unique( ctx->GetMemoryPool(), ctx->GetExecutor(), arrow_schema, root_path_, ctx->GetCommitUser(), From 5bf04dc28c1da632eae67c3adfa794f63cb7a9ab Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Wed, 28 Oct 2026 15:57:27 +0800 Subject: [PATCH 22/30] fix --- docs/source/user_guide/catalog.rst | 6 +- include/paimon/commit_context.h | 55 +++++- include/paimon/read_context.h | 7 + include/paimon/scan_context.h | 40 ++++- include/paimon/write_context.h | 29 ++++ src/paimon/core/catalog/snapshot_commit.cpp | 7 +- src/paimon/core/operation/commit_context.cpp | 61 +++++-- .../core/operation/commit_context_test.cpp | 37 +++- .../core/operation/file_store_commit_test.cpp | 4 +- .../core/operation/read_context_test.cpp | 1 + src/paimon/core/operation/scan_context.cpp | 50 ++++-- .../core/operation/scan_context_test.cpp | 60 +++++++ src/paimon/core/operation/write_context.cpp | 23 ++- .../core/operation/write_context_test.cpp | 27 +++ src/paimon/core/table/file_store_table.cpp | 3 +- .../core/table/file_store_table_factory.cpp | 163 ++++++------------ .../table/system/audit_log_system_table.cpp | 9 +- .../system/read_optimized_system_table.cpp | 9 +- .../core/table/system/system_table_test.cpp | 12 ++ src/paimon/core/table/table_test.cpp | 61 +++++++ 20 files changed, 508 insertions(+), 156 deletions(-) diff --git a/docs/source/user_guide/catalog.rst b/docs/source/user_guide/catalog.rst index fd9d16cf3..162799511 100644 --- a/docs/source/user_guide/catalog.rst +++ b/docs/source/user_guide/catalog.rst @@ -167,9 +167,9 @@ current snapshot. Both builders use the file system of the table being written or committed for manifests and data, so a catalog that issues temporary credentials per table serves them through ``Catalog::GetTableFileSystem``. ``WithFileSystem`` overrides -it; writers also allow ``WithFileSystemSchemeToIdentifierMap`` to override -file-system selection. A format table is written through the table's own -``NewWrite`` instead; see :doc:`format_table`. +it; both also allow ``WithFileSystemSchemeToIdentifierMap`` to route by URI +scheme instead, which the catalog is then not asked for. A format table is +written through the table's own ``NewWrite`` instead; see :doc:`format_table`. A file system passed to ``WithFileSystem`` is used exactly as it is: neither the built-in file systems nor the table's own credentials are involved, so a custom diff --git a/include/paimon/commit_context.h b/include/paimon/commit_context.h index c38cdca4c..688b10ff9 100644 --- a/include/paimon/commit_context.h +++ b/include/paimon/commit_context.h @@ -57,13 +57,26 @@ class PAIMON_EXPORT CommitContext { const std::shared_ptr& memory_pool, const std::shared_ptr& executor, const std::shared_ptr& specific_file_system, - const std::map& options); + const std::map& options, const std::string& branch, + const std::optional& table_schema = std::nullopt, + const std::map& fs_scheme_to_identifier_map = {}); ~CommitContext(); const std::string& GetRootPath() const { return root_path_; } + /// Returns the table root path. An alias of `GetRootPath()`, so that every context + /// exposes the table location under the same name. + const std::string& GetPath() const { + return GetRootPath(); + } + + /// Returns the branch the builder resolved out of the identifier and the options. + const std::string& GetBranch() const { + return branch_; + } + const std::string& GetCommitUser() const { return commit_user_; } @@ -76,6 +89,11 @@ class PAIMON_EXPORT CommitContext { return use_rest_catalog_commit_; } + /// Returns the table schema the caller set, or nullopt when the commit loads it. + const std::optional& GetSpecificTableSchema() const { + return table_schema_; + } + /// Returns the configured catalog, or null if unset. const std::shared_ptr& GetCatalog() const { return catalog_; @@ -107,12 +125,17 @@ class PAIMON_EXPORT CommitContext { return specific_file_system_; } + const std::map& GetFileSystemSchemeToIdentifierMap() const { + return fs_scheme_to_identifier_map_; + } + const std::map& GetOptions() const { return options_; } private: std::string root_path_; + std::string branch_; std::string commit_user_; bool ignore_empty_commit_; bool use_rest_catalog_commit_; @@ -123,7 +146,9 @@ class PAIMON_EXPORT CommitContext { std::shared_ptr memory_pool_; std::shared_ptr executor_; std::shared_ptr specific_file_system_; + std::map fs_scheme_to_identifier_map_; std::map options_; + std::optional table_schema_; }; /// `CommitContextBuilder` used to build a `CommitContext`, has input validation. @@ -218,6 +243,34 @@ class PAIMON_EXPORT CommitContextBuilder { /// @note If not set, use default file system (configured in `Options::FILE_SYSTEM`) CommitContextBuilder& WithFileSystem(const std::shared_ptr& file_system); + /// Sets a mapping from URI schemes (e.g., "file", "oss") to registered file system + /// identifiers. This allows selecting different pre-registered file system implementations + /// based on the URI scheme at runtime. + /// + /// @param fs_scheme_to_identifier_map Map from URI scheme (like "oss") to the corresponding + /// file system identifier. + /// @return Reference to this builder for method chaining. + /// @note + /// - This method is intended for environments where multiple file systems are pre-registered. + /// - The specified identifiers must correspond to file systems that have been registered at + /// compile time or initialization. + /// - Cannot be used together with `WithFileSystem()`. + /// - If not set, use default file system (configured in `Options::FILE_SYSTEM`). + /// Example: + /// builder.WithFileSystemSchemeToIdentifierMap({{"oss", "jindo"}, {"file", "local"}}); + /// + CommitContextBuilder& WithFileSystemSchemeToIdentifierMap( + const std::map& fs_scheme_to_identifier_map); + + /// Set the table schema as a string to avoid schema loading I/O operations. + /// + /// @param table_schema String representation of the table schema. + /// @return Reference to this builder for method chaining. + /// @note The user must ensure that the schema string is valid and matches the table. + /// @note If not set, the schema is read from the catalog of `WithCatalog()` on the main + /// branch, and loaded from the table path otherwise. + CommitContextBuilder& SetTableSchema(const std::string& table_schema); + /// Build and return a `CommitContext` instance with input validation. /// @return Result containing the constructed `CommitContext` or an error status. Result> Finish(); diff --git a/include/paimon/read_context.h b/include/paimon/read_context.h index 568315ef6..2ee575c4f 100644 --- a/include/paimon/read_context.h +++ b/include/paimon/read_context.h @@ -179,6 +179,13 @@ class PAIMON_EXPORT ReadContext { return identifier_; } + /// Returns the catalog table UUID, so that every context answers `GetTableId()` the way + /// `CommitContext` does; a read carries none. + const std::optional& GetTableId() const { + static const std::optional kNoTableId; + return kNoTableId; + } + /// Whether a read schema (C ArrowSchema) for nested column pruning was provided. bool HasReadSchema() const { return read_schema_ != nullptr && read_schema_->release != nullptr; diff --git a/include/paimon/scan_context.h b/include/paimon/scan_context.h index 99f5057eb..30f87baeb 100644 --- a/include/paimon/scan_context.h +++ b/include/paimon/scan_context.h @@ -57,7 +57,8 @@ class PAIMON_EXPORT ScanContext { const std::optional& table_schema, const std::map& options, const std::shared_ptr& cache, const std::shared_ptr& catalog, - const std::optional& identifier); + const std::optional& identifier, const std::string& branch, + const std::map& fs_scheme_to_identifier_map = {}); ~ScanContext(); @@ -65,6 +66,11 @@ class PAIMON_EXPORT ScanContext { return path_; } + /// Returns the branch the builder resolved out of the identifier and the options. + const std::string& GetBranch() const { + return branch_; + } + bool IsStreamingMode() const { return is_streaming_mode_; } @@ -100,6 +106,10 @@ class PAIMON_EXPORT ScanContext { return specific_file_system_; } + const std::map& GetFileSystemSchemeToIdentifierMap() const { + return fs_scheme_to_identifier_map_; + } + const std::optional& GetSpecificTableSchema() const { return table_schema_; } @@ -118,8 +128,16 @@ class PAIMON_EXPORT ScanContext { return identifier_; } + /// Returns the catalog table UUID, so that every context answers `GetTableId()` the way + /// `CommitContext` does; a scan carries none. + const std::optional& GetTableId() const { + static const std::optional kNoTableId; + return kNoTableId; + } + private: std::string path_; + std::string branch_; bool is_streaming_mode_; std::optional limit_; std::shared_ptr scan_filters_; @@ -128,6 +146,7 @@ class PAIMON_EXPORT ScanContext { std::shared_ptr memory_pool_; std::shared_ptr executor_; std::shared_ptr specific_file_system_; + std::map fs_scheme_to_identifier_map_; std::optional table_schema_; std::map options_; std::shared_ptr cache_; @@ -221,6 +240,25 @@ class PAIMON_EXPORT ScanContextBuilder { /// @note If not set, use default file system (configured in `Options::FILE_SYSTEM`) ScanContextBuilder& WithFileSystem(const std::shared_ptr& file_system); + /// Sets a mapping from URI schemes (e.g., "file", "oss") to registered file system + /// identifiers. This allows selecting different pre-registered file system implementations + /// based on the URI scheme at runtime. + /// + /// @param fs_scheme_to_identifier_map Map from URI scheme (like "oss") to the corresponding + /// file system identifier. + /// @return Reference to this builder for method chaining. + /// @note + /// - This method is intended for environments where multiple file systems are pre-registered. + /// - The specified identifiers must correspond to file systems that have been registered at + /// compile time or initialization. + /// - Cannot be used together with `WithFileSystem()`. + /// - If not set, use default file system (configured in `Options::FILE_SYSTEM`). + /// Example: + /// builder.WithFileSystemSchemeToIdentifierMap({{"oss", "jindo"}, {"file", "local"}}); + /// + ScanContextBuilder& WithFileSystemSchemeToIdentifierMap( + const std::map& fs_scheme_to_identifier_map); + /// Plans a native table through its own file system - including the per-table temporary /// credentials a catalog that issues them hands out through `Catalog::GetTableFileSystem`. /// This is a shorthand for `WithFileSystem(catalog->GetTableFileSystem(identifier))`; an diff --git a/include/paimon/write_context.h b/include/paimon/write_context.h index ba55f37e8..d34bfe257 100644 --- a/include/paimon/write_context.h +++ b/include/paimon/write_context.h @@ -52,6 +52,7 @@ class PAIMON_EXPORT WriteContext { const std::map& fs_scheme_to_identifier_map, const std::shared_ptr& realtime_context, const std::map& options, + const std::optional& table_schema = std::nullopt, const std::shared_ptr& catalog = nullptr, const std::optional& identifier = std::nullopt); @@ -61,6 +62,12 @@ class PAIMON_EXPORT WriteContext { return root_path_; } + /// Returns the table root path. An alias of `GetRootPath()`, so that every context + /// exposes the table location under the same name. + const std::string& GetPath() const { + return GetRootPath(); + } + const std::string& GetCommitUser() const { return commit_user_; } @@ -122,6 +129,11 @@ class PAIMON_EXPORT WriteContext { return realtime_context_; } + /// Returns the table schema the caller set, or nullopt when the write loads it. + const std::optional& GetSpecificTableSchema() const { + return table_schema_; + } + /// Returns the catalog supplying table metadata, or null if unset. const std::shared_ptr& GetCatalog() const { return catalog_; @@ -132,6 +144,13 @@ class PAIMON_EXPORT WriteContext { return identifier_; } + /// Returns the catalog table UUID, so that every context answers `GetTableId()` the way + /// `CommitContext` does; a write carries none. + const std::optional& GetTableId() const { + static const std::optional kNoTableId; + return kNoTableId; + } + private: std::string root_path_; std::string commit_user_; @@ -149,6 +168,7 @@ class PAIMON_EXPORT WriteContext { std::map fs_scheme_to_identifier_map_; std::shared_ptr realtime_context_; std::map options_; + std::optional table_schema_; std::shared_ptr catalog_; std::optional identifier_; }; @@ -281,6 +301,15 @@ class PAIMON_EXPORT WriteContextBuilder { WriteContextBuilder& WithFileSystemSchemeToIdentifierMap( const std::map& fs_scheme_to_identifier_map); + /// Set the table schema as a string to avoid schema loading I/O operations. + /// + /// @param table_schema String representation of the table schema. + /// @return Reference to this builder for method chaining. + /// @note The user must ensure that the schema string is valid and matches the table. + /// @note If not set, the schema is read from the catalog of `WithCatalog()` on the main + /// branch, and loaded from the table path otherwise. + WriteContextBuilder& SetTableSchema(const std::string& table_schema); + /// Set the thread number for write buffer spill operations. (default is 0) /// If <= 0, threading is disabled for spill IPC read/write. /// If > 0, sets arrow CPU thread pool capacity for spill operations. diff --git a/src/paimon/core/catalog/snapshot_commit.cpp b/src/paimon/core/catalog/snapshot_commit.cpp index 5d1ec7b86..c9fac382b 100644 --- a/src/paimon/core/catalog/snapshot_commit.cpp +++ b/src/paimon/core/catalog/snapshot_commit.cpp @@ -43,10 +43,9 @@ Result> SnapshotCommit::Create( if (!catalog_identifier) { return Status::Invalid("a catalog commit requires a table identifier"); } - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr lock, - AsVersionManaged(catalog) - ->CreateCommitLock(catalog_identifier.value(), table_id, commit_user, options)); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr lock, + AsVersionManaged(catalog)->CreateCommitLock( + catalog_identifier.value(), table_id, commit_user, options)); // Reuse the caller's UUID so table recreation cannot redirect this commit. return std::shared_ptr(std::make_shared( catalog, catalog_identifier.value(), table_id, std::move(lock))); diff --git a/src/paimon/core/operation/commit_context.cpp b/src/paimon/core/operation/commit_context.cpp index c632ac9c6..ba0713ec0 100644 --- a/src/paimon/core/operation/commit_context.cpp +++ b/src/paimon/core/operation/commit_context.cpp @@ -46,19 +46,24 @@ CommitContext::CommitContext(const std::string& root_path, const std::string& co : CommitContext(root_path, commit_user, ignore_empty_commit, use_rest_catalog_commit, /*catalog=*/nullptr, /*identifier=*/std::nullopt, /*table_id=*/std::nullopt, append_commit_check_conflict, memory_pool, executor, specific_file_system, - options) {} + options, + // Without an identifier the option is the only name a branch can have. + BranchManager::NormalizeBranch( + options.count(Options::BRANCH) ? options.at(Options::BRANCH) : ""), + /*table_schema=*/std::nullopt, /*fs_scheme_to_identifier_map=*/{}) {} -CommitContext::CommitContext(const std::string& root_path, const std::string& commit_user, - bool ignore_empty_commit, bool use_rest_catalog_commit, - const std::shared_ptr& catalog, - const std::optional& identifier, - const std::optional& table_id, - bool append_commit_check_conflict, - const std::shared_ptr& memory_pool, - const std::shared_ptr& executor, - const std::shared_ptr& specific_file_system, - const std::map& options) +CommitContext::CommitContext( + const std::string& root_path, const std::string& commit_user, bool ignore_empty_commit, + bool use_rest_catalog_commit, const std::shared_ptr& catalog, + const std::optional& identifier, const std::optional& table_id, + bool append_commit_check_conflict, const std::shared_ptr& memory_pool, + const std::shared_ptr& executor, + const std::shared_ptr& specific_file_system, + const std::map& options, const std::string& branch, + const std::optional& table_schema, + const std::map& fs_scheme_to_identifier_map) : root_path_(root_path), + branch_(branch), commit_user_(commit_user), ignore_empty_commit_(ignore_empty_commit), use_rest_catalog_commit_(use_rest_catalog_commit), @@ -69,7 +74,9 @@ CommitContext::CommitContext(const std::string& root_path, const std::string& co memory_pool_(memory_pool), executor_(executor), specific_file_system_(specific_file_system), - options_(options) {} + fs_scheme_to_identifier_map_(fs_scheme_to_identifier_map), + options_(options), + table_schema_(table_schema) {} CommitContext::~CommitContext() = default; @@ -87,6 +94,8 @@ class CommitContextBuilder::Impl { memory_pool_ = GetDefaultPool(); executor_ = CreateDefaultExecutor(); specific_file_system_.reset(); + fs_scheme_to_identifier_map_.clear(); + table_schema_ = std::nullopt; options_.clear(); } @@ -102,6 +111,8 @@ class CommitContextBuilder::Impl { std::shared_ptr memory_pool_ = GetDefaultPool(); std::shared_ptr executor_ = CreateDefaultExecutor(); std::shared_ptr specific_file_system_; + std::map fs_scheme_to_identifier_map_; + std::optional table_schema_; std::map options_; }; @@ -172,10 +183,25 @@ CommitContextBuilder& CommitContextBuilder::WithFileSystem( return *this; } +CommitContextBuilder& CommitContextBuilder::WithFileSystemSchemeToIdentifierMap( + const std::map& fs_scheme_to_identifier_map) { + impl_->fs_scheme_to_identifier_map_ = fs_scheme_to_identifier_map; + return *this; +} + +CommitContextBuilder& CommitContextBuilder::SetTableSchema(const std::string& table_schema) { + impl_->table_schema_ = table_schema; + return *this; +} + Result> CommitContextBuilder::Finish() { if (impl_->catalog_ == nullptr && impl_->identifier_) { return Status::Invalid("cannot commit through a null catalog"); } + if (impl_->specific_file_system_ && !impl_->fs_scheme_to_identifier_map_.empty()) { + return Status::Invalid( + "WithFileSystem() and WithFileSystemSchemeToIdentifierMap() cannot be used together"); + } if (impl_->identifier_) { // Before the branch is read out of the identifier: the branch of `tbl$branch_dev$options` // reads back as `dev`, so a commit built for such a name would be aimed at another branch. @@ -210,6 +236,14 @@ Result> CommitContextBuilder::Finish() { "a commit either goes to the catalog of WithCatalog() or is handed back as a request " "for the caller to send, which is what UseRESTCatalogCommit() asks for"); } + if (impl_->catalog_ != nullptr && impl_->specific_file_system_ == nullptr && + impl_->fs_scheme_to_identifier_map_.empty()) { + // A catalog issuing per-table temporary credentials only hands them out through + // GetTableFileSystem, so the commit uses the credentials of this table. A scheme map + // routes by scheme instead, so the catalog is not asked when one is set. + PAIMON_ASSIGN_OR_RAISE(impl_->specific_file_system_, + impl_->catalog_->GetTableFileSystem(impl_->identifier_.value())); + } PAIMON_ASSIGN_OR_RAISE(impl_->root_path_, PathUtil::NormalizePath(impl_->root_path_)); if (impl_->root_path_.empty()) { return Status::Invalid("root path is empty"); @@ -218,7 +252,8 @@ Result> CommitContextBuilder::Finish() { impl_->root_path_, impl_->commit_user_, impl_->ignore_empty_commit_, impl_->use_rest_catalog_commit_, impl_->catalog_, impl_->identifier_, impl_->table_id_, impl_->append_commit_check_conflict_, impl_->memory_pool_, impl_->executor_, - impl_->specific_file_system_, impl_->options_); + impl_->specific_file_system_, impl_->options_, branch, impl_->table_schema_, + impl_->fs_scheme_to_identifier_map_); impl_->Reset(); return ctx; } diff --git a/src/paimon/core/operation/commit_context_test.cpp b/src/paimon/core/operation/commit_context_test.cpp index d86a79149..bcbef8bcc 100644 --- a/src/paimon/core/operation/commit_context_test.cpp +++ b/src/paimon/core/operation/commit_context_test.cpp @@ -42,6 +42,8 @@ TEST(CommitContextTest, TestLegacyConstructor) { CommitContext context("table_root_path", "commit_user_1", false, true, true, pool, executor, fs, options); ASSERT_EQ(context.GetRootPath(), "table_root_path"); + // Without an identifier the option is the only name a branch can have. + ASSERT_EQ(context.GetBranch(), "main"); ASSERT_EQ(context.GetCommitUser(), "commit_user_1"); ASSERT_FALSE(context.IgnoreEmptyCommit()); ASSERT_TRUE(context.UseRESTCatalogCommit()); @@ -53,6 +55,7 @@ TEST(CommitContextTest, TestLegacyConstructor) { ASSERT_EQ(context.GetCatalog(), nullptr); ASSERT_FALSE(context.GetIdentifier()); ASSERT_FALSE(context.GetTableId()); + ASSERT_TRUE(context.GetFileSystemSchemeToIdentifierMap().empty()); } TEST(CommitContextTest, TestDefaultValue) { @@ -62,6 +65,7 @@ TEST(CommitContextTest, TestDefaultValue) { ASSERT_OK_AND_ASSIGN(auto expected_root_path, PathUtil::NormalizePath("table_root_path")); ASSERT_EQ(ctx->GetRootPath(), expected_root_path); + ASSERT_EQ(ctx->GetBranch(), "main"); ASSERT_EQ(ctx->GetCommitUser(), "commit_user_1"); ASSERT_TRUE(ctx->IgnoreEmptyCommit()); ASSERT_FALSE(ctx->UseRESTCatalogCommit()); @@ -73,6 +77,28 @@ TEST(CommitContextTest, TestDefaultValue) { ASSERT_TRUE(ctx->GetExecutor()); ASSERT_FALSE(ctx->GetSpecificFileSystem()); ASSERT_TRUE(ctx->GetOptions().empty()); + ASSERT_TRUE(ctx->GetFileSystemSchemeToIdentifierMap().empty()); + ASSERT_FALSE(ctx->GetSpecificTableSchema()); + ASSERT_FALSE(ctx->GetTableId()); +} + +TEST(CommitContextTest, TestFileSystemAndSchemeMapConflict) { + CommitContextBuilder builder("table_root_path", "commit_user_1"); + auto fs = std::make_shared(); + builder.WithFileSystem(fs); + builder.WithFileSystemSchemeToIdentifierMap({{"file", "local"}}); + ASSERT_NOK_WITH_MSG( + builder.Finish(), + "WithFileSystem() and WithFileSystemSchemeToIdentifierMap() cannot be used together"); +} + +TEST(CommitContextTest, TestSchemeMapWithoutFileSystem) { + CommitContextBuilder builder("table_root_path", "commit_user_1"); + builder.WithFileSystemSchemeToIdentifierMap({{"file", "local"}}); + ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish()); + std::map expected_fs_map = {{"file", "local"}}; + ASSERT_EQ(expected_fs_map, ctx->GetFileSystemSchemeToIdentifierMap()); + ASSERT_FALSE(ctx->GetSpecificFileSystem()); } TEST(CommitContextTest, TestSetContent) { @@ -89,6 +115,7 @@ TEST(CommitContextTest, TestSetContent) { .WithMemoryPool(memory_pool) .WithExecutor(executor) .WithFileSystem(fs) + .SetTableSchema("table-schema-json") .AddOption("key", "value") .Finish()); @@ -102,6 +129,8 @@ TEST(CommitContextTest, TestSetContent) { ASSERT_EQ(ctx->GetMemoryPool(), memory_pool); ASSERT_EQ(ctx->GetExecutor(), executor); ASSERT_EQ(ctx->GetSpecificFileSystem(), fs); + ASSERT_TRUE(ctx->GetSpecificTableSchema().has_value()); + ASSERT_EQ("table-schema-json", ctx->GetSpecificTableSchema().value()); std::map expected_options = {{"key", "value"}}; ASSERT_EQ(ctx->GetOptions(), expected_options); @@ -142,6 +171,8 @@ TEST(CommitContextTest, TestCatalogAddressesBranchByIdentifier) { auto branch_ctx, branch_builder.WithCatalog(catalog, Identifier("db1", "t1$branch_dev")).Finish()); ASSERT_EQ(branch_ctx->GetIdentifier().value(), Identifier("db1", "t1$branch_dev")); + // Finish() stores the resolved branch, so the factory reads it instead of resolving again. + ASSERT_EQ(branch_ctx->GetBranch(), "dev"); CommitContextBuilder agreeing_builder("table_root_path", "commit_user_1"); ASSERT_OK(agreeing_builder.WithCatalog(catalog, Identifier("db1", "t1$branch_dev")) .AddOption(Options::BRANCH, "dev") @@ -151,6 +182,7 @@ TEST(CommitContextTest, TestCatalogAddressesBranchByIdentifier) { auto constructed_ctx, constructed_builder.WithCatalog(catalog, Identifier("db1", "t1", "dev")).Finish()); ASSERT_EQ(constructed_ctx->GetIdentifier().value(), Identifier("db1", "t1$branch_dev")); + ASSERT_EQ(constructed_ctx->GetBranch(), "dev"); CommitContextBuilder main_builder("table_root_path", "commit_user_1"); ASSERT_OK(main_builder.WithCatalog(catalog, Identifier("db1", "t1$branch_main")) .AddOption(Options::BRANCH, "main") @@ -193,7 +225,9 @@ TEST(CommitContextTest, TestCatalogAddressesBranchByIdentifier) { TEST(CommitContextTest, TestBranch) { CommitContextBuilder file_system_builder("table_root_path", "commit_user_1"); - ASSERT_OK(file_system_builder.AddOption(Options::BRANCH, "dev").Finish()); + ASSERT_OK_AND_ASSIGN(auto fs_branch_ctx, + file_system_builder.AddOption(Options::BRANCH, "dev").Finish()); + ASSERT_EQ(fs_branch_ctx->GetBranch(), "dev"); CommitContextBuilder upper_main_builder("table_root_path", "commit_user_1"); ASSERT_OK_AND_ASSIGN(auto upper_main_ctx, @@ -207,6 +241,7 @@ TEST(CommitContextTest, TestBranch) { auto rest_ctx, rest_builder.UseRESTCatalogCommit(true).AddOption(Options::BRANCH, "dev").Finish()); ASSERT_TRUE(rest_ctx->UseRESTCatalogCommit()); + ASSERT_EQ(rest_ctx->GetBranch(), "dev"); ASSERT_EQ(rest_ctx->GetOptions().at(Options::BRANCH), "dev"); CommitContextBuilder rest_upper_main_builder("table_root_path", "commit_user_1"); ASSERT_NOK_WITH_MSG(rest_upper_main_builder.UseRESTCatalogCommit(true) diff --git a/src/paimon/core/operation/file_store_commit_test.cpp b/src/paimon/core/operation/file_store_commit_test.cpp index c67c761c5..732f83bd0 100644 --- a/src/paimon/core/operation/file_store_commit_test.cpp +++ b/src/paimon/core/operation/file_store_commit_test.cpp @@ -41,6 +41,7 @@ #include "paimon/core/operation/file_store_commit_impl.h" #include "paimon/core/schema/table_schema.h" #include "paimon/core/table/sink/commit_message_impl.h" +#include "paimon/core/utils/branch_manager.h" #include "paimon/core/utils/snapshot_manager.h" #include "paimon/defs.h" #include "paimon/executor.h" @@ -99,7 +100,8 @@ TEST(FileStoreCommitTest, TestCreateWithCatalogRequiresIdentifier) { dir->Str(), "commit_user", /*ignore_empty_commit=*/true, /*use_rest_catalog_commit=*/false, catalog, /*identifier=*/std::nullopt, /*table_id=*/std::nullopt, /*append_commit_check_conflict=*/false, GetDefaultPool(), - CreateDefaultExecutor(), /*specific_file_system=*/nullptr, options); + CreateDefaultExecutor(), /*specific_file_system=*/nullptr, options, + BranchManager::DEFAULT_MAIN_BRANCH); ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(commit_context)), "a catalog commit requires a table identifier"); } diff --git a/src/paimon/core/operation/read_context_test.cpp b/src/paimon/core/operation/read_context_test.cpp index 4fb5e00b4..fd1b1ebc5 100644 --- a/src/paimon/core/operation/read_context_test.cpp +++ b/src/paimon/core/operation/read_context_test.cpp @@ -116,6 +116,7 @@ TEST(ReadContextTest, TestDefaultValue) { ReadContextBuilder builder("table_root_path"); ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish()); ASSERT_EQ(ctx->GetPath(), "table_root_path"); + ASSERT_FALSE(ctx->GetTableId()); ASSERT_TRUE(ctx->GetMemoryPool()); ASSERT_TRUE(ctx->GetExecutor()); ASSERT_TRUE(ctx->GetReadFieldNames().empty()); diff --git a/src/paimon/core/operation/scan_context.cpp b/src/paimon/core/operation/scan_context.cpp index 3fd859c5b..5d1c7610c 100644 --- a/src/paimon/core/operation/scan_context.cpp +++ b/src/paimon/core/operation/scan_context.cpp @@ -22,6 +22,7 @@ #include "paimon/catalog/catalog.h" #include "paimon/common/utils/path_util.h" +#include "paimon/core/utils/branch_manager.h" #include "paimon/executor.h" #include "paimon/memory/memory_pool.h" #include "paimon/status.h" @@ -29,17 +30,22 @@ namespace paimon { class Predicate; -ScanContext::ScanContext( - const std::string& path, bool is_streaming_mode, std::optional limit, - const std::shared_ptr& scan_filter, - const std::shared_ptr& global_index_result, - const std::shared_ptr& realtime_context, - const std::shared_ptr& memory_pool, const std::shared_ptr& executor, - const std::shared_ptr& specific_file_system, - const std::optional& table_schema, - const std::map& options, const std::shared_ptr& cache, - const std::shared_ptr& catalog, const std::optional& identifier) +ScanContext::ScanContext(const std::string& path, bool is_streaming_mode, + std::optional limit, + const std::shared_ptr& scan_filter, + const std::shared_ptr& global_index_result, + const std::shared_ptr& realtime_context, + const std::shared_ptr& memory_pool, + const std::shared_ptr& executor, + const std::shared_ptr& specific_file_system, + const std::optional& table_schema, + const std::map& options, + const std::shared_ptr& cache, + const std::shared_ptr& catalog, + const std::optional& identifier, const std::string& branch, + const std::map& fs_scheme_to_identifier_map) : path_(path), + branch_(branch), is_streaming_mode_(is_streaming_mode), limit_(limit), scan_filters_(scan_filter), @@ -48,6 +54,7 @@ ScanContext::ScanContext( memory_pool_(memory_pool), executor_(executor), specific_file_system_(specific_file_system), + fs_scheme_to_identifier_map_(fs_scheme_to_identifier_map), table_schema_(table_schema), options_(options), cache_(cache), @@ -71,6 +78,7 @@ class ScanContextBuilder::Impl { memory_pool_ = GetDefaultPool(); executor_.reset(); specific_file_system_.reset(); + fs_scheme_to_identifier_map_.clear(); table_schema_ = std::nullopt; options_.clear(); cache_.reset(); @@ -91,6 +99,7 @@ class ScanContextBuilder::Impl { // Resolved in Finish(); a builder never owns an executor of its own. std::shared_ptr executor_; std::shared_ptr specific_file_system_; + std::map fs_scheme_to_identifier_map_; std::optional table_schema_; std::map options_; std::shared_ptr cache_; @@ -171,6 +180,12 @@ ScanContextBuilder& ScanContextBuilder::WithFileSystem( return *this; } +ScanContextBuilder& ScanContextBuilder::WithFileSystemSchemeToIdentifierMap( + const std::map& fs_scheme_to_identifier_map) { + impl_->fs_scheme_to_identifier_map_ = fs_scheme_to_identifier_map; + return *this; +} + ScanContextBuilder& ScanContextBuilder::WithCatalog(const std::shared_ptr& catalog, const Identifier& identifier) { impl_->catalog_ = catalog; @@ -196,12 +211,21 @@ Result> ScanContextBuilder::Finish() { if (impl_->catalog_ == nullptr && impl_->identifier_) { return Status::Invalid("cannot scan through a null catalog"); } - if (impl_->catalog_ != nullptr && impl_->specific_file_system_ == nullptr) { + if (impl_->specific_file_system_ && !impl_->fs_scheme_to_identifier_map_.empty()) { + return Status::Invalid( + "WithFileSystem() and WithFileSystemSchemeToIdentifierMap() cannot be used together"); + } + if (impl_->catalog_ != nullptr && impl_->specific_file_system_ == nullptr && + impl_->fs_scheme_to_identifier_map_.empty()) { // A catalog issuing per-table temporary credentials only hands them out through - // GetTableFileSystem, so the scan uses the credentials of this table. + // GetTableFileSystem, so the scan uses the credentials of this table. A scheme map + // routes by scheme instead, so the catalog is not asked when one is set. PAIMON_ASSIGN_OR_RAISE(impl_->specific_file_system_, impl_->catalog_->GetTableFileSystem(impl_->identifier_.value())); } + PAIMON_ASSIGN_OR_RAISE(std::string branch, + BranchManager::ResolveBranch(impl_->identifier_, impl_->options_, + /*explicit_branch=*/std::nullopt, "scan")); std::shared_ptr executor = impl_->executor_ ? impl_->executor_ : CreateDefaultExecutor(); auto ctx = std::make_unique( @@ -210,7 +234,7 @@ Result> ScanContextBuilder::Finish() { impl_->bucket_filter_), impl_->global_index_result_, impl_->realtime_context_, impl_->memory_pool_, executor, impl_->specific_file_system_, impl_->table_schema_, impl_->options_, impl_->cache_, - impl_->catalog_, impl_->identifier_); + impl_->catalog_, impl_->identifier_, branch, impl_->fs_scheme_to_identifier_map_); impl_->Reset(); return ctx; } diff --git a/src/paimon/core/operation/scan_context_test.cpp b/src/paimon/core/operation/scan_context_test.cpp index 4ce2395b3..7a0e9e1bc 100644 --- a/src/paimon/core/operation/scan_context_test.cpp +++ b/src/paimon/core/operation/scan_context_test.cpp @@ -70,16 +70,75 @@ TEST(ScanContextTest, TestWithFileSystemOverridesCatalog) { ASSERT_TRUE(catalog->TableFileSystemRequests().empty()); } +TEST(ScanContextTest, TestFileSystemAndSchemeMapConflict) { + ScanContextBuilder builder("table_root_path"); + auto fs = std::make_shared(); + builder.WithFileSystem(fs); + builder.WithFileSystemSchemeToIdentifierMap({{"file", "local"}}); + ASSERT_NOK_WITH_MSG( + builder.Finish(), + "WithFileSystem() and WithFileSystemSchemeToIdentifierMap() cannot be used together"); +} + +TEST(ScanContextTest, TestSchemeMapWithoutFileSystem) { + ScanContextBuilder builder("table_root_path"); + builder.WithFileSystemSchemeToIdentifierMap({{"file", "local"}}); + ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish()); + std::map expected_fs_map = {{"file", "local"}}; + ASSERT_EQ(expected_fs_map, ctx->GetFileSystemSchemeToIdentifierMap()); + ASSERT_FALSE(ctx->GetSpecificFileSystem()); +} + +TEST(ScanContextTest, TestSchemeMapOverridesCatalogFileSystem) { + // A scheme map routes by scheme instead of a per-table file system, so the catalog is not + // asked for one when a map is set. + auto table_fs = std::make_shared(); + auto catalog = std::make_shared(); + catalog->SetTableFileSystem(table_fs); + + ScanContextBuilder builder("table_root_path"); + ASSERT_OK_AND_ASSIGN(auto ctx, builder.WithCatalog(catalog, Identifier("db1", "t1")) + .WithFileSystemSchemeToIdentifierMap({{"file", "local"}}) + .Finish()); + ASSERT_FALSE(ctx->GetSpecificFileSystem()); + std::map expected_fs_map = {{"file", "local"}}; + ASSERT_EQ(expected_fs_map, ctx->GetFileSystemSchemeToIdentifierMap()); + ASSERT_TRUE(catalog->TableFileSystemRequests().empty()); +} + TEST(ScanContextTest, TestWithCatalogNullRejected) { ScanContextBuilder builder("table_root_path"); ASSERT_NOK_WITH_MSG(builder.WithCatalog(nullptr, Identifier("db1", "t1")).Finish(), "cannot scan through a null catalog"); } +TEST(ScanContextTest, TestBranchResolution) { + // Finish() resolves the branch out of the identifier and the options and stores it, so the + // factory reads a resolved branch instead of resolving it on the spot. + ScanContextBuilder builder("table_root_path"); + ASSERT_OK_AND_ASSIGN(auto option_ctx, builder.AddOption(Options::BRANCH, "dev").Finish()); + ASSERT_EQ("dev", option_ctx->GetBranch()); + + auto catalog = std::make_shared(); + ASSERT_OK_AND_ASSIGN( + auto identifier_ctx, + builder.WithCatalog(catalog, Identifier("db1", "t1$branch_data")).Finish()); + ASSERT_EQ("data", identifier_ctx->GetBranch()); + + // Names which disagree are refused rather than silently chosen between. + ScanContextBuilder mismatch_builder("table_root_path"); + ASSERT_NOK_WITH_MSG(mismatch_builder.WithCatalog(catalog, Identifier("db1", "t1$branch_data")) + .AddOption(Options::BRANCH, "rt") + .Finish(), + "a scan is aimed at one branch, but both 'data' and 'rt' were named"); +} + TEST(ScanContextTest, TestDefaultValue) { ScanContextBuilder builder("table_root_path"); ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish()); ASSERT_EQ(ctx->GetPath(), "table_root_path"); + ASSERT_EQ("main", ctx->GetBranch()); + ASSERT_FALSE(ctx->GetTableId()); ASSERT_FALSE(ctx->IsStreamingMode()); ASSERT_FALSE(ctx->GetLimit()); ASSERT_TRUE(ctx->GetMemoryPool()); @@ -91,6 +150,7 @@ TEST(ScanContextTest, TestDefaultValue) { ASSERT_TRUE(ctx->GetOptions().empty()); ASSERT_FALSE(ctx->GetGlobalIndexResult()); ASSERT_FALSE(ctx->GetSpecificFileSystem()); + ASSERT_TRUE(ctx->GetFileSystemSchemeToIdentifierMap().empty()); } TEST(ScanContextTest, TestSetContent) { diff --git a/src/paimon/core/operation/write_context.cpp b/src/paimon/core/operation/write_context.cpp index 2b890f934..fbcbfe368 100644 --- a/src/paimon/core/operation/write_context.cpp +++ b/src/paimon/core/operation/write_context.cpp @@ -22,6 +22,7 @@ #include "arrow/util/thread_pool.h" #include "fmt/format.h" +#include "paimon/catalog/catalog.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/path_util.h" #include "paimon/core/catalog/catalog_utils.h" @@ -42,7 +43,8 @@ WriteContext::WriteContext( const std::shared_ptr& specific_file_system, const std::map& fs_scheme_to_identifier_map, const std::shared_ptr& realtime_context, - const std::map& options, const std::shared_ptr& catalog, + const std::map& options, + const std::optional& table_schema, const std::shared_ptr& catalog, const std::optional& identifier) : root_path_(root_path), commit_user_(commit_user), @@ -60,6 +62,7 @@ WriteContext::WriteContext( fs_scheme_to_identifier_map_(fs_scheme_to_identifier_map), realtime_context_(realtime_context), options_(options), + table_schema_(table_schema), catalog_(catalog), identifier_(identifier) {} @@ -85,6 +88,7 @@ class WriteContextBuilder::Impl { realtime_context_.reset(); catalog_.reset(); identifier_.reset(); + table_schema_ = std::nullopt; options_.clear(); } @@ -108,6 +112,7 @@ class WriteContextBuilder::Impl { std::shared_ptr realtime_context_; std::shared_ptr catalog_; std::optional identifier_; + std::optional table_schema_; std::map options_; }; @@ -190,6 +195,11 @@ WriteContextBuilder& WriteContextBuilder::SetWriteBufferSpillThreadNumber(int32_ return *this; } +WriteContextBuilder& WriteContextBuilder::SetTableSchema(const std::string& table_schema) { + impl_->table_schema_ = table_schema; + return *this; +} + WriteContextBuilder& WriteContextBuilder::WithFileSystem( const std::shared_ptr& file_system) { impl_->specific_file_system_ = file_system; @@ -239,6 +249,14 @@ Result> WriteContextBuilder::Finish() { branch, table_name, branch)); } } + if (impl_->catalog_ != nullptr && impl_->specific_file_system_ == nullptr && + impl_->fs_scheme_to_identifier_map_.empty()) { + // A catalog issuing per-table temporary credentials only hands them out through + // GetTableFileSystem, so the write uses the credentials of this table. A scheme map + // routes by scheme instead, so the catalog is not asked when one is set. + PAIMON_ASSIGN_OR_RAISE(impl_->specific_file_system_, + impl_->catalog_->GetTableFileSystem(impl_->identifier_.value())); + } bool enable_multi_thread_spill = impl_->spill_thread_number_ > 0; if (enable_multi_thread_spill) { PAIMON_RETURN_NOT_OK_FROM_ARROW( @@ -249,7 +267,8 @@ Result> WriteContextBuilder::Finish() { impl_->ignore_num_bucket_check_, impl_->ignore_previous_files_, enable_multi_thread_spill, impl_->write_id_, branch, impl_->write_schema_, impl_->memory_pool_, impl_->executor_, impl_->temp_directory_, impl_->specific_file_system_, impl_->fs_scheme_to_identifier_map_, - impl_->realtime_context_, impl_->options_, impl_->catalog_, impl_->identifier_); + impl_->realtime_context_, impl_->options_, impl_->table_schema_, impl_->catalog_, + impl_->identifier_); impl_->Reset(); return ctx; } diff --git a/src/paimon/core/operation/write_context_test.cpp b/src/paimon/core/operation/write_context_test.cpp index 5e777c351..a0137bf59 100644 --- a/src/paimon/core/operation/write_context_test.cpp +++ b/src/paimon/core/operation/write_context_test.cpp @@ -51,8 +51,10 @@ TEST(WriteContextTest, TestDefaultValue) { ASSERT_TRUE(ctx->GetOptions().empty()); ASSERT_TRUE(ctx->GetFileSystemSchemeToIdentifierMap().empty()); ASSERT_FALSE(ctx->GetSpecificFileSystem()); + ASSERT_FALSE(ctx->GetSpecificTableSchema()); ASSERT_FALSE(ctx->GetCatalog()); ASSERT_FALSE(ctx->GetIdentifier()); + ASSERT_FALSE(ctx->GetTableId()); } TEST(WriteContextTest, TestWithCatalog) { @@ -69,6 +71,28 @@ TEST(WriteContextTest, TestWithCatalog) { "cannot write through a null catalog"); } +TEST(WriteContextTest, TestWithCatalogResolvesTableFileSystem) { + // WithCatalog resolves the per-table file system when Finish() builds the context, which the + // factory then reads instead of asking the catalog again. + auto table_fs = std::make_shared(); + auto catalog = std::make_shared(); + catalog->SetTableFileSystem(table_fs); + + WriteContextBuilder builder("table_root_path", "commit_user_1"); + ASSERT_OK_AND_ASSIGN(auto ctx, builder.WithCatalog(catalog, Identifier("db1", "t1")).Finish()); + ASSERT_EQ(ctx->GetSpecificFileSystem(), table_fs); + ASSERT_EQ(catalog->TableFileSystemRequests().size(), 1U); + ASSERT_EQ(catalog->TableFileSystemRequests().front(), Identifier("db1", "t1")); + + // A scheme map routes by scheme instead, so the catalog is not asked for a file system. + WriteContextBuilder map_builder("table_root_path", "commit_user_1"); + ASSERT_OK_AND_ASSIGN(auto map_ctx, map_builder.WithCatalog(catalog, Identifier("db1", "t1")) + .WithFileSystemSchemeToIdentifierMap({{"file", "local"}}) + .Finish()); + ASSERT_FALSE(map_ctx->GetSpecificFileSystem()); + ASSERT_EQ(catalog->TableFileSystemRequests().size(), 1U); +} + TEST(WriteContextTest, TestCatalogAddressesBranchByIdentifier) { auto catalog = std::make_shared(); WriteContextBuilder builder("table_root_path", "commit_user_1"); @@ -172,6 +196,7 @@ TEST(WriteContextTest, TestSetContent) { .WithWriteSchema(write_schema) .WithFileSystemSchemeToIdentifierMap(fs_scheme_to_identifier_map) .WithFileSystem(file_system) + .SetTableSchema("table-schema-json") .AddOption("key", "value") .Finish()); @@ -187,6 +212,8 @@ TEST(WriteContextTest, TestSetContent) { ASSERT_EQ(ctx->GetWriteSchema(), write_schema); ASSERT_EQ(ctx->GetFileSystemSchemeToIdentifierMap(), fs_scheme_to_identifier_map); ASSERT_EQ(ctx->GetSpecificFileSystem(), file_system); + ASSERT_TRUE(ctx->GetSpecificTableSchema().has_value()); + ASSERT_EQ("table-schema-json", ctx->GetSpecificTableSchema().value()); std::map expected_options = {{"key", "value"}}; ASSERT_EQ(expected_options, ctx->GetOptions()); } diff --git a/src/paimon/core/table/file_store_table.cpp b/src/paimon/core/table/file_store_table.cpp index f2d9f2546..7d5153956 100644 --- a/src/paimon/core/table/file_store_table.cpp +++ b/src/paimon/core/table/file_store_table.cpp @@ -513,7 +513,8 @@ Result> FileStoreTable::NewCommit( } PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog())); std::shared_ptr specific_fs = ctx->GetSpecificFileSystem(); - if (specific_fs == nullptr && ctx->GetCatalog() != nullptr && !ctx->GetIdentifier()) { + if (specific_fs == nullptr && ctx->GetFileSystemSchemeToIdentifierMap().empty() && + ctx->GetCatalog() != nullptr && !ctx->GetIdentifier()) { return Status::Invalid("a catalog commit requires a table identifier"); } PAIMON_ASSIGN_OR_RAISE(std::optional catalog_identifier, diff --git a/src/paimon/core/table/file_store_table_factory.cpp b/src/paimon/core/table/file_store_table_factory.cpp index 4e068d0a6..79bada00e 100644 --- a/src/paimon/core/table/file_store_table_factory.cpp +++ b/src/paimon/core/table/file_store_table_factory.cpp @@ -50,27 +50,23 @@ namespace paimon { namespace { -Result> WithContextOptions( - const std::shared_ptr
& table, const std::shared_ptr& schema, - const std::map& options) { - if (table == nullptr || schema == nullptr || options.empty()) { - return table; - } - const std::map& schema_options = schema->Options(); - for (const auto& [key, value] : options) { - auto schema_option = schema_options.find(key); - if (schema_option == schema_options.end() || schema_option->second != value) { - return table->Copy(options); - } - } - return table; -} - -Result> TableOfLoadedSchema( +Result> ManagedTableOf( + const std::shared_ptr& file_system, const std::shared_ptr& explicit_file_system, const std::string& path, const std::string& branch, const std::map& options, - const std::shared_ptr& latest_schema, const std::optional& identifier, + const std::optional& schema_json, const std::optional& identifier, const std::string& uuid, const std::map& fs_scheme_map) { + SchemaManager schema_manager(file_system, path, branch); + std::shared_ptr latest_schema; + if (branch == BranchManager::DEFAULT_MAIN_BRANCH && schema_json) { + PAIMON_ASSIGN_OR_RAISE(latest_schema, TableSchema::CreateFromJson(schema_json.value())); + } else { + PAIMON_ASSIGN_OR_RAISE(std::optional> latest, + schema_manager.Latest()); + if (latest) { + latest_schema = *latest; + } + } if (latest_schema != nullptr) { PAIMON_ASSIGN_OR_RAISE(bool is_format_table, TableTypeDefine::IsFormatTable(latest_schema->Options())); @@ -92,28 +88,19 @@ Result> TableOfLoadedSchema( FileStoreTable::Create(explicit_file_system, path, latest_schema, table_identifier, uuid, branch, /*schema_manager=*/nullptr, /*snapshot_manager=*/nullptr, fs_scheme_map, /*dynamic_options=*/{})); - return WithContextOptions(table, latest_schema, options); -} - -Result> ManagedTableOf( - const std::shared_ptr& file_system, - const std::shared_ptr& explicit_file_system, const std::string& path, - const std::string& branch, const std::map& options, - const std::optional& schema_json, const std::optional& identifier, - const std::string& uuid, const std::map& fs_scheme_map) { - SchemaManager schema_manager(file_system, path, branch); - std::shared_ptr latest_schema; - if (branch == BranchManager::DEFAULT_MAIN_BRANCH && schema_json) { - PAIMON_ASSIGN_OR_RAISE(latest_schema, TableSchema::CreateFromJson(schema_json.value())); - } else { - PAIMON_ASSIGN_OR_RAISE(std::optional> latest, - schema_manager.Latest()); - if (latest) { - latest_schema = *latest; + // Fold in the context options, but only where one differs from the schema's own: a table is + // the product of its load, and `Copy` is the only way to restyle it. + if (table == nullptr || latest_schema == nullptr || options.empty()) { + return table; + } + const std::map& schema_options = latest_schema->Options(); + for (const auto& [key, value] : options) { + auto schema_option = schema_options.find(key); + if (schema_option == schema_options.end() || schema_option->second != value) { + return table->Copy(options); } } - return TableOfLoadedSchema(explicit_file_system, path, branch, options, latest_schema, - identifier, uuid, fs_scheme_map); + return table; } Result> CatalogMainBranchSchemaJson( @@ -134,97 +121,53 @@ Result> CatalogMainBranchSchemaJson( return std::optional(); } -} // namespace - -Result> FileStoreTableFactory::Create(const ScanContext& ctx) { +template +Result> ManagedTable(const Context& ctx, const char* kind) { + PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx.GetCatalog())); PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, - CoreOptions::FromMap(ctx.GetOptions(), ctx.GetSpecificFileSystem(), {})); + CoreOptions::FromMap(ctx.GetOptions(), ctx.GetSpecificFileSystem(), + ctx.GetFileSystemSchemeToIdentifierMap())); PAIMON_ASSIGN_OR_RAISE(std::optional system_table_path, SystemTableLoader::TryParsePath(ctx.GetPath())); if (system_table_path) { return Status::Invalid(fmt::format( "the path {} names a system table, which is not loaded from a path: a catalog " "hands the table back through GetTable, and its operators come from " - "table->NewScan/NewRead on it", + "table->NewScan/NewRead/NewWrite/NewCommit on it", ctx.GetPath())); } - const std::string branch = BranchManager::NormalizeBranch(tmp_options.GetBranch()); + // A schema the caller gave replaces the catalog's; the caller answers for its correctness. + std::optional catalog_table_schema = ctx.GetSpecificTableSchema(); + if (!catalog_table_schema) { + PAIMON_ASSIGN_OR_RAISE( + std::optional catalog_identifier, + CatalogUtils::BranchIdentifier(ctx.GetIdentifier(), ctx.GetBranch())); + PAIMON_ASSIGN_OR_RAISE(catalog_table_schema, + CatalogMainBranchSchemaJson(ctx.GetCatalog(), catalog_identifier, + ctx.GetBranch(), kind)); + } return ManagedTableOf(tmp_options.GetFileSystem(), ctx.GetSpecificFileSystem(), ctx.GetPath(), - branch, ctx.GetOptions(), ctx.GetSpecificTableSchema(), - ctx.GetIdentifier(), - /*uuid=*/"", /*fs_scheme_map=*/{}); + ctx.GetBranch(), ctx.GetOptions(), catalog_table_schema, + ctx.GetIdentifier(), ctx.GetTableId().value_or(""), + ctx.GetFileSystemSchemeToIdentifierMap()); } -Result> FileStoreTableFactory::Create(const ReadContext& ctx) { - PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_core_options, - CoreOptions::FromMap(ctx.GetOptions(), ctx.GetSpecificFileSystem(), - ctx.GetFileSystemSchemeToIdentifierMap())); - PAIMON_ASSIGN_OR_RAISE(std::optional system_table_path, - SystemTableLoader::TryParsePath(ctx.GetPath())); - if (system_table_path) { - return Status::Invalid(fmt::format( - "the path {} names a system table, which is not loaded from a path: a catalog " - "hands the table back through GetTable, and its operators come from " - "table->NewScan/NewRead on it", - ctx.GetPath())); - } +} // namespace - const std::string branch = ctx.GetBranch(); - return ManagedTableOf(tmp_core_options.GetFileSystem(), ctx.GetSpecificFileSystem(), - ctx.GetPath(), branch, ctx.GetOptions(), ctx.GetSpecificTableSchema(), - ctx.GetIdentifier(), /*uuid=*/"", - ctx.GetFileSystemSchemeToIdentifierMap()); +Result> FileStoreTableFactory::Create(const ScanContext& ctx) { + return ManagedTable(ctx, "scan"); +} + +Result> FileStoreTableFactory::Create(const ReadContext& ctx) { + return ManagedTable(ctx, "read"); } Result> FileStoreTableFactory::Create(const WriteContext& ctx) { - PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx.GetCatalog())); - std::shared_ptr specific_fs = ctx.GetSpecificFileSystem(); - if (specific_fs == nullptr && ctx.GetFileSystemSchemeToIdentifierMap().empty() && - ctx.GetCatalog() != nullptr) { - if (!ctx.GetIdentifier()) { - return Status::Invalid("a catalog write requires a table identifier"); - } - PAIMON_ASSIGN_OR_RAISE(specific_fs, - ctx.GetCatalog()->GetTableFileSystem(ctx.GetIdentifier().value())); - } - PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, - CoreOptions::FromMap(ctx.GetOptions(), specific_fs, - ctx.GetFileSystemSchemeToIdentifierMap())); - std::string branch = ctx.GetBranch(); - PAIMON_ASSIGN_OR_RAISE(std::optional catalog_identifier, - CatalogUtils::BranchIdentifier(ctx.GetIdentifier(), branch)); - PAIMON_ASSIGN_OR_RAISE( - std::optional catalog_table_schema, - CatalogMainBranchSchemaJson(ctx.GetCatalog(), catalog_identifier, branch, "write")); - return ManagedTableOf(tmp_options.GetFileSystem(), specific_fs, ctx.GetRootPath(), branch, - ctx.GetOptions(), catalog_table_schema, ctx.GetIdentifier(), /*uuid=*/"", - ctx.GetFileSystemSchemeToIdentifierMap()); + return ManagedTable(ctx, "write"); } Result> FileStoreTableFactory::Create(const CommitContext& ctx) { - PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx.GetCatalog())); - std::shared_ptr specific_fs = ctx.GetSpecificFileSystem(); - if (specific_fs == nullptr && ctx.GetCatalog() != nullptr) { - if (!ctx.GetIdentifier()) { - return Status::Invalid("a catalog commit requires a table identifier"); - } - PAIMON_ASSIGN_OR_RAISE(specific_fs, - ctx.GetCatalog()->GetTableFileSystem(ctx.GetIdentifier().value())); - } - PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, - CoreOptions::FromMap(ctx.GetOptions(), specific_fs)); - const std::string& root_path = ctx.GetRootPath(); - PAIMON_ASSIGN_OR_RAISE(std::string branch, BranchManager::ResolveBranch( - ctx.GetIdentifier(), ctx.GetOptions(), - /*explicit_branch=*/std::nullopt, "commit")); - PAIMON_ASSIGN_OR_RAISE(std::optional catalog_identifier, - CatalogUtils::BranchIdentifier(ctx.GetIdentifier(), branch)); - PAIMON_ASSIGN_OR_RAISE( - std::optional catalog_table_schema, - CatalogMainBranchSchemaJson(ctx.GetCatalog(), catalog_identifier, branch, "commit")); - return ManagedTableOf(tmp_options.GetFileSystem(), specific_fs, root_path, branch, - ctx.GetOptions(), catalog_table_schema, ctx.GetIdentifier(), - ctx.GetTableId().value_or(""), /*fs_scheme_map=*/{}); + return ManagedTable(ctx, "commit"); } } // namespace paimon diff --git a/src/paimon/core/table/system/audit_log_system_table.cpp b/src/paimon/core/table/system/audit_log_system_table.cpp index 671081e54..2c3adec8c 100644 --- a/src/paimon/core/table/system/audit_log_system_table.cpp +++ b/src/paimon/core/table/system/audit_log_system_table.cpp @@ -396,13 +396,16 @@ Result> AuditLogSystemTable::NewScan( } std::shared_ptr scan_filter = context->GetScanFilters(); ScanContextBuilder builder(table_path_); + std::shared_ptr data_table_fs = context->GetSpecificFileSystem(); + if (data_table_fs == nullptr && context->GetFileSystemSchemeToIdentifierMap().empty()) { + data_table_fs = GetFileSystem(); + } builder.SetOptions(options_) .WithStreamingMode(context->IsStreamingMode()) .WithMemoryPool(context->GetMemoryPool()) .WithExecutor(context->GetExecutor()) - .WithFileSystem(context->GetSpecificFileSystem() != nullptr - ? context->GetSpecificFileSystem() - : GetFileSystem()) + .WithFileSystem(data_table_fs) + .WithFileSystemSchemeToIdentifierMap(context->GetFileSystemSchemeToIdentifierMap()) .WithCache(context->GetCache()); if (scan_filter) { if (scan_filter->GetBucketFilter()) { diff --git a/src/paimon/core/table/system/read_optimized_system_table.cpp b/src/paimon/core/table/system/read_optimized_system_table.cpp index 89f0455b0..ff6135178 100644 --- a/src/paimon/core/table/system/read_optimized_system_table.cpp +++ b/src/paimon/core/table/system/read_optimized_system_table.cpp @@ -65,13 +65,16 @@ Result> ReadOptimizedSystemTable::NewScan( } auto options = ReadOptimizedOptions(); ScanContextBuilder builder(table_path_); + std::shared_ptr data_table_fs = context->GetSpecificFileSystem(); + if (data_table_fs == nullptr && context->GetFileSystemSchemeToIdentifierMap().empty()) { + data_table_fs = GetFileSystem(); + } builder.SetOptions(options) .WithStreamingMode(context->IsStreamingMode()) .WithMemoryPool(context->GetMemoryPool()) .WithExecutor(context->GetExecutor()) - .WithFileSystem(context->GetSpecificFileSystem() != nullptr - ? context->GetSpecificFileSystem() - : GetFileSystem()) + .WithFileSystem(data_table_fs) + .WithFileSystemSchemeToIdentifierMap(context->GetFileSystemSchemeToIdentifierMap()) .WithCache(context->GetCache()); if (context->GetLimit().has_value()) { builder.SetLimit(context->GetLimit().value()); diff --git a/src/paimon/core/table/system/system_table_test.cpp b/src/paimon/core/table/system/system_table_test.cpp index 9ea5f3389..4e8a03516 100644 --- a/src/paimon/core/table/system/system_table_test.cpp +++ b/src/paimon/core/table/system/system_table_test.cpp @@ -31,6 +31,7 @@ #include "arrow/ipc/json_simple.h" #include "gtest/gtest.h" #include "paimon/catalog/identifier.h" +#include "paimon/commit_context.h" #include "paimon/common/data/generic_row.h" #include "paimon/core/catalog/file_system_catalog.h" #include "paimon/core/core_options.h" @@ -42,6 +43,8 @@ #include "paimon/core/table/system/read_optimized_system_table.h" #include "paimon/core/table/system/system_table_scan.h" #include "paimon/defs.h" +#include "paimon/file_store_commit.h" +#include "paimon/file_store_write.h" #include "paimon/fs/file_system.h" #include "paimon/fs/file_system_factory.h" #include "paimon/memory/memory_pool.h" @@ -60,6 +63,7 @@ #include "paimon/testing/mock/mock_file_batch_reader.h" #include "paimon/testing/utils/read_result_collector.h" #include "paimon/testing/utils/testharness.h" +#include "paimon/write_context.h" namespace paimon::test { namespace { @@ -335,6 +339,14 @@ TEST(SystemTableTest, TestTheOperatorEntriesRefuseASystemTablePath) { ReadContextBuilder read_builder(system_table_path); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), "names a system table"); + + WriteContextBuilder write_builder(system_table_path, "commit_user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); + ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_context)), "names a system table"); + + CommitContextBuilder commit_builder(system_table_path, "commit_user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); + ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(commit_context)), "names a system table"); } TEST(SystemTableTest, TestNewReadRefusesASearch) { diff --git a/src/paimon/core/table/table_test.cpp b/src/paimon/core/table/table_test.cpp index e2261518d..5154f9771 100644 --- a/src/paimon/core/table/table_test.cpp +++ b/src/paimon/core/table/table_test.cpp @@ -1477,4 +1477,65 @@ TEST(TableTest, TestCreateBuildsTheOperator) { FileStoreCommit::Create(std::move(commit_ctx))); } +TEST(TableTest, TestCreateWithSpecificTableSchemaSkipsTheCatalogSchema) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + CreateManagedTable(dir->GetFileSystem(), dir->Str(), identifier)); + + auto catalog = std::make_shared(); + catalog->SetFileSystem(dir->GetFileSystem()); + catalog->SetTableSchema(table->LatestSchema()); + + // A schema the caller gave replaces the catalog's on the main branch, so neither the write + // nor the commit asks the catalog for one. + ASSERT_OK_AND_ASSIGN(std::string schema_json, table->LatestSchema()->GetJsonSchema()); + + WriteContextBuilder write_builder(dir->Str(), "commit-user"); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr write_ctx, + write_builder.WithCatalog(catalog, identifier).SetTableSchema(schema_json).Finish()); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr write, + FileStoreWrite::Create(std::move(write_ctx))); + + CommitContextBuilder commit_builder(dir->Str(), "commit-user"); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr commit_ctx, + commit_builder.WithCatalog(catalog, identifier).SetTableSchema(schema_json).Finish()); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr commit, + FileStoreCommit::Create(std::move(commit_ctx))); + + ASSERT_EQ(catalog->LoadTableSchemaCalls(), 0U); +} + +TEST(TableTest, TestCreateWithCatalogReadsTheMainBranchSchemaFromIt) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + CreateManagedTable(dir->GetFileSystem(), dir->Str(), identifier)); + + auto catalog = std::make_shared(); + catalog->SetFileSystem(dir->GetFileSystem()); + catalog->SetTableSchema(table->LatestSchema()); + + // All four loads take the schema the catalog holds on the main branch when the caller did + // not set one, so each of them asks the catalog exactly once. + ScanContextBuilder scan_builder(dir->Str()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_ctx, + scan_builder.WithCatalog(catalog, identifier).Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
scanned, FileStoreTableFactory::Create(*scan_ctx)); + ASSERT_EQ(scanned->FullName(), "db.tbl"); + + ReadContextBuilder read_builder(dir->Str()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_ctx, + read_builder.WithCatalog(catalog, identifier).Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
read_table, + FileStoreTableFactory::Create(*read_ctx)); + ASSERT_EQ(read_table->FullName(), "db.tbl"); + + ASSERT_EQ(catalog->LoadTableSchemaCalls(), 2U); +} + } // namespace paimon::test From 00ccb4abcce69eb82577dc2359de2f7e64121d6d Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Wed, 28 Oct 2026 17:18:13 +0800 Subject: [PATCH 23/30] fix --- .../core/table/file_store_table_factory.cpp | 51 ++++++++++++------- src/paimon/core/table/table_test.cpp | 34 +++++++++++++ test/inte/blob_table_inte_test.cpp | 19 ++++--- 3 files changed, 80 insertions(+), 24 deletions(-) diff --git a/src/paimon/core/table/file_store_table_factory.cpp b/src/paimon/core/table/file_store_table_factory.cpp index 79bada00e..597db8b28 100644 --- a/src/paimon/core/table/file_store_table_factory.cpp +++ b/src/paimon/core/table/file_store_table_factory.cpp @@ -34,10 +34,12 @@ #include "paimon/core/core_options.h" #include "paimon/core/options/table_type.h" #include "paimon/core/schema/schema_manager.h" +#include "paimon/core/schema/schema_validation.h" #include "paimon/core/schema/table_schema.h" #include "paimon/core/table/file_store_table.h" #include "paimon/core/table/system/system_table.h" #include "paimon/core/utils/branch_manager.h" +#include "paimon/defs.h" #include "paimon/fs/file_system.h" #include "paimon/read_context.h" #include "paimon/result.h" @@ -80,27 +82,40 @@ Result> ManagedTableOf( } Identifier table_identifier = identifier.value_or(CatalogUtils::IdentifierFromPath(path, branch)); + // Fold the context options into the schema before the store resolves them: resolving the + // schema's own options first would fail on a format plugin the context overrides away, + // before the override has a chance to take effect. Only options that differ from the + // schema's own restyle the load, the way `Copy` decides; the rest leave the schema and the + // dynamic options as they are. + std::shared_ptr loaded_schema = latest_schema; + std::map dynamic_options; + if (latest_schema != nullptr && !options.empty()) { + const std::map& schema_options = latest_schema->Options(); + bool options_differ = false; + for (const auto& [key, value] : options) { + auto schema_option = schema_options.find(key); + if (schema_option == schema_options.end() || schema_option->second != value) { + options_differ = true; + break; + } + } + if (options_differ) { + dynamic_options = options; + dynamic_options.erase(Options::TYPE); + dynamic_options.erase(Options::BRANCH); + PAIMON_RETURN_NOT_OK( + SchemaValidation::CheckImmutableOptions(*latest_schema, dynamic_options)); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merged_schema, + latest_schema->MergeOptions(dynamic_options)); + loaded_schema = std::shared_ptr(std::move(merged_schema)); + } + } // The schema was read through a file system the context's options resolved, while the table is // served through the one the schema's own options resolve: the store builds its managers over // that resolution rather than taking over the reader's. - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr table, - FileStoreTable::Create(explicit_file_system, path, latest_schema, table_identifier, uuid, - branch, /*schema_manager=*/nullptr, /*snapshot_manager=*/nullptr, - fs_scheme_map, /*dynamic_options=*/{})); - // Fold in the context options, but only where one differs from the schema's own: a table is - // the product of its load, and `Copy` is the only way to restyle it. - if (table == nullptr || latest_schema == nullptr || options.empty()) { - return table; - } - const std::map& schema_options = latest_schema->Options(); - for (const auto& [key, value] : options) { - auto schema_option = schema_options.find(key); - if (schema_option == schema_options.end() || schema_option->second != value) { - return table->Copy(options); - } - } - return table; + return FileStoreTable::Create(explicit_file_system, path, loaded_schema, table_identifier, uuid, + branch, /*schema_manager=*/nullptr, + /*snapshot_manager=*/nullptr, fs_scheme_map, dynamic_options); } Result> CatalogMainBranchSchemaJson( diff --git a/src/paimon/core/table/table_test.cpp b/src/paimon/core/table/table_test.cpp index 5154f9771..f255519da 100644 --- a/src/paimon/core/table/table_test.cpp +++ b/src/paimon/core/table/table_test.cpp @@ -1440,6 +1440,40 @@ TEST(TableTest, TestALoadNamesTheDatabaseThePathLaysOut) { } } +TEST(TableTest, TestALoadResolvesTheContextFormatOverTheSchemaOne) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto schema = arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false)}); + // A schema whose stored format this process cannot resolve: resolving the schema's own + // options at load would fail on it, so the load only serves the table when the context + // overrides the format, and only when the override is merged before the resolution. + ASSERT_OK_AND_ASSIGN(std::unique_ptr stored_schema, + TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, + {{"file.format", "unavailable-format"}})); + ASSERT_OK_AND_ASSIGN(std::string schema_json, stored_schema->GetJsonSchema()); + { + ScanContextBuilder builder(dir->Str()); + builder.SetTableSchema(schema_json); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, builder.Finish()); + ASSERT_NOK(FileStoreTableFactory::Create(*ctx)); + } + { + ScanContextBuilder builder(dir->Str()); + builder.SetTableSchema(schema_json).AddOption(Options::FILE_FORMAT, "parquet"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, FileStoreTableFactory::Create(*ctx)); + ASSERT_NE(table, nullptr); + // The scan builds its path factory over the format the merged options resolve, so it + // only opens when the override, not the stored format, is what the table serves. + ScanContextBuilder scan_builder(dir->Str()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_ctx, scan_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, + table->NewScan(std::move(scan_ctx))); + ASSERT_NE(scan, nullptr); + } +} + TEST(TableTest, TestCreateBuildsTheOperator) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); diff --git a/test/inte/blob_table_inte_test.cpp b/test/inte/blob_table_inte_test.cpp index 4cbfda4df..689ca9513 100644 --- a/test/inte/blob_table_inte_test.cpp +++ b/test/inte/blob_table_inte_test.cpp @@ -216,6 +216,12 @@ class BlobTableInteTest : public testing::Test, public ::testing::WithParamInter return fields; } + /// The identifier `WriteDanglingBlobViewRows` stamps on row 0: no table exists where the + /// catalog issues it, so the catalog the blob view lookup resolves the reference through + /// reports the identifier not exist, and every read that reaches the reference fails on it. + static constexpr const char* kDanglingUpstreamNotFound = + "Identifier{database='nonexistent_db', object='nonexistent_table'} not exist"; + /// Writes and commits the two rows the blob-view skip tests share: row 0 holds a blob view /// reference to an upstream table that does not exist, row 1 holds no blob view at all. /// Resolving row 0 fails the whole read, so any read the pre-read still reaches it from must @@ -4589,10 +4595,11 @@ TEST_P(BlobTableInteTest, TestBlobViewFailsWhenUpstreamTableAbsent) { ASSERT_OK(WriteDanglingBlobViewRows(table_path)); - // reading fails because the upstream table does not exist at the catalog-issued location + // reading fails because the upstream table does not exist at the catalog-issued location: + // the catalog the lookup resolves the reference through reports its identifier not exist ASSERT_OK_AND_ASSIGN(auto plan, ScanTable(table_path)); ASSERT_NOK_WITH_MSG(ReadTable(table_path, schema->field_names(), plan, /*predicate=*/nullptr), - "not found latest schema"); + kDanglingUpstreamNotFound); } TEST_P(BlobTableInteTest, TestBlobViewSkipsDanglingReferenceOfDeletedRow) { @@ -4611,7 +4618,7 @@ TEST_P(BlobTableInteTest, TestBlobViewSkipsDanglingReferenceOfDeletedRow) { // without the deletion vector the dangling reference fails the read ASSERT_OK_AND_ASSIGN(auto plan, ScanTable(table_path)); ASSERT_NOK_WITH_MSG(ReadTable(table_path, schema->field_names(), plan, /*predicate=*/nullptr), - "not found latest schema"); + kDanglingUpstreamNotFound); ASSERT_OK_AND_ASSIGN(std::string anchor_file_name, DeletionVectorTestHelper::RetrieveAnchorFileName(commit_msgs)); @@ -4655,7 +4662,7 @@ TEST_P(BlobTableInteTest, TestBlobViewSkipsDanglingReferenceInEveryRowRangeGroup // without deletion vectors both groups' dangling references fail the read ASSERT_OK_AND_ASSIGN(auto plan, ScanTable(table_path)); ASSERT_NOK_WITH_MSG(ReadTable(table_path, schema->field_names(), plan, /*predicate=*/nullptr), - "not found latest schema"); + kDanglingUpstreamNotFound); // each group's anchor gets a deletion vector dropping its row 0, the dangling one ASSERT_OK_AND_ASSIGN(std::string anchor_file_name0, @@ -4692,7 +4699,7 @@ TEST_P(BlobTableInteTest, TestBlobViewPreReadHonorsRowRangesNotPredicate) { // reading the whole split fails on the dangling reference of row 0 ASSERT_OK_AND_ASSIGN(auto plan, ScanTable(table_path)); ASSERT_NOK_WITH_MSG(ReadTable(table_path, schema->field_names(), plan, /*predicate=*/nullptr), - "not found latest schema"); + kDanglingUpstreamNotFound); // a predicate keeping only row 1 does not rescue the query: it is evaluated above the split // read, while the pre-read takes none, so row 0's reference is still collected and resolved @@ -4700,7 +4707,7 @@ TEST_P(BlobTableInteTest, TestBlobViewPreReadHonorsRowRangesNotPredicate) { FieldType::INT, Literal(101)); ASSERT_OK_AND_ASSIGN(auto filtered_plan, ScanTable(table_path, keep_row_1)); ASSERT_NOK_WITH_MSG(ReadTable(table_path, schema->field_names(), filtered_plan, keep_row_1), - "not found latest schema"); + kDanglingUpstreamNotFound); // dropping the same row through the row ranges does: the pre-read honors them the same way // the main read does, so row 0 is never resolved From 6179d9b8caa6dab266d49bec019a879ceeb56c91 Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Wed, 28 Oct 2026 17:21:46 +0800 Subject: [PATCH 24/30] fix --- src/paimon/core/table/table_test.cpp | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/paimon/core/table/table_test.cpp b/src/paimon/core/table/table_test.cpp index f255519da..fc9056449 100644 --- a/src/paimon/core/table/table_test.cpp +++ b/src/paimon/core/table/table_test.cpp @@ -1447,10 +1447,10 @@ TEST(TableTest, TestALoadResolvesTheContextFormatOverTheSchemaOne) { // A schema whose stored format this process cannot resolve: resolving the schema's own // options at load would fail on it, so the load only serves the table when the context // overrides the format, and only when the override is merged before the resolution. - ASSERT_OK_AND_ASSIGN(std::unique_ptr stored_schema, - TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{}, - /*primary_keys=*/{}, - {{"file.format", "unavailable-format"}})); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr stored_schema, + TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, {{"file.format", "unavailable-format"}})); ASSERT_OK_AND_ASSIGN(std::string schema_json, stored_schema->GetJsonSchema()); { ScanContextBuilder builder(dir->Str()); @@ -1468,8 +1468,7 @@ TEST(TableTest, TestALoadResolvesTheContextFormatOverTheSchemaOne) { // only opens when the override, not the stored format, is what the table serves. ScanContextBuilder scan_builder(dir->Str()); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_ctx, scan_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, - table->NewScan(std::move(scan_ctx))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(scan_ctx))); ASSERT_NE(scan, nullptr); } } From ebde648b27961991f52ff110f8b2c36f5ae477f2 Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Wed, 28 Oct 2026 17:29:33 +0800 Subject: [PATCH 25/30] fix --- src/paimon/core/table/file_store.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/paimon/core/table/file_store.h b/src/paimon/core/table/file_store.h index 7acc5ca38..ccca2e402 100644 --- a/src/paimon/core/table/file_store.h +++ b/src/paimon/core/table/file_store.h @@ -75,6 +75,11 @@ class FileStore { /// builds one over `root_path` on `branch`. /// @param fs_scheme_map The scheme-to-identifier map the caller's contexts resolve custom /// schemes through, kept so a copy resolves as this store does. + // TODO(yonghao.fyh): Resolve the final CoreOptions before the table and the store are + // constructed, so this takes (root_path, core_options) and consumes the resolution: file + // system and branch from the options, both managers built inside, the schema left to + // FileStoreTable. CoreOptions retains FromMap's inputs - the specified file system and the + // scheme map - so a copy or a branch switch resolves through them again. static Result> Create( const std::shared_ptr& file_system, const std::string& root_path, const std::shared_ptr& schema, const std::string& branch, From b0b20369b1be3a633c0879f2a662bc8d394af6fd Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Wed, 28 Oct 2026 18:21:40 +0800 Subject: [PATCH 26/30] fix --- .../core/catalog/file_system_catalog.cpp | 3 +- .../table/append_only_file_store_table.cpp | 10 ++- .../core/table/append_only_file_store_table.h | 8 +- src/paimon/core/table/file_store.cpp | 65 +++----------- src/paimon/core/table/file_store.h | 87 +++++-------------- src/paimon/core/table/file_store_table.cpp | 65 ++++++++------ src/paimon/core/table/file_store_table.h | 40 ++++++--- .../core/table/file_store_table_factory.cpp | 3 +- .../table/primary_key_file_store_table.cpp | 11 +-- .../core/table/primary_key_file_store_table.h | 8 +- .../core/table/source/table_read_test.cpp | 2 - .../table/system/global_system_tables.cpp | 3 + src/paimon/core/table/table_test.cpp | 52 +++++------ src/paimon/rest/rest_catalog.cpp | 3 +- src/paimon/testing/mock/mock_catalog.h | 4 +- 15 files changed, 153 insertions(+), 211 deletions(-) diff --git a/src/paimon/core/catalog/file_system_catalog.cpp b/src/paimon/core/catalog/file_system_catalog.cpp index d5365d2ab..656a7e39c 100644 --- a/src/paimon/core/catalog/file_system_catalog.cpp +++ b/src/paimon/core/catalog/file_system_catalog.cpp @@ -361,8 +361,7 @@ Result> FileSystemCatalog::GetTable(const Identifier& ide "FileSystemCatalog::GetTable")); PAIMON_ASSIGN_OR_RAISE(std::string branch, identifier.GetBranchNameOrDefault()); return FileStoreTable::Create(fs_, table_path, checked_pointer_cast(*latest_schema), - identifier, /*uuid=*/"", branch, /*schema_manager=*/nullptr, - /*snapshot_manager=*/nullptr, /*fs_scheme_map=*/{}, + identifier, /*uuid=*/"", branch, /*fs_scheme_map=*/{}, /*dynamic_options=*/{}); } diff --git a/src/paimon/core/table/append_only_file_store_table.cpp b/src/paimon/core/table/append_only_file_store_table.cpp index 713c9160b..77df95336 100644 --- a/src/paimon/core/table/append_only_file_store_table.cpp +++ b/src/paimon/core/table/append_only_file_store_table.cpp @@ -34,6 +34,7 @@ #include "paimon/core/operation/data_evolution_file_store_scan.h" #include "paimon/core/operation/file_store_scan.h" #include "paimon/core/realtime/realtime_schema_layout.h" +#include "paimon/core/schema/table_schema.h" #include "paimon/core/snapshot.h" #include "paimon/core/table/bucket_mode.h" #include "paimon/core/table/file_store.h" @@ -48,9 +49,10 @@ namespace paimon { AppendOnlyFileStoreTable::AppendOnlyFileStoreTable(const std::shared_ptr& store, + const std::shared_ptr& schema, const Identifier& identifier, const std::string& uuid) - : FileStoreTable(store, identifier, uuid) {} + : FileStoreTable(store, schema, identifier, uuid) {} Result> AppendOnlyFileStoreTable::CreateWrite( const WriteContext& ctx, const std::shared_ptr& arrow_schema, @@ -58,8 +60,8 @@ Result> AppendOnlyFileStoreTable::CreateWrite( const std::shared_ptr& path_factory, const std::shared_ptr& snapshot_manager, const std::shared_ptr& io_manager, bool ignore_previous_files) const { - const std::shared_ptr& schema = store_->GetSchema(); const bool realtime_deduplicate = !options.GetRealtimeDeduplicateKeyFields().empty(); + std::shared_ptr schema = std::static_pointer_cast(schema_); bool need_dv_maintainer_factory = options.DeletionVectorsEnabled(); if (options.GetBucket() == -1) { need_dv_maintainer_factory = false; @@ -137,7 +139,7 @@ Result> AppendOnlyFileStoreTable::NewFileStoreSca const std::shared_ptr& executor, const std::shared_ptr& memory_pool, const ScanContext* context) const { // TODO(liancheng.lsz): support fallback branch in scan - const std::shared_ptr& table_schema = store_->GetSchema(); + std::shared_ptr table_schema = std::static_pointer_cast(schema_); std::unique_ptr scan; if (core_options.DataEvolutionEnabled()) { PAIMON_ASSIGN_OR_RAISE( @@ -177,7 +179,7 @@ CommitScanner::ScanSupplier AppendOnlyFileStoreTable::NewCommitScanSupplier( const std::shared_ptr& arrow_schema, const CoreOptions& options, const std::shared_ptr& executor, const std::shared_ptr& pool) const { const std::shared_ptr& schema_manager = GetSchemaManager(); - const std::shared_ptr& table_schema = store_->GetSchema(); + std::shared_ptr table_schema = std::static_pointer_cast(schema_); return [snapshot_manager, schema_manager, manifest_list, manifest_file, table_schema, arrow_schema, options, executor, pool](const std::shared_ptr& scan_filter) -> Result> { diff --git a/src/paimon/core/table/append_only_file_store_table.h b/src/paimon/core/table/append_only_file_store_table.h index 1f94d580a..5dee43d11 100644 --- a/src/paimon/core/table/append_only_file_store_table.h +++ b/src/paimon/core/table/append_only_file_store_table.h @@ -41,9 +41,11 @@ class TableSchema; class AppendOnlyFileStoreTable : public FileStoreTable { public: /// The constructor `FileStoreTable::Create` builds an append-only table through: the store - /// carries every state, and the base class holds it. - AppendOnlyFileStoreTable(const std::shared_ptr& store, const Identifier& identifier, - const std::string& uuid); + /// carries the storage state, `schema` is the schema the table was loaded with, and the base + /// class holds both. + AppendOnlyFileStoreTable(const std::shared_ptr& store, + const std::shared_ptr& schema, + const Identifier& identifier, const std::string& uuid); private: Result> CreateWrite( diff --git a/src/paimon/core/table/file_store.cpp b/src/paimon/core/table/file_store.cpp index b48f8cb8c..157b807f4 100644 --- a/src/paimon/core/table/file_store.cpp +++ b/src/paimon/core/table/file_store.cpp @@ -19,7 +19,6 @@ #include "paimon/core/table/file_store.h" -#include #include #include #include @@ -36,68 +35,27 @@ #include "paimon/core/manifest/manifest_file.h" #include "paimon/core/manifest/manifest_list.h" #include "paimon/core/schema/schema_manager.h" -#include "paimon/core/schema/table_schema.h" #include "paimon/core/utils/branch_manager.h" #include "paimon/core/utils/file_store_path_factory.h" #include "paimon/core/utils/index_file_path_factories.h" #include "paimon/core/utils/snapshot_manager.h" -#include "paimon/defs.h" #include "paimon/result.h" #include "paimon/status.h" namespace paimon { -Result> FileStore::Create( - const std::shared_ptr& file_system, const std::string& root_path, - const std::shared_ptr& schema, const std::string& branch, - const std::shared_ptr& schema_manager, - const std::shared_ptr& snapshot_manager, - const std::map& fs_scheme_map) { - auto store = std::shared_ptr(new FileStore( - file_system, root_path, schema, branch, schema_manager, snapshot_manager, fs_scheme_map)); - if (schema != nullptr) { - std::map options = schema->Options(); - if (BranchManager::IsMainBranch(store->branch_)) { - options.erase(Options::BRANCH); - } else { - options[Options::BRANCH] = store->branch_; - } - PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, - CoreOptions::FromMap(options, file_system, fs_scheme_map)); - store->core_options_ = std::move(core_options); - } - if (store->core_options_.has_value()) { - store->file_system_ = store->core_options_->GetFileSystem(); - } else if (file_system != nullptr) { - store->file_system_ = file_system; - } else { - PAIMON_ASSIGN_OR_RAISE(CoreOptions default_options, CoreOptions::FromMap({})); - store->file_system_ = default_options.GetFileSystem(); - } - if (store->schema_manager_ == nullptr) { - store->schema_manager_ = - std::make_shared(store->file_system_, root_path, store->branch_); - } - if (store->snapshot_manager_ == nullptr) { - store->snapshot_manager_ = - std::make_shared(store->file_system_, root_path, store->branch_); - } - return store; +std::shared_ptr FileStore::Create(const std::string& root_path, + CoreOptions core_options) { + return std::shared_ptr(new FileStore(root_path, std::move(core_options))); } -FileStore::FileStore(const std::shared_ptr& file_system, const std::string& root_path, - const std::shared_ptr& schema, const std::string& branch, - const std::shared_ptr& schema_manager, - const std::shared_ptr& snapshot_manager, - const std::map& fs_scheme_map) - : file_system_(file_system), - specified_file_system_(file_system), - fs_scheme_map_(fs_scheme_map), +FileStore::FileStore(const std::string& root_path, CoreOptions core_options) + : file_system_(core_options.GetFileSystem()), root_path_(root_path), - schema_(schema), - branch_(BranchManager::NormalizeBranch(branch)), - schema_manager_(schema_manager), - snapshot_manager_(snapshot_manager) {} + branch_(BranchManager::NormalizeBranch(core_options.GetBranch())), + schema_manager_(std::make_shared(file_system_, root_path_, branch_)), + snapshot_manager_(std::make_shared(file_system_, root_path_, branch_)), + core_options_(std::move(core_options)) {} Result> FileStore::NewPathFactory( const std::shared_ptr& arrow_schema, @@ -175,9 +133,10 @@ Result> FileStore::NewIndexFileHandler( } Result> FileStore::NewPartitionComputer( - const std::shared_ptr& arrow_schema, const CoreOptions& options, + const std::shared_ptr& arrow_schema, + const std::vector& partition_keys, const CoreOptions& options, const std::shared_ptr& memory_pool) const { - return BinaryRowPartitionComputer::Create(schema_->PartitionKeys(), arrow_schema, + return BinaryRowPartitionComputer::Create(partition_keys, arrow_schema, options.GetPartitionDefaultName(), options.LegacyPartitionNameEnabled(), memory_pool); } diff --git a/src/paimon/core/table/file_store.h b/src/paimon/core/table/file_store.h index ccca2e402..58812a76e 100644 --- a/src/paimon/core/table/file_store.h +++ b/src/paimon/core/table/file_store.h @@ -19,7 +19,6 @@ #pragma once -#include #include #include #include @@ -45,10 +44,9 @@ class IndexFileHandler; class IndexManifestFile; class ManifestFile; class ManifestList; -class TableSchema; /// Holds the storage a managed table reads and writes through: the file system, root path, -/// schema, branch, options and managers. Java Paimon carries the same state in `AbstractFileStore`. +/// branch, options and managers. Java Paimon carries the same state in `AbstractFileStore`. /// `FileStoreTable` builds one when it loads and asks it for the pieces each operator opens with. /// /// The factories below take the options of the call, so an operator can serve a cache of its own @@ -56,36 +54,15 @@ class TableSchema; /// cached one: a snapshot manager a write armed with a catalog loader must not serve a scan. class FileStore { public: - /// Builds the store. It parses the schema's options and fails when they do not parse, so the - /// constructor is private and callers come here. - /// @param file_system The file system the caller opened the table through. It wins over the - /// access configuration in the schema's options. Null leaves the resolution to that - /// configuration, and to the default local file system when the options name none. + /// Builds the store over options resolved before the call - `FileStoreTable::Create` resolves + /// them once - so the store consumes the resolution and parses nothing itself. /// @param root_path Root path of the table in the file system. - /// @param schema The schema the table was loaded with. Its partition keys name the partitions - /// the paths below are built for, and `GetOptions()` parses its options. Null when the - /// caller had none, which the operators then report. - /// @param branch The branch the store serves, normalized here. - /// @param schema_manager A schema manager to carry instead of building one: the one a table - /// asked for the branch it already serves hands over, whose resolution this store's - /// options therefore name as well. Null - what a load, a copy and a branch switch pass, - /// each resolving through a schema of its own - builds one over the resolved file - /// system, so no manager is ever left reading through a backend its table moved off. - /// @param snapshot_manager A snapshot manager to carry instead of building one, as above. Null - /// builds one over `root_path` on `branch`. - /// @param fs_scheme_map The scheme-to-identifier map the caller's contexts resolve custom - /// schemes through, kept so a copy resolves as this store does. - // TODO(yonghao.fyh): Resolve the final CoreOptions before the table and the store are - // constructed, so this takes (root_path, core_options) and consumes the resolution: file - // system and branch from the options, both managers built inside, the schema left to - // FileStoreTable. CoreOptions retains FromMap's inputs - the specified file system and the - // scheme map - so a copy or a branch switch resolves through them again. - static Result> Create( - const std::shared_ptr& file_system, const std::string& root_path, - const std::shared_ptr& schema, const std::string& branch, - const std::shared_ptr& schema_manager, - const std::shared_ptr& snapshot_manager, - const std::map& fs_scheme_map); + /// @param core_options The final options the table serves. The store reads and writes through + /// their file system and serves their branch, normalized here; both managers are built + /// over the two, so no manager is ever left reading through a backend its table moved + /// off. + static std::shared_ptr Create(const std::string& root_path, + CoreOptions core_options); const std::string& CurrentBranch() const { return branch_; @@ -103,38 +80,20 @@ class FileStore { return snapshot_manager_; } - /// The schema the table was loaded with, null when the caller had none. - const std::shared_ptr& GetSchema() const { - return schema_; - } - - /// The file system the table's data is accessed through: the one the options resolve, unless - /// the caller opened the table through one of its own. + /// The file system the table's data is accessed through: the one the options resolved, which + /// is the one the caller opened the table through when it opened it through one of its own. const std::shared_ptr& GetFileSystem() const { return file_system_; } - /// The file system the caller opened the table through, as handed to `Create`. Null when the - /// call had none, so a copy resolves from its own options again. - const std::shared_ptr& GetSpecifiedFileSystem() const { - return specified_file_system_; - } - - /// The scheme-to-identifier map the options were resolved with, kept so a copy can resolve - /// through it again. - const std::map& GetFsSchemeMap() const { - return fs_scheme_map_; - } - const std::string& GetRootPath() const { return root_path_; } - /// The options of the schema loaded on `CurrentBranch()`. They exist only over a schema, so an - /// operator reports that no schema was found before it ever reads them. + /// The final options the table serves, as `Create` took them. Every operator reads them; a + /// copy or a branch switch resolves its own, from the inputs the table keeps. const CoreOptions& GetOptions() const { - assert(core_options_.has_value()); - return *core_options_; + return core_options_; } /// Builds the path factory an operator names its metadata paths through, over this store's @@ -184,28 +143,22 @@ class FileStore { const std::shared_ptr& memory_pool) const; /// Builds the partition computer a commit names the partitions it writes with, over the - /// partition keys of this store's schema and the caller's arrow schema. + /// caller's partition keys and arrow schema. Result> NewPartitionComputer( - const std::shared_ptr& arrow_schema, const CoreOptions& options, + const std::shared_ptr& arrow_schema, + const std::vector& partition_keys, const CoreOptions& options, const std::shared_ptr& memory_pool) const; private: - FileStore(const std::shared_ptr& file_system, const std::string& root_path, - const std::shared_ptr& schema, const std::string& branch, - const std::shared_ptr& schema_manager, - const std::shared_ptr& snapshot_manager, - const std::map& fs_scheme_map); + FileStore(const std::string& root_path, CoreOptions core_options); std::shared_ptr file_system_; - std::shared_ptr specified_file_system_; - std::map fs_scheme_map_; std::string root_path_; - std::shared_ptr schema_; std::string branch_; std::shared_ptr schema_manager_; std::shared_ptr snapshot_manager_; - /// The options of `schema_` on `branch_`, unset when there is no schema. - std::optional core_options_; + /// The final options the table serves, as `Create` took them. + CoreOptions core_options_; }; } // namespace paimon diff --git a/src/paimon/core/table/file_store_table.cpp b/src/paimon/core/table/file_store_table.cpp index 7d5153956..6e306a2c8 100644 --- a/src/paimon/core/table/file_store_table.cpp +++ b/src/paimon/core/table/file_store_table.cpp @@ -97,16 +97,15 @@ class Executor; class MemoryPool; FileStoreTable::FileStoreTable(const std::shared_ptr& store, + const std::shared_ptr& schema, const Identifier& identifier, const std::string& uuid) - : Table(store->GetFileSystem(), store->GetRootPath(), store->GetSchema(), identifier, uuid), + : Table(store->GetFileSystem(), store->GetRootPath(), schema, identifier, uuid), store_(store) {} Result> FileStoreTable::Create( const std::shared_ptr& file_system, const std::string& root_path, const std::shared_ptr& schema, const Identifier& identifier, const std::string& uuid, - const std::string& branch, const std::shared_ptr& schema_manager, - const std::shared_ptr& snapshot_manager, - const std::map& fs_scheme_map, + const std::string& branch, const std::map& fs_scheme_map, const std::map& dynamic_options) { std::shared_ptr table_schema; if (schema != nullptr) { @@ -116,16 +115,30 @@ Result> FileStoreTable::Create( fmt::format("{} is not a managed table schema", identifier.ToString())); } } - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr store, - FileStore::Create(file_system, root_path, table_schema, branch, - schema_manager, snapshot_manager, fs_scheme_map)); + // Resolve the final options once, before the store is built: the schema's own options as the + // branch restyles them - the main branch carries no `branch` option, another branch names + // itself. A caller with no schema resolves the defaults over its branch, and the operators + // report the missing schema before they ever serve its options. + std::map options = + table_schema != nullptr ? table_schema->Options() : std::map{}; + const std::string normalized_branch = BranchManager::NormalizeBranch(branch); + if (BranchManager::IsMainBranch(normalized_branch)) { + options.erase(Options::BRANCH); + } else { + options[Options::BRANCH] = normalized_branch; + } + PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, + CoreOptions::FromMap(options, file_system, fs_scheme_map)); + std::shared_ptr store = FileStore::Create(root_path, std::move(core_options)); std::shared_ptr table; if (table_schema != nullptr && !table_schema->PrimaryKeys().empty()) { - table = std::make_shared(store, identifier, uuid); + table = std::make_shared(store, table_schema, identifier, uuid); } else { - table = std::make_shared(store, identifier, uuid); + table = std::make_shared(store, table_schema, identifier, uuid); } table->dynamic_options_ = dynamic_options; + table->specified_file_system_ = file_system; + table->fs_scheme_map_ = fs_scheme_map; return table; } @@ -147,20 +160,17 @@ Result> FileStoreTable::Copy( } // A copy resolves its file system from its own merged options, so it takes over no manager of // the table it was copied from: the store builds both over the resolution the copy names. - return FileStoreTable::Create(store_->GetSpecifiedFileSystem(), root_path_, - std::shared_ptr(std::move(merged_schema)), - identifier_, CatalogUuid().value_or(""), CurrentBranch(), - /*schema_manager=*/nullptr, /*snapshot_manager=*/nullptr, - store_->GetFsSchemeMap(), accumulated); + return FileStoreTable::Create( + specified_file_system_, root_path_, std::shared_ptr(std::move(merged_schema)), + identifier_, CatalogUuid().value_or(""), CurrentBranch(), fs_scheme_map_, accumulated); } Result> FileStoreTable::SwitchToBranch(const std::string& branch) const { const std::string target_branch = BranchManager::NormalizeBranch(branch); if (target_branch == CurrentBranch()) { - return FileStoreTable::Create(store_->GetSpecifiedFileSystem(), root_path_, schema_, - identifier_, CatalogUuid().value_or(""), CurrentBranch(), - GetSchemaManager(), GetSnapshotManager(), - store_->GetFsSchemeMap(), dynamic_options_); + return FileStoreTable::Create(specified_file_system_, root_path_, schema_, identifier_, + CatalogUuid().value_or(""), CurrentBranch(), fs_scheme_map_, + dynamic_options_); } // The branch is read through the file system this table serves, and then served through the // one its own schema resolves: the store builds the managers over that resolution, so none of @@ -179,11 +189,10 @@ Result> FileStoreTable::SwitchToBranch(const std::string& (*latest)->MergeOptions(overridable)); PAIMON_ASSIGN_OR_RAISE(Identifier branch_identifier, CatalogUtils::BranchIdentifier(identifier_, target_branch)); - return FileStoreTable::Create(store_->GetSpecifiedFileSystem(), root_path_, + return FileStoreTable::Create(specified_file_system_, root_path_, std::shared_ptr(std::move(branch_schema)), branch_identifier, CatalogUuid().value_or(""), target_branch, - /*schema_manager=*/nullptr, /*snapshot_manager=*/nullptr, - store_->GetFsSchemeMap(), /*dynamic_options=*/{}); + fs_scheme_map_, /*dynamic_options=*/{}); } Status FileStoreTable::RestoreRealtimeCommittedProgress( @@ -295,7 +304,7 @@ Result> FileStoreTable::NewWrite( } PAIMON_ASSIGN_OR_RAISE(std::optional catalog_identifier, CatalogUtils::BranchIdentifier(ctx->GetIdentifier(), CurrentBranch())); - const std::shared_ptr& schema = store_->GetSchema(); + std::shared_ptr schema = std::static_pointer_cast(schema_); if (schema == nullptr) { return Status::Invalid( fmt::format("not found latest schema in branch {}", CurrentBranch())); @@ -342,7 +351,7 @@ Result> FileStoreTable::NewWrite( Result> FileStoreTable::NewScan( std::unique_ptr context) const { PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "scan")); - const std::shared_ptr& table_schema = store_->GetSchema(); + std::shared_ptr table_schema = std::static_pointer_cast(schema_); if (table_schema == nullptr) { return Status::Invalid("not found latest schema"); } @@ -470,7 +479,7 @@ Result> FileStoreTable::NewRead( std::unique_ptr context) const { std::shared_ptr read_context = std::move(context); PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(read_context, "read")); - const std::shared_ptr& loaded_schema = store_->GetSchema(); + std::shared_ptr loaded_schema = std::static_pointer_cast(schema_); if (loaded_schema == nullptr) { return Status::Invalid("not found latest schema"); } @@ -538,7 +547,7 @@ Result> FileStoreTable::NewCommit( return data_schema->Id(); }; } - const std::shared_ptr& schema = store_->GetSchema(); + std::shared_ptr schema = std::static_pointer_cast(schema_); if (schema == nullptr) { return Status::Invalid( fmt::format("not found latest schema in branch {}", CurrentBranch())); @@ -559,9 +568,9 @@ Result> FileStoreTable::NewCommit( return Status::NotImplemented( "commit operation does not support object store file system for now"); } - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr partition_computer, - store_->NewPartitionComputer(arrow_schema, options, ctx->GetMemoryPool())); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr partition_computer, + store_->NewPartitionComputer(arrow_schema, schema->PartitionKeys(), + options, ctx->GetMemoryPool())); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr path_factory, store_->NewPathFactory(arrow_schema, schema->PartitionKeys(), options, diff --git a/src/paimon/core/table/file_store_table.h b/src/paimon/core/table/file_store_table.h index 20f811a5c..6c775d65a 100644 --- a/src/paimon/core/table/file_store_table.h +++ b/src/paimon/core/table/file_store_table.h @@ -59,26 +59,28 @@ class FileStoreTable : public Table { /// `AppendOnlyFileStoreTable` when there are none, a `PrimaryKeyFileStoreTable` when there are. /// Java Paimon's `FileStoreTableFactory` picks the kind the same way. A schema change can /// neither add nor drop primary keys, so every schema the root path ever held names one kind. + /// The final options are resolved here once, before the store is built: the schema's own + /// options as `branch` restyles them - the main branch carries no `branch` option, another + /// branch names itself - are parsed, and a parse failure is this call's error. Every operator + /// and both managers are built over that one resolution. + /// @param file_system The file system the caller opened the table through. It wins over the + /// access configuration in the schema's options. Null leaves the resolution to that + /// configuration, and to the default local file system when the options name none. Kept + /// so `Copy` and `SwitchToBranch` resolve through it again. /// @param schema The schema the table was loaded with. Its primary keys name the kind and its /// options are what the operators serve. Null when the caller had none, in which case /// the table is built as the append-only kind and its operators report that none was /// found. /// @param branch The branch the table serves, which the managers below read and write on. - /// @param schema_manager A schema manager to carry instead of building one. See - /// `FileStore::Create`. - /// @param snapshot_manager A snapshot manager to carry instead of building one. See - /// `FileStore::Create`. /// @param fs_scheme_map The scheme-to-identifier map the caller's contexts resolve custom - /// schemes through, which the store's options resolve through as well. `Copy` hands its - /// table's map over. + /// schemes through, which the options resolve through as well. Kept so `Copy` and + /// `SwitchToBranch` resolve through it again. /// @param dynamic_options The options a copy was given over the ones its schema stored, which /// `Copy` accumulates. See the member of the same name. static Result> Create( const std::shared_ptr& file_system, const std::string& root_path, const std::shared_ptr& schema, const Identifier& identifier, const std::string& uuid, const std::string& branch, - const std::shared_ptr& schema_manager, - const std::shared_ptr& snapshot_manager, const std::map& fs_scheme_map, const std::map& dynamic_options); @@ -113,7 +115,8 @@ class FileStoreTable : public Table { /// to is served as it is: its schema is read from its own directory and its managers are built /// over the file system that schema resolves, so neither this table's dynamic options nor its /// managers are carried over. Switching to the branch this table already serves hands back a - /// copy of it with the schema and the managers shared. A branch with no schema does not exist. + /// copy of it with the schema shared and the managers built over the same resolution again. A + /// branch with no schema does not exist. Result> SwitchToBranch(const std::string& branch) const; Result> NewScan(std::unique_ptr context) const override; @@ -161,9 +164,12 @@ class FileStoreTable : public Table { const std::shared_ptr& executor) const; protected: - /// The base constructor the kinds of managed table build through. The store carries every state - /// they hand down; see `Create` for what each piece of it is. - FileStoreTable(const std::shared_ptr& store, const Identifier& identifier, + /// The base constructor the kinds of managed table build through. The store carries the + /// storage they hand down; `schema` is the schema the table was loaded with, which the `Table` + /// base holds and the operators read - null when the caller had none. See `Create` for what + /// each piece is. + FileStoreTable(const std::shared_ptr& store, + const std::shared_ptr& schema, const Identifier& identifier, const std::string& uuid); /// Restores the committed progress of a real-time write from the latest snapshot the snapshot @@ -221,6 +227,16 @@ class FileStoreTable : public Table { /// merged into because a fallback reader needs them on their own: it serves another branch's /// schema and merges the same options over it again. std::map dynamic_options_; + + /// The file system the caller opened this table through, as the options were resolved with - + /// null when the load had none. A copy and a branch switch resolve their own options, and the + /// resolution that built this table's store is one-way: neither the file system it produced + /// nor the options hand the inputs back, so they are kept here for the next resolution. + std::shared_ptr specified_file_system_; + + /// The scheme-to-identifier map the options were resolved with, kept for the same next + /// resolution `specified_file_system_` is kept for. + std::map fs_scheme_map_; }; } // namespace paimon diff --git a/src/paimon/core/table/file_store_table_factory.cpp b/src/paimon/core/table/file_store_table_factory.cpp index 597db8b28..6231b791b 100644 --- a/src/paimon/core/table/file_store_table_factory.cpp +++ b/src/paimon/core/table/file_store_table_factory.cpp @@ -114,8 +114,7 @@ Result> ManagedTableOf( // served through the one the schema's own options resolve: the store builds its managers over // that resolution rather than taking over the reader's. return FileStoreTable::Create(explicit_file_system, path, loaded_schema, table_identifier, uuid, - branch, /*schema_manager=*/nullptr, - /*snapshot_manager=*/nullptr, fs_scheme_map, dynamic_options); + branch, fs_scheme_map, dynamic_options); } Result> CatalogMainBranchSchemaJson( diff --git a/src/paimon/core/table/primary_key_file_store_table.cpp b/src/paimon/core/table/primary_key_file_store_table.cpp index 90ed47a30..355ebf148 100644 --- a/src/paimon/core/table/primary_key_file_store_table.cpp +++ b/src/paimon/core/table/primary_key_file_store_table.cpp @@ -93,9 +93,10 @@ Result HasHistoricalPrimaryKeyBTreeDefinition( } // namespace PrimaryKeyFileStoreTable::PrimaryKeyFileStoreTable(const std::shared_ptr& store, + const std::shared_ptr& schema, const Identifier& identifier, const std::string& uuid) - : FileStoreTable(store, identifier, uuid) {} + : FileStoreTable(store, schema, identifier, uuid) {} Result> PrimaryKeyFileStoreTable::CreateWrite( const WriteContext& ctx, const std::shared_ptr& arrow_schema, @@ -103,7 +104,7 @@ Result> PrimaryKeyFileStoreTable::CreateWrite( const std::shared_ptr& path_factory, const std::shared_ptr& snapshot_manager, const std::shared_ptr& io_manager, bool ignore_previous_files) const { - const std::shared_ptr& schema = store_->GetSchema(); + std::shared_ptr schema = std::static_pointer_cast(schema_); if (ctx.GetRealtimeContext()) { PAIMON_RETURN_NOT_OK(PrimaryKeyTableUtils::ValidateRealtimeOptions(options, *schema)); if (ignore_previous_files) { @@ -222,7 +223,7 @@ Result> PrimaryKeyFileStoreTable::NewFileStoreSca const std::shared_ptr& executor, const std::shared_ptr& memory_pool, const ScanContext* context) const { // TODO(liancheng.lsz): support fallback branch in scan - const std::shared_ptr& table_schema = store_->GetSchema(); + std::shared_ptr table_schema = std::static_pointer_cast(schema_); std::unique_ptr scan; PAIMON_ASSIGN_OR_RAISE( scan, KeyValueFileStoreScan::Create(GetSnapshotManager(), GetSchemaManager(), manifest_list, @@ -237,7 +238,7 @@ Result> PrimaryKeyFileStoreTable::NewSplitGenera auto source_split_target_size = core_options.GetSourceSplitTargetSize(); auto source_split_open_file_cost = core_options.GetSourceSplitOpenFileCost(); // TODO(liancheng.lsz): support evolution - const std::shared_ptr& table_schema = store_->GetSchema(); + std::shared_ptr table_schema = std::static_pointer_cast(schema_); PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_keys, table_schema->TrimmedPrimaryKeys()); PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_pk_fields, @@ -257,7 +258,7 @@ CommitScanner::ScanSupplier PrimaryKeyFileStoreTable::NewCommitScanSupplier( const std::shared_ptr& arrow_schema, const CoreOptions& options, const std::shared_ptr& executor, const std::shared_ptr& pool) const { const std::shared_ptr& schema_manager = GetSchemaManager(); - const std::shared_ptr& table_schema = store_->GetSchema(); + std::shared_ptr table_schema = std::static_pointer_cast(schema_); return [snapshot_manager, schema_manager, manifest_list, manifest_file, table_schema, arrow_schema, options, executor, pool](const std::shared_ptr& scan_filter) -> Result> { diff --git a/src/paimon/core/table/primary_key_file_store_table.h b/src/paimon/core/table/primary_key_file_store_table.h index 2d3390e86..121081735 100644 --- a/src/paimon/core/table/primary_key_file_store_table.h +++ b/src/paimon/core/table/primary_key_file_store_table.h @@ -41,9 +41,11 @@ class TableSchema; class PrimaryKeyFileStoreTable : public FileStoreTable { public: /// The constructor `FileStoreTable::Create` builds a primary key table through: the store - /// carries every state, and the base class holds it. - PrimaryKeyFileStoreTable(const std::shared_ptr& store, const Identifier& identifier, - const std::string& uuid); + /// carries the storage state, `schema` is the schema the table was loaded with, and the base + /// class holds both. + PrimaryKeyFileStoreTable(const std::shared_ptr& store, + const std::shared_ptr& schema, + const Identifier& identifier, const std::string& uuid); private: Result> CreateWrite( diff --git a/src/paimon/core/table/source/table_read_test.cpp b/src/paimon/core/table/source/table_read_test.cpp index bd608bd74..c7724529b 100644 --- a/src/paimon/core/table/source/table_read_test.cpp +++ b/src/paimon/core/table/source/table_read_test.cpp @@ -139,7 +139,6 @@ TEST(TableReadTest, TestFullTextMatchExecutionIsNotImplemented) { std::shared_ptr table, FileStoreTable::Create(fs, table_path, schema, Identifier("append_09.db", "append_09"), /*uuid=*/"", /*branch=*/BranchManager::DEFAULT_MAIN_BRANCH, - /*schema_manager=*/nullptr, /*snapshot_manager=*/nullptr, /*fs_scheme_map=*/{}, /*dynamic_options=*/{})); for (bool table_entry : {false, true}) { ReadContextBuilder builder(table_path); @@ -169,7 +168,6 @@ TEST(TableReadTest, TestPerFieldScoreExecutionIsNotImplemented) { std::shared_ptr table, FileStoreTable::Create(fs, path, schema, Identifier("append_09.db", "append_09"), /*uuid=*/"", /*branch=*/BranchManager::DEFAULT_MAIN_BRANCH, - /*schema_manager=*/nullptr, /*snapshot_manager=*/nullptr, /*fs_scheme_map=*/{}, /*dynamic_options=*/{})); for (bool table_entry : {false, true}) { ReadContextBuilder builder(path); diff --git a/src/paimon/core/table/system/global_system_tables.cpp b/src/paimon/core/table/system/global_system_tables.cpp index 050964d62..e420125a7 100644 --- a/src/paimon/core/table/system/global_system_tables.cpp +++ b/src/paimon/core/table/system/global_system_tables.cpp @@ -320,6 +320,9 @@ Result> GlobalSystemTableLoader::LoadForSchema( } PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, entry.factory(context)); + // TODO(yonghao.fyh): the loader-driven AdoptOwnSchema() is an init-style coupling; + // consider overriding LatestSchema() on SystemTable, as FormatTable does, to serve + // its own read-only schema and drop this call from every loader. PAIMON_RETURN_NOT_OK(system_table->AdoptOwnSchema()); return system_table; } diff --git a/src/paimon/core/table/table_test.cpp b/src/paimon/core/table/table_test.cpp index fc9056449..43065e16b 100644 --- a/src/paimon/core/table/table_test.cpp +++ b/src/paimon/core/table/table_test.cpp @@ -74,8 +74,7 @@ Result> CreateManagedTable( return FileStoreTable::Create( file_system, path, checked_pointer_cast(std::shared_ptr(std::move(created))), identifier, - /*uuid=*/"", branch, /*schema_manager=*/nullptr, /*snapshot_manager=*/nullptr, - /*fs_scheme_map=*/{}, /*dynamic_options=*/{}); + /*uuid=*/"", branch, /*fs_scheme_map=*/{}, /*dynamic_options=*/{}); } void WriteAndCommitTwoRows(const std::shared_ptr
& table, const std::string& root_path) { @@ -238,8 +237,8 @@ TEST(TableTest, TestACopyHandsItsDynamicOptionsToTheFallbackReader) { FileStoreTable::Create( fs, path, checked_pointer_cast(std::shared_ptr(std::move(created))), - identifier, /*uuid=*/"", /*branch=*/"main", /*schema_manager=*/nullptr, - /*snapshot_manager=*/nullptr, /*fs_scheme_map=*/{}, /*dynamic_options=*/{})); + identifier, /*uuid=*/"", /*branch=*/"main", /*fs_scheme_map=*/{}, + /*dynamic_options=*/{})); ASSERT_OK_AND_ASSIGN(std::shared_ptr
copy, table->Copy({{Options::KEY_VALUE_SEQUENCE_NUMBER_ENABLED, "true"}})); @@ -276,8 +275,8 @@ TEST(TableTest, TestACopyOfACopyServesTheLatestValueOfARepeatedDynamicOption) { FileStoreTable::Create( fs, path, checked_pointer_cast(std::shared_ptr(std::move(created))), - identifier, /*uuid=*/"", /*branch=*/"main", /*schema_manager=*/nullptr, - /*snapshot_manager=*/nullptr, /*fs_scheme_map=*/{}, /*dynamic_options=*/{})); + identifier, /*uuid=*/"", /*branch=*/"main", /*fs_scheme_map=*/{}, + /*dynamic_options=*/{})); ASSERT_OK_AND_ASSIGN(std::shared_ptr
off, table->Copy({{Options::KEY_VALUE_SEQUENCE_NUMBER_ENABLED, "false"}})); @@ -311,8 +310,8 @@ TEST(TableTest, TestAFallbackReaderServesItsBranchsOwnSchema) { FileStoreTable::Create( fs, path, checked_pointer_cast(std::shared_ptr(std::move(main_created))), - identifier, /*uuid=*/"", /*branch=*/"main", /*schema_manager=*/nullptr, - /*snapshot_manager=*/nullptr, /*fs_scheme_map=*/{}, /*dynamic_options=*/{})); + identifier, /*uuid=*/"", /*branch=*/"main", /*fs_scheme_map=*/{}, + /*dynamic_options=*/{})); SchemaManager fallback_schema_manager(fs, path, "fb"); ASSERT_OK_AND_ASSIGN( @@ -325,8 +324,8 @@ TEST(TableTest, TestAFallbackReaderServesItsBranchsOwnSchema) { FileStoreTable::Create( fs, path, checked_pointer_cast(std::shared_ptr(std::move(fallback_created))), - identifier, /*uuid=*/"", /*branch=*/"fb", /*schema_manager=*/nullptr, - /*snapshot_manager=*/nullptr, /*fs_scheme_map=*/{}, /*dynamic_options=*/{})); + identifier, /*uuid=*/"", /*branch=*/"fb", /*fs_scheme_map=*/{}, + /*dynamic_options=*/{})); WriteAndCommitRow(fallback, path, /*id=*/1, /*v=*/10); WriteAndCommitRow(fallback, path, /*id=*/1, /*v=*/20); @@ -392,8 +391,8 @@ TEST(TableTest, TestAFallbackReaderServesItsBranchsOwnPartitionKeys) { FileStoreTable::Create( fs, path, checked_pointer_cast(std::shared_ptr(std::move(main_created))), - identifier, /*uuid=*/"", /*branch=*/"main", /*schema_manager=*/nullptr, - /*snapshot_manager=*/nullptr, /*fs_scheme_map=*/{}, /*dynamic_options=*/{})); + identifier, /*uuid=*/"", /*branch=*/"main", /*fs_scheme_map=*/{}, + /*dynamic_options=*/{})); SchemaManager fallback_schema_manager(fs, path, "fb"); auto row_schema = arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false), @@ -408,8 +407,8 @@ TEST(TableTest, TestAFallbackReaderServesItsBranchsOwnPartitionKeys) { FileStoreTable::Create( fs, path, checked_pointer_cast(std::shared_ptr(std::move(fallback_created))), - identifier, /*uuid=*/"", /*branch=*/"fb", /*schema_manager=*/nullptr, - /*snapshot_manager=*/nullptr, /*fs_scheme_map=*/{}, /*dynamic_options=*/{})); + identifier, /*uuid=*/"", /*branch=*/"fb", /*fs_scheme_map=*/{}, + /*dynamic_options=*/{})); WriteAndCommitRow(fallback, path, /*id=*/1, /*v=*/20); ReadContextBuilder read_builder(path); @@ -476,8 +475,8 @@ TEST(TableTest, TestAFallbackReaderIgnoresReadContextOptionsAbsentFromTheTable) return FileStoreTable::Create( fs, path, checked_pointer_cast(std::shared_ptr(std::move(created))), - Identifier("db", "tbl"), /*uuid=*/"", /*branch=*/"main", /*schema_manager=*/nullptr, - /*snapshot_manager=*/nullptr, /*fs_scheme_map=*/{}, /*dynamic_options=*/{}); + Identifier("db", "tbl"), /*uuid=*/"", /*branch=*/"main", /*fs_scheme_map=*/{}, + /*dynamic_options=*/{}); }; ASSERT_OK_AND_ASSIGN(std::shared_ptr
plain, make_table(dir->Str() + "/plain", {})); ASSERT_OK_AND_ASSIGN( @@ -560,10 +559,16 @@ TEST(TableTest, TestSwitchToBranch) { dev_file_store_table->SwitchToBranch("dev")); auto same_file_store_table = std::dynamic_pointer_cast(same_table); ASSERT_TRUE(same_file_store_table != nullptr); + // Switching to the branch a table already serves shares its schema and resolves again: the + // managers it hands back are instances of their own, reading the same path on the same branch. EXPECT_EQ(same_table->LatestSchema(), dev_table->LatestSchema()); - EXPECT_EQ(same_file_store_table->GetSchemaManager(), dev_file_store_table->GetSchemaManager()); - EXPECT_EQ(same_file_store_table->GetSnapshotManager(), + EXPECT_NE(same_file_store_table->GetSchemaManager(), dev_file_store_table->GetSchemaManager()); + EXPECT_NE(same_file_store_table->GetSnapshotManager(), dev_file_store_table->GetSnapshotManager()); + EXPECT_EQ("dev", same_file_store_table->CurrentBranch()); + EXPECT_EQ(dev_table->GetRootPath(), same_table->GetRootPath()); + EXPECT_EQ(path, same_file_store_table->GetSnapshotManager()->RootPath()); + EXPECT_EQ("dev", same_file_store_table->GetSnapshotManager()->Branch()); ASSERT_OK_AND_ASSIGN(std::shared_ptr
main_again, dev_file_store_table->SwitchToBranch("main")); @@ -597,9 +602,8 @@ TEST(TableTest, TestSwitchToBranchFromATableThatHasNoSchemaOfItsOwn) { std::shared_ptr table, FileStoreTable::Create(/*file_system=*/nullptr, path, /*schema=*/nullptr, Identifier("db", "tbl"), /*uuid=*/"", - /*branch=*/BranchManager::DEFAULT_MAIN_BRANCH, - /*schema_manager=*/nullptr, /*snapshot_manager=*/nullptr, - /*fs_scheme_map=*/{}, /*dynamic_options=*/{})); + /*branch=*/BranchManager::DEFAULT_MAIN_BRANCH, /*fs_scheme_map=*/{}, + /*dynamic_options=*/{})); ASSERT_OK_AND_ASSIGN(std::shared_ptr
dev_table, table->SwitchToBranch("dev")); auto dev_file_store_table = std::dynamic_pointer_cast(dev_table); ASSERT_TRUE(dev_file_store_table != nullptr); @@ -795,10 +799,7 @@ TEST(TableTest, TestMainBranchAliasResolvesToTheMainBranch) { ASSERT_OK_AND_ASSIGN(std::string branch, main_identifier.GetBranchNameOrDefault()); ASSERT_OK_AND_ASSIGN(std::shared_ptr table, FileStoreTable::Create(fs, dir->Str(), schema, main_identifier, - /*uuid=*/"", branch, - /*schema_manager=*/nullptr, - /*snapshot_manager=*/nullptr, - /*fs_scheme_map=*/{}, + /*uuid=*/"", branch, /*fs_scheme_map=*/{}, /*dynamic_options=*/{})); EXPECT_EQ("main", table->CurrentBranch()); } @@ -1278,7 +1279,6 @@ TEST(TableTest, TestCreateFailsWhenTheSchemaOptionsDoNotParse) { ASSERT_NOK_WITH_MSG( FileStoreTable::Create(dir->GetFileSystem(), dir->Str(), schema, Identifier("db", "tbl"), /*uuid=*/"", /*branch=*/BranchManager::DEFAULT_MAIN_BRANCH, - /*schema_manager=*/nullptr, /*snapshot_manager=*/nullptr, /*fs_scheme_map=*/{}, /*dynamic_options=*/{}), "target-file-row-num should be at least 1"); } diff --git a/src/paimon/rest/rest_catalog.cpp b/src/paimon/rest/rest_catalog.cpp index ccc10351b..af144f8a0 100644 --- a/src/paimon/rest/rest_catalog.cpp +++ b/src/paimon/rest/rest_catalog.cpp @@ -484,8 +484,7 @@ Result> RestCatalog::GetTable(const Identifier& identifie return FileStoreTable::Create( table_file_system, table_path, schema, identifier, uuid, NormalizeBranch(std::move(branch)).value_or(Identifier::kDefaultMainBranch), - /*schema_manager=*/nullptr, /*snapshot_manager=*/nullptr, /*fs_scheme_map=*/{}, - /*dynamic_options=*/{}); + /*fs_scheme_map=*/{}, /*dynamic_options=*/{}); } std::string RestCatalog::GetRootPath() const { diff --git a/src/paimon/testing/mock/mock_catalog.h b/src/paimon/testing/mock/mock_catalog.h index 346203c74..2890330c3 100644 --- a/src/paimon/testing/mock/mock_catalog.h +++ b/src/paimon/testing/mock/mock_catalog.h @@ -119,8 +119,8 @@ class MockVersionManagedCatalog : public Catalog, public VersionManagedCatalog { return FileStoreTable::Create( table_file_system_ != nullptr ? table_file_system_ : file_system_, /*root_path=*/"", table_schema_, identifier, table_uuid_, - /*branch=*/BranchManager::DEFAULT_MAIN_BRANCH, /*schema_manager=*/nullptr, - /*snapshot_manager=*/nullptr, /*fs_scheme_map=*/{}, /*dynamic_options=*/{}); + /*branch=*/BranchManager::DEFAULT_MAIN_BRANCH, /*fs_scheme_map=*/{}, + /*dynamic_options=*/{}); } void SetTableSchema(const std::shared_ptr& table_schema) { From ba063c616aeb4c6b77ddc2b6d948eeb6dca3f146 Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Thu, 29 Oct 2026 17:21:13 +0800 Subject: [PATCH 27/30] fix --- cmake_modules/SetupCxxFlags.cmake | 6 +- docs/code-style.md | 4 +- docs/source/user_guide/catalog.rst | 14 +- docs/source/user_guide/format_table.rst | 15 +- include/paimon/catalog/table.h | 8 +- include/paimon/commit_context.h | 10 + include/paimon/file_store_commit.h | 3 + include/paimon/file_store_write.h | 3 + include/paimon/read_context.h | 10 + include/paimon/scan_context.h | 10 + include/paimon/table/source/table_read.h | 3 + include/paimon/table/source/table_scan.h | 3 + include/paimon/write_context.h | 10 + .../core/catalog/file_system_catalog.cpp | 10 +- .../core/catalog/file_system_catalog_test.cpp | 5 + src/paimon/core/core_options.cpp | 1 + src/paimon/core/core_options_test.cpp | 1 + src/paimon/core/table/file_store_table.cpp | 158 ++++++---- src/paimon/core/table/file_store_table.h | 62 ++-- .../core/table/file_store_table_factory.cpp | 67 ++--- .../core/table/file_store_table_factory.h | 8 +- src/paimon/core/table/format/format_table.cpp | 2 +- .../table/system/audit_log_system_table.cpp | 5 +- .../table/system/audit_log_system_table.h | 5 +- .../core/table/system/binlog_system_table.cpp | 5 +- .../core/table/system/binlog_system_table.h | 5 +- .../table/system/metadata_system_tables.cpp | 16 +- .../system/read_optimized_system_table.cpp | 3 +- .../system/read_optimized_system_table.h | 5 +- src/paimon/core/table/system/system_table.cpp | 79 +++-- src/paimon/core/table/system/system_table.h | 15 +- .../core/table/system/system_table_test.cpp | 38 ++- src/paimon/core/table/table_test.cpp | 282 ++++++++++++------ src/paimon/rest/rest_catalog.cpp | 25 +- src/paimon/testing/mock/mock_catalog.h | 13 +- 35 files changed, 597 insertions(+), 312 deletions(-) diff --git a/cmake_modules/SetupCxxFlags.cmake b/cmake_modules/SetupCxxFlags.cmake index 855d8e1d3..cb08e6e84 100644 --- a/cmake_modules/SetupCxxFlags.cmake +++ b/cmake_modules/SetupCxxFlags.cmake @@ -106,7 +106,6 @@ if("${BUILD_WARNING_LEVEL}" STREQUAL "CHECKIN") elseif(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wall") set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-conversion") - set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-deprecated-declarations") set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-sign-conversion") set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-unused-variable") else() @@ -163,6 +162,11 @@ if(MSVC) # (required for protobuf, see https://github.com/protocolbuffers/protobuf/issues/6885) set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} /wd4065") elseif(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + # Paimon's own deprecated entry points stay exercised by its tests until they are removed, so + # deprecation warnings stay off for every GCC build, as they already are for every Clang build + # below. Downstream consumers compiling the public headers with their own flags see them. + set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-deprecated-declarations") + if(CMAKE_CXX_COMPILER_VERSION VERSION_EQUAL "7.0" OR CMAKE_CXX_COMPILER_VERSION VERSION_GREATER "7.0") # Without this, gcc >= 7 warns related to changes in C++17 diff --git a/docs/code-style.md b/docs/code-style.md index cb8c648e8..63530fbc8 100644 --- a/docs/code-style.md +++ b/docs/code-style.md @@ -168,10 +168,10 @@ All fallible functions return `Status` (no value) or `Result` (with value). * ```cpp // ✅ Good — explicit type PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, - TableScan::Create(std::move(ctx))); + table->NewScan(std::move(ctx))); // ❌ Bad — auto -PAIMON_ASSIGN_OR_RAISE(auto scan, TableScan::Create(std::move(ctx))); +PAIMON_ASSIGN_OR_RAISE(auto scan, table->NewScan(std::move(ctx))); ``` If you need a `shared_ptr` from a function returning `unique_ptr`, write the target type directly — the macro already moves: diff --git a/docs/source/user_guide/catalog.rst b/docs/source/user_guide/catalog.rst index 162799511..a7a9ccdc4 100644 --- a/docs/source/user_guide/catalog.rst +++ b/docs/source/user_guide/catalog.rst @@ -152,8 +152,10 @@ offsets when a real-time writer is created: write_builder.WithCatalog(catalog, paimon::Identifier("db", "tbl")); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, write_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, + catalog->GetTable(paimon::Identifier("db", "tbl"))); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr writer, - paimon::FileStoreWrite::Create(std::move(write_context))); + table->NewWrite(std::move(write_context))); For real-time writes, also set ``WithStreamingMode(true)`` and ``WithRealtimeContext(...)`` with ``realtime.enabled=true``. After @@ -209,13 +211,13 @@ keep it across recovery. .. code-block:: cpp PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, - shared_catalog->GetTable(paimon::Identifier("db", "tbl"))); + catalog->GetTable(paimon::Identifier("db", "tbl"))); std::optional table_uuid = table->CatalogUuid(); if (!table_uuid) { return paimon::Status::Invalid("a commit lease needs the table UUID"); } paimon::CommitContextBuilder builder(table_path, "writer-1"); - builder.WithCatalog(shared_catalog, paimon::Identifier("db", "tbl")) + builder.WithCatalog(catalog, paimon::Identifier("db", "tbl")) .WithTableId(table_uuid.value()) .AddOption(paimon::Options::REST_COMMIT_ACQUIRE_LOCK, "true"); @@ -320,15 +322,17 @@ for that table's file system and passing it in through ``WithFileSystem``: scan_builder.WithCatalog(catalog, paimon::Identifier("db", "tbl")); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, + catalog->GetTable(paimon::Identifier("db", "tbl"))); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, - paimon::TableScan::Create(std::move(scan_context))); + table->NewScan(std::move(scan_context))); paimon::ReadContextBuilder read_builder(table_path); read_builder.WithCatalog(catalog, paimon::Identifier("db", "tbl")); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, - paimon::TableRead::Create(std::move(read_context))); + table->NewRead(std::move(read_context))); When ``Finish()`` builds the context, it asks the catalog for the table's file system through ``Catalog::GetTableFileSystem`` and uses it as-is for the schema, diff --git a/docs/source/user_guide/format_table.rst b/docs/source/user_guide/format_table.rst index 7579b9dff..037300ca1 100644 --- a/docs/source/user_guide/format_table.rst +++ b/docs/source/user_guide/format_table.rst @@ -45,9 +45,11 @@ its ``type`` option says what it is, and the call hands back the ``FormatTable`` through the ``Table`` interface the two kinds of table share. Reading and writing go through the table itself: a caller holding one calls ``NewScan()``, -``NewRead()``, ``NewWrite()`` or ``NewCommit()`` on it, with a context its usual builder builds from -the table's location. This is what Java serves through ``FormatTable.newReadBuilder()`` and -``newBatchWriteBuilder()``: +``NewRead()``, ``NewWrite()`` or ``NewCommit()`` on it, with a context its usual builder builds: +``ScanContextBuilder(path)`` and ``ReadContextBuilder(path)`` take the table's location, while +``WriteContextBuilder(root_path, commit_user)`` and ``CommitContextBuilder(root_path, commit_user)`` +also take the commit user that identifies the writer. This is what Java serves through +``FormatTable.newReadBuilder()`` and ``newBatchWriteBuilder()``: .. code-block:: cpp @@ -68,9 +70,10 @@ the file system it was loaded through, so nothing under the location has to answ given at the call still win over the ones the schema stored, as they do everywhere else. The generic entry points - ``TableScan::Create``, ``TableRead::Create``, ``FileStoreWrite::Create`` -and ``FileStoreCommit::Create`` - serve a data table only: a load from a path is the data table's -way in, and a format table the schema under the path names is refused with a message pointing at -the catalog and the table's own operators above. +and ``FileStoreCommit::Create`` - are deprecated in favour of the table's own operators above and +serve a data table only: a load from a path is the data table's way in, and a format table the +schema under the path names is refused with a message pointing at the catalog and the table's own +operators above. ``FormatTable`` is a derived class of ``Table``, so a caller holding one through a ``std::shared_ptr`` calls the same operators on it without the cast: they come back diff --git a/include/paimon/catalog/table.h b/include/paimon/catalog/table.h index 9164623c0..6e401ff6b 100644 --- a/include/paimon/catalog/table.h +++ b/include/paimon/catalog/table.h @@ -46,9 +46,9 @@ class WriteContext; /// /// The state every kind of table carries - file system, root path, schema and identifier - lives /// here, and each kind fills in the operations. The `NewXxx` operators below are the table-first -/// entry points: `TableScan::Create()` and its friends load the table a context names and hand the -/// context to it. A table refuses by name whatever a context asks for that it cannot honour, -/// rather than dropping it silently. +/// entry points: the deprecated `TableScan::Create()` and its friends load the table a context +/// names and hand the context to it. A table refuses by name whatever a context asks for that it +/// cannot honour, rather than dropping it silently. class PAIMON_EXPORT Table { public: virtual ~Table() = default; @@ -129,7 +129,7 @@ class PAIMON_EXPORT Table { /// @param uuid Metastore UUID, or empty if unavailable. Table(const std::shared_ptr& file_system, const std::string& root_path, const std::shared_ptr& schema, const Identifier& identifier, - const std::string& uuid = ""); + const std::string& uuid); std::shared_ptr file_system_; std::string root_path_; diff --git a/include/paimon/commit_context.h b/include/paimon/commit_context.h index 688b10ff9..c76498e35 100644 --- a/include/paimon/commit_context.h +++ b/include/paimon/commit_context.h @@ -25,6 +25,7 @@ #include #include "paimon/catalog/identifier.h" +#include "paimon/macros.h" #include "paimon/result.h" #include "paimon/type_fwd.h" #include "paimon/visibility.h" @@ -168,6 +169,12 @@ class PAIMON_EXPORT CommitContextBuilder { /// @note The options map will clear the options added by `AddOption()` before. /// @param options The configuration options map. /// @return Reference to this builder for method chaining. + /// @deprecated These options reach an operator only through the deprecated + /// `FileStoreCommit::Create()`-style entry points, which merge them into the table they + /// load. An operator created from a `Table` serves that table's own configuration, so pass + /// dynamic options through `Table::Copy()` instead. A key no table serves, such as a + /// `header.` key, is still read from the context. + PAIMON_DEPRECATED("Pass dynamic options through Table::Copy() instead") CommitContextBuilder& SetOptions(const std::map& options); /// Add a single configuration option, served the way `SetOptions()` describes. @@ -177,6 +184,9 @@ class PAIMON_EXPORT CommitContextBuilder { /// @param key The option key. /// @param value The option value. /// @return Reference to this builder for method chaining. + /// @deprecated Use `Table::Copy()` to pass dynamic options, for the reason `SetOptions()` + /// gives. + PAIMON_DEPRECATED("Pass dynamic options through Table::Copy() instead") CommitContextBuilder& AddOption(const std::string& key, const std::string& value); /// Sets whether to ignore empty commits (default is true). diff --git a/include/paimon/file_store_commit.h b/include/paimon/file_store_commit.h index b0fcb17e1..c495cf281 100644 --- a/include/paimon/file_store_commit.h +++ b/include/paimon/file_store_commit.h @@ -28,6 +28,7 @@ #include "paimon/defs.h" #include "paimon/executor.h" +#include "paimon/macros.h" #include "paimon/memory/memory_pool.h" #include "paimon/metrics.h" #include "paimon/realtime/realtime_commit_progress.h" @@ -57,6 +58,8 @@ class PAIMON_EXPORT FileStoreCommit { /// @param context A unique pointer to the `CommitContext` used for commit operations. /// /// @return A Result containing a unique pointer to the `FileStoreCommit` instance. + /// @deprecated Use `Table::NewCommit()` on the table a catalog hands back instead. + PAIMON_DEPRECATED("Use Table::NewCommit() instead") static Result> Create(std::unique_ptr context); virtual ~FileStoreCommit() = default; diff --git a/include/paimon/file_store_write.h b/include/paimon/file_store_write.h index 351b98b06..697541aac 100644 --- a/include/paimon/file_store_write.h +++ b/include/paimon/file_store_write.h @@ -27,6 +27,7 @@ #include "paimon/commit_message.h" #include "paimon/defs.h" #include "paimon/executor.h" +#include "paimon/macros.h" #include "paimon/memory/memory_pool.h" #include "paimon/metrics.h" #include "paimon/realtime/realtime_commit_progress.h" @@ -49,6 +50,8 @@ class PAIMON_EXPORT FileStoreWrite { /// @param context A unique pointer to the `WriteContext` used for write operations. /// /// @return A Result containing a unique pointer to the `FileStoreWrite` instance. + /// @deprecated Use `Table::NewWrite()` on the table a catalog hands back instead. + PAIMON_DEPRECATED("Use Table::NewWrite() instead") static Result> Create(std::unique_ptr context); virtual ~FileStoreWrite() = default; diff --git a/include/paimon/read_context.h b/include/paimon/read_context.h index 2ee575c4f..0a926dc0c 100644 --- a/include/paimon/read_context.h +++ b/include/paimon/read_context.h @@ -28,6 +28,7 @@ #include "arrow/c/abi.h" #include "paimon/cache/cache.h" #include "paimon/catalog/identifier.h" +#include "paimon/macros.h" #include "paimon/predicate/predicate.h" #include "paimon/result.h" #include "paimon/type_fwd.h" @@ -323,6 +324,12 @@ class PAIMON_EXPORT ReadContextBuilder { /// @note The options map will clear the options added by `AddOption()` before. /// @param options The configuration options map. /// @return Reference to this builder for method chaining. + /// @deprecated These options reach an operator only through the deprecated + /// `TableRead::Create()`-style entry points, which merge them into the table they load. An + /// operator created from a `Table` serves that table's own configuration, so pass dynamic + /// options through `Table::Copy()` instead. A key no table serves, such as a `header.` key, + /// is still read from the context. + PAIMON_DEPRECATED("Pass dynamic options through Table::Copy() instead") ReadContextBuilder& SetOptions(const std::map& options); /// Add a single configuration option, served the way `SetOptions()` describes. @@ -332,6 +339,9 @@ class PAIMON_EXPORT ReadContextBuilder { /// @param key The option key. /// @param value The option value. /// @return Reference to this builder for method chaining. + /// @deprecated Use `Table::Copy()` to pass dynamic options, for the reason `SetOptions()` + /// gives. + PAIMON_DEPRECATED("Pass dynamic options through Table::Copy() instead") ReadContextBuilder& AddOption(const std::string& key, const std::string& value); /// Set a predicate for filtering data during reading. diff --git a/include/paimon/scan_context.h b/include/paimon/scan_context.h index 30f87baeb..52fddef73 100644 --- a/include/paimon/scan_context.h +++ b/include/paimon/scan_context.h @@ -28,6 +28,7 @@ #include "paimon/cache/cache.h" #include "paimon/catalog/identifier.h" #include "paimon/global_index/global_index_result.h" +#include "paimon/macros.h" #include "paimon/predicate/predicate.h" #include "paimon/result.h" #include "paimon/type_fwd.h" @@ -210,6 +211,9 @@ class PAIMON_EXPORT ScanContextBuilder { const std::shared_ptr& realtime_context); /// Add a single configuration option, served the way `SetOptions()` describes. + /// @deprecated Use `Table::Copy()` to pass dynamic options, for the reason `SetOptions()` + /// gives. + PAIMON_DEPRECATED("Pass dynamic options through Table::Copy() instead") ScanContextBuilder& AddOption(const std::string& key, const std::string& value); /// Set a configuration options map to set some option entries which are not defined in the /// table schema or whose values you want to overwrite. A load merges them into the table it @@ -218,6 +222,12 @@ class PAIMON_EXPORT ScanContextBuilder { /// @note The options map will clear the options added by `AddOption()` before. /// @param options The configuration options map. /// @return Reference to this builder for method chaining. + /// @deprecated These options reach an operator only through the deprecated + /// `TableScan::Create()`-style entry points, which merge them into the table they load. An + /// operator created from a `Table` serves that table's own configuration, so pass dynamic + /// options through `Table::Copy()` instead. A key no table serves, such as a `header.` key, + /// is still read from the context. + PAIMON_DEPRECATED("Pass dynamic options through Table::Copy() instead") ScanContextBuilder& SetOptions(const std::map& options); /// Set whether the scan is in streaming mode. diff --git a/include/paimon/table/source/table_read.h b/include/paimon/table/source/table_read.h index fa1aef730..36de26880 100644 --- a/include/paimon/table/source/table_read.h +++ b/include/paimon/table/source/table_read.h @@ -23,6 +23,7 @@ #include #include "paimon/executor.h" +#include "paimon/macros.h" #include "paimon/read_context.h" #include "paimon/reader/batch_reader.h" #include "paimon/reader/count_reader.h" @@ -48,6 +49,8 @@ class PAIMON_EXPORT TableRead { /// /// @param context A unique pointer to the `ReadContext` used for read operations. /// @return A Result containing a unique pointer to the `TableRead` instance. + /// @deprecated Use `Table::NewRead()` on the table a catalog hands back instead. + PAIMON_DEPRECATED("Use Table::NewRead() instead") static Result> Create(std::unique_ptr context); /// Creates a `BatchReader` instance for reading data. diff --git a/include/paimon/table/source/table_scan.h b/include/paimon/table/source/table_scan.h index edecb59c8..ea14f4fb8 100644 --- a/include/paimon/table/source/table_scan.h +++ b/include/paimon/table/source/table_scan.h @@ -24,6 +24,7 @@ #include #include +#include "paimon/macros.h" #include "paimon/result.h" #include "paimon/table/source/plan.h" #include "paimon/table/source/scan_metrics.h" @@ -42,6 +43,8 @@ class PAIMON_EXPORT TableScan { /// /// @param context A unique pointer to the `ScanContext` used for scan operations. /// @return A Result containing a unique pointer to the `TableScan` instance. + /// @deprecated Use `Table::NewScan()` on the table a catalog hands back instead. + PAIMON_DEPRECATED("Use Table::NewScan() instead") static Result> Create(std::unique_ptr context); virtual ~TableScan() = default; diff --git a/include/paimon/write_context.h b/include/paimon/write_context.h index d34bfe257..c3d3ea8b8 100644 --- a/include/paimon/write_context.h +++ b/include/paimon/write_context.h @@ -26,6 +26,7 @@ #include #include "paimon/catalog/identifier.h" +#include "paimon/macros.h" #include "paimon/result.h" #include "paimon/type_fwd.h" #include "paimon/visibility.h" @@ -190,6 +191,12 @@ class PAIMON_EXPORT WriteContextBuilder { /// @note The options map will clear the options added by `AddOption()` before. /// @param options The configuration options map. /// @return Reference to this builder for method chaining. + /// @deprecated These options reach an operator only through the deprecated + /// `FileStoreWrite::Create()`-style entry points, which merge them into the table they load. + /// An operator created from a `Table` serves that table's own configuration, so pass dynamic + /// options through `Table::Copy()` instead. A key no table serves, such as a `header.` key, + /// is still read from the context. + PAIMON_DEPRECATED("Pass dynamic options through Table::Copy() instead") WriteContextBuilder& SetOptions(const std::map& options); /// Add a single configuration option, served the way `SetOptions()` describes. @@ -199,6 +206,9 @@ class PAIMON_EXPORT WriteContextBuilder { /// @param key The option key. /// @param value The option value. /// @return Reference to this builder for method chaining. + /// @deprecated Use `Table::Copy()` to pass dynamic options, for the reason `SetOptions()` + /// gives. + PAIMON_DEPRECATED("Pass dynamic options through Table::Copy() instead") WriteContextBuilder& AddOption(const std::string& key, const std::string& value); /// Set whether to enable streaming mode (default is false) diff --git a/src/paimon/core/catalog/file_system_catalog.cpp b/src/paimon/core/catalog/file_system_catalog.cpp index 656a7e39c..a7dcd01ed 100644 --- a/src/paimon/core/catalog/file_system_catalog.cpp +++ b/src/paimon/core/catalog/file_system_catalog.cpp @@ -360,9 +360,8 @@ Result> FileSystemCatalog::GetTable(const Identifier& ide PAIMON_RETURN_NOT_OK(CatalogUtils::CheckManagedTableType(identifier, *latest_schema, "FileSystemCatalog::GetTable")); PAIMON_ASSIGN_OR_RAISE(std::string branch, identifier.GetBranchNameOrDefault()); - return FileStoreTable::Create(fs_, table_path, checked_pointer_cast(*latest_schema), - identifier, /*uuid=*/"", branch, /*fs_scheme_map=*/{}, - /*dynamic_options=*/{}); + return FileStoreTable::Create(fs_, table_path, *latest_schema, identifier, /*uuid=*/"", branch, + /*fs_scheme_map=*/{}, /*dynamic_options=*/{}); } Result> FileSystemCatalog::LoadSystemTable( @@ -386,8 +385,11 @@ Result> FileSystemCatalog::LoadSystemTable( dynamic_options[Options::BRANCH] = branch.value(); } PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(data_identifier)); + // The data table's logical name, branch included, so that a view of a branch answers with the + // name the branch is addressed by rather than the bare directory name. return SystemTableLoader::Load(system_table_name.value(), fs_, table_path, - latest_schema.value(), dynamic_options); + latest_schema.value(), dynamic_options, + data_identifier.GetObjectName()); } Status FileSystemCatalog::DropDatabase(const std::string& name, bool ignore_if_not_exists, diff --git a/src/paimon/core/catalog/file_system_catalog_test.cpp b/src/paimon/core/catalog/file_system_catalog_test.cpp index cc8fea8d8..7f63c8aef 100644 --- a/src/paimon/core/catalog/file_system_catalog_test.cpp +++ b/src/paimon/core/catalog/file_system_catalog_test.cpp @@ -437,6 +437,11 @@ TEST(FileSystemCatalogTest, TestBranchTableLoadsBranchSchema) { ASSERT_TRUE(exists); ASSERT_OK_AND_ASSIGN(loaded_schema, catalog.LoadTableSchema(audit_log_identifier)); ASSERT_EQ((std::vector{"rowkind", "b1_c0"}), loaded_schema->FieldNames()); + // A view of a branch answers with the name the branch is addressed by, so reloading that + // name lands on the same branch rather than on the main one. + ASSERT_OK_AND_ASSIGN(std::shared_ptr
branch_view, + catalog.GetTable(audit_log_identifier)); + ASSERT_EQ(branch_view->Name(), "tbl1$branch_b1$audit_log"); for (const auto& missing_identifier : {Identifier("db1", "tbl1$branch_missing"), Identifier("db1", "tbl1$branch_missing$options")}) { diff --git a/src/paimon/core/core_options.cpp b/src/paimon/core/core_options.cpp index c5018f6aa..0e9cee757 100644 --- a/src/paimon/core/core_options.cpp +++ b/src/paimon/core/core_options.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include #include "fmt/format.h" diff --git a/src/paimon/core/core_options_test.cpp b/src/paimon/core/core_options_test.cpp index 31750da0d..c76332dcf 100644 --- a/src/paimon/core/core_options_test.cpp +++ b/src/paimon/core/core_options_test.cpp @@ -19,6 +19,7 @@ #include "paimon/core/core_options.h" #include +#include #include #include #include diff --git a/src/paimon/core/table/file_store_table.cpp b/src/paimon/core/table/file_store_table.cpp index 6e306a2c8..da853a43a 100644 --- a/src/paimon/core/table/file_store_table.cpp +++ b/src/paimon/core/table/file_store_table.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include @@ -49,6 +50,7 @@ #include "paimon/core/operation/file_store_commit_impl.h" #include "paimon/core/operation/file_store_scan.h" #include "paimon/core/operation/internal_read_context.h" +#include "paimon/core/options/table_type.h" #include "paimon/core/realtime/realtime_context_impl.h" #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/schema_validation.h" @@ -93,6 +95,14 @@ class Schema; namespace paimon { +namespace { + +/// The escape hatch that lets an integration test commit on an object store, read where +/// `NewCommit` refuses one below. +constexpr const char kObjectStoreCommitInteTestOption[] = "enable-object-store-commit-in-inte-test"; + +} // namespace + class Executor; class MemoryPool; @@ -103,24 +113,16 @@ FileStoreTable::FileStoreTable(const std::shared_ptr& store, store_(store) {} Result> FileStoreTable::Create( - const std::shared_ptr& file_system, const std::string& root_path, - const std::shared_ptr& schema, const Identifier& identifier, const std::string& uuid, - const std::string& branch, const std::map& fs_scheme_map, + const std::shared_ptr& specified_file_system, const std::string& root_path, + const std::shared_ptr& schema, const Identifier& identifier, + const std::string& uuid, const std::string& branch, + const std::map& fs_scheme_map, const std::map& dynamic_options) { - std::shared_ptr table_schema; - if (schema != nullptr) { - table_schema = std::dynamic_pointer_cast(schema); - if (table_schema == nullptr) { - return Status::Invalid( - fmt::format("{} is not a managed table schema", identifier.ToString())); - } + if (schema == nullptr) { + return Status::Invalid( + fmt::format("not found latest schema of {}", identifier.GetFullName())); } - // Resolve the final options once, before the store is built: the schema's own options as the - // branch restyles them - the main branch carries no `branch` option, another branch names - // itself. A caller with no schema resolves the defaults over its branch, and the operators - // report the missing schema before they ever serve its options. - std::map options = - table_schema != nullptr ? table_schema->Options() : std::map{}; + std::map options = schema->Options(); const std::string normalized_branch = BranchManager::NormalizeBranch(branch); if (BranchManager::IsMainBranch(normalized_branch)) { options.erase(Options::BRANCH); @@ -128,16 +130,16 @@ Result> FileStoreTable::Create( options[Options::BRANCH] = normalized_branch; } PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, - CoreOptions::FromMap(options, file_system, fs_scheme_map)); + CoreOptions::FromMap(options, specified_file_system, fs_scheme_map)); std::shared_ptr store = FileStore::Create(root_path, std::move(core_options)); std::shared_ptr table; - if (table_schema != nullptr && !table_schema->PrimaryKeys().empty()) { - table = std::make_shared(store, table_schema, identifier, uuid); + if (!schema->PrimaryKeys().empty()) { + table = std::make_shared(store, schema, identifier, uuid); } else { - table = std::make_shared(store, table_schema, identifier, uuid); + table = std::make_shared(store, schema, identifier, uuid); } table->dynamic_options_ = dynamic_options; - table->specified_file_system_ = file_system; + table->specified_file_system_ = specified_file_system; table->fs_scheme_map_ = fs_scheme_map; return table; } @@ -145,16 +147,62 @@ Result> FileStoreTable::Create( Result> FileStoreTable::Copy( const std::map& dynamic_options) const { std::shared_ptr loaded_schema = std::static_pointer_cast(schema_); - if (loaded_schema == nullptr) { - return Status::Invalid(fmt::format("not found latest schema of {}", FullName())); + // `type` is immutable the way Java Paimon holds it: the kind of a copy is named by the primary + // keys of the schema it serves, never by this option. A value naming another type is refused, + // and one naming the same type - whatever its case, an absent `type` reading as `table` - is + // dropped, as Java's unchanged-key check drops it. + auto type_option = dynamic_options.find(Options::TYPE); + if (type_option != dynamic_options.end()) { + PAIMON_ASSIGN_OR_RAISE(TableType schema_type, + TableTypeDefine::FromOptions(loaded_schema->Options())); + std::map named{{Options::TYPE, type_option->second}}; + PAIMON_ASSIGN_OR_RAISE(TableType named_type, TableTypeDefine::FromOptions(named)); + if (named_type != schema_type) { + return Status::Invalid(fmt::format("Change '{}' is not supported yet.", Options::TYPE)); + } + } + // `branch` names the branch the copy serves, the way it routes a load: a copy on the branch + // this table serves keeps this table's schema, while one on another branch reads that branch's + // own latest schema and refuses a branch that holds none, as `SwitchToBranch` does. + std::string target_branch = CurrentBranch(); + auto branch_option = dynamic_options.find(Options::BRANCH); + if (branch_option != dynamic_options.end()) { + target_branch = BranchManager::NormalizeBranch(branch_option->second); } std::map overridable = dynamic_options; overridable.erase(Options::TYPE); overridable.erase(Options::BRANCH); - PAIMON_RETURN_NOT_OK(SchemaValidation::CheckImmutableOptions(*loaded_schema, overridable)); + std::shared_ptr base_schema = loaded_schema; + // An `Identifier` is immutable once built, so the branch one is held beside this table's + // and the copy carries whichever names the branch it serves. + std::optional branch_identifier; + if (target_branch != CurrentBranch()) { + // The branch is read through the file system this table serves, the same read + // `SwitchToBranch` makes, and then served through the one its own schema resolves. + SchemaManager schema_manager(file_system_, root_path_, target_branch); + PAIMON_ASSIGN_OR_RAISE(std::optional> latest, + schema_manager.Latest()); + if (!latest) { + return Status::Invalid(fmt::format("branch {} does not exist", target_branch)); + } + base_schema = *latest; + PAIMON_ASSIGN_OR_RAISE(Identifier branch_named, + CatalogUtils::BranchIdentifier(identifier_, target_branch)); + branch_identifier.emplace(std::move(branch_named)); + } + // The schema options name the branch the copy serves, the way `SwitchToBranch` styles them: + // another branch names itself in them, the main branch carries no `branch` option. The key is + // folded into the schema alone, not into the dynamic options the copy accumulates. + std::map schema_overridable = overridable; + if (target_branch != CurrentBranch() && target_branch != BranchManager::DEFAULT_MAIN_BRANCH) { + schema_overridable[Options::BRANCH] = target_branch; + } + PAIMON_RETURN_NOT_OK(SchemaValidation::CheckImmutableOptions(*base_schema, overridable)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merged_schema, - loaded_schema->MergeOptions(overridable)); + base_schema->MergeOptions(schema_overridable)); std::map accumulated = dynamic_options_; + accumulated.erase(Options::TYPE); + accumulated.erase(Options::BRANCH); for (const auto& [key, value] : overridable) { accumulated.insert_or_assign(key, value); } @@ -162,13 +210,15 @@ Result> FileStoreTable::Copy( // the table it was copied from: the store builds both over the resolution the copy names. return FileStoreTable::Create( specified_file_system_, root_path_, std::shared_ptr(std::move(merged_schema)), - identifier_, CatalogUuid().value_or(""), CurrentBranch(), fs_scheme_map_, accumulated); + branch_identifier ? *branch_identifier : identifier_, CatalogUuid().value_or(""), + target_branch, fs_scheme_map_, accumulated); } Result> FileStoreTable::SwitchToBranch(const std::string& branch) const { const std::string target_branch = BranchManager::NormalizeBranch(branch); if (target_branch == CurrentBranch()) { - return FileStoreTable::Create(specified_file_system_, root_path_, schema_, identifier_, + return FileStoreTable::Create(specified_file_system_, root_path_, + std::static_pointer_cast(schema_), identifier_, CatalogUuid().value_or(""), CurrentBranch(), fs_scheme_map_, dynamic_options_); } @@ -214,16 +264,39 @@ Status FileStoreTable::RestoreRealtimeCommittedProgress( } Status FileStoreTable::CheckOptionsAgreeWithTable( - const TableSchema& table_schema, const std::map& context_options) { - const std::map& table_options = table_schema.Options(); + const TableSchema& table_schema, + const std::map& context_options) const { + // The configuration the operators serve: the schema's own options as the load styled them, + // with the dynamic options of the load on top. `NewXxx` reads it through `store_->GetOptions()` + // and adds nothing from a context, so a context option that is not among these keys is one the + // operators cannot apply. + std::map table_options = table_schema.Options(); + for (const auto& [key, value] : dynamic_options_) { + table_options.insert_or_assign(key, value); + } for (const auto& [key, value] : context_options) { auto table_option = table_options.find(key); - if (table_option != table_options.end() && table_option->second != value) { - return Status::Invalid( - fmt::format("option '{}' is '{}' in the context but '{}' on the table", key, - SensitiveConfigUtils::RedactValue(key, value), - SensitiveConfigUtils::RedactValue(key, table_option->second))); + if (table_option != table_options.end()) { + if (table_option->second != value) { + return Status::Invalid( + fmt::format("option '{}' is '{}' in the context but '{}' on the table", key, + SensitiveConfigUtils::RedactValue(key, value), + SensitiveConfigUtils::RedactValue(key, table_option->second))); + } + continue; + } + // `branch` and `type` route a load to the table it names and are folded into it there - by + // the load or by `Copy` - so the spelling a context gives them is checked against the + // identifier and the schema, not against these keys. + if (key == Options::BRANCH || key == Options::TYPE) { + continue; } + return Status::Invalid(fmt::format( + "option '{}' is '{}' in the context but the table was not loaded with it: the " + "operator serves the table's own configuration, so the context value would be " + "dropped in silence; load the table with this option or apply it through " + "Table::Copy()", + key, SensitiveConfigUtils::RedactValue(key, value))); } return Status::OK(); } @@ -305,10 +378,6 @@ Result> FileStoreTable::NewWrite( PAIMON_ASSIGN_OR_RAISE(std::optional catalog_identifier, CatalogUtils::BranchIdentifier(ctx->GetIdentifier(), CurrentBranch())); std::shared_ptr schema = std::static_pointer_cast(schema_); - if (schema == nullptr) { - return Status::Invalid( - fmt::format("not found latest schema in branch {}", CurrentBranch())); - } PAIMON_RETURN_NOT_OK(CheckOptionsAgreeWithTable(*schema, ctx->GetOptions())); CoreOptions options = store_->GetOptions(); auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(schema->Fields()); @@ -352,9 +421,6 @@ Result> FileStoreTable::NewScan( std::unique_ptr context) const { PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "scan")); std::shared_ptr table_schema = std::static_pointer_cast(schema_); - if (table_schema == nullptr) { - return Status::Invalid("not found latest schema"); - } PAIMON_RETURN_NOT_OK(CheckOptionsAgreeWithTable(*table_schema, context->GetOptions())); PAIMON_ASSIGN_OR_RAISE(bool read_optimized, OptionsUtils::GetValueFromMap(context->GetOptions(), @@ -480,9 +546,6 @@ Result> FileStoreTable::NewRead( std::shared_ptr read_context = std::move(context); PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(read_context, "read")); std::shared_ptr loaded_schema = std::static_pointer_cast(schema_); - if (loaded_schema == nullptr) { - return Status::Invalid("not found latest schema"); - } PAIMON_RETURN_NOT_OK(CheckOptionsAgreeWithTable(*loaded_schema, read_context->GetOptions())); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr internal_context, InternalReadContext::CreateForTable(read_context, loaded_schema, @@ -548,10 +611,6 @@ Result> FileStoreTable::NewCommit( }; } std::shared_ptr schema = std::static_pointer_cast(schema_); - if (schema == nullptr) { - return Status::Invalid( - fmt::format("not found latest schema in branch {}", CurrentBranch())); - } std::shared_ptr arrow_schema = DataField::ConvertDataFieldsToArrowSchema(schema->Fields()); PAIMON_RETURN_NOT_OK(CheckOptionsAgreeWithTable(*schema, ctx->GetOptions())); @@ -563,8 +622,7 @@ Result> FileStoreTable::NewCommit( PAIMON_ASSIGN_OR_RAISE(bool is_object_store, FileSystem::IsObjectStore(root_path_)); if (is_object_store && !ctx->UseRESTCatalogCommit() && AsVersionManaged(ctx->GetCatalog()) == nullptr && - ctx->GetOptions().find("enable-object-store-commit-in-inte-test") == - ctx->GetOptions().end()) { + ctx->GetOptions().find(kObjectStoreCommitInteTestOption) == ctx->GetOptions().end()) { return Status::NotImplemented( "commit operation does not support object store file system for now"); } diff --git a/src/paimon/core/table/file_store_table.h b/src/paimon/core/table/file_store_table.h index 6c775d65a..64be124ef 100644 --- a/src/paimon/core/table/file_store_table.h +++ b/src/paimon/core/table/file_store_table.h @@ -27,6 +27,7 @@ #include "paimon/catalog/table.h" #include "paimon/core/operation/commit/commit_scanner.h" #include "paimon/core/schema/schema_manager.h" +#include "paimon/core/schema/table_schema.h" #include "paimon/core/table/file_store.h" #include "paimon/core/utils/branch_manager.h" #include "paimon/core/utils/snapshot_manager.h" @@ -40,7 +41,6 @@ class IOManager; class ManifestFile; class ManifestList; class RealtimeContext; -class Schema; class SplitGenerator; /// The managed table a catalog hands out: one with snapshots and manifests, described by the @@ -63,14 +63,14 @@ class FileStoreTable : public Table { /// options as `branch` restyles them - the main branch carries no `branch` option, another /// branch names itself - are parsed, and a parse failure is this call's error. Every operator /// and both managers are built over that one resolution. - /// @param file_system The file system the caller opened the table through. It wins over the - /// access configuration in the schema's options. Null leaves the resolution to that - /// configuration, and to the default local file system when the options name none. Kept - /// so `Copy` and `SwitchToBranch` resolve through it again. - /// @param schema The schema the table was loaded with. Its primary keys name the kind and its - /// options are what the operators serve. Null when the caller had none, in which case - /// the table is built as the append-only kind and its operators report that none was - /// found. + /// @param specified_file_system The file system the caller opened the table through. It + /// wins over the access configuration in the schema's options. Null leaves the + /// resolution to that configuration, and to the default local file system when the + /// options name none. Kept so `Copy` and `SwitchToBranch` resolve through it again. + /// @param schema The schema the table was loaded with, a `TableSchema` as every managed + /// table's is. Its primary keys name the kind and its options are what the operators + /// serve. Null is refused: a path a load found no schema under hands back no table, + /// and the failure belongs to the call that opens one. /// @param branch The branch the table serves, which the managers below read and write on. /// @param fs_scheme_map The scheme-to-identifier map the caller's contexts resolve custom /// schemes through, which the options resolve through as well. Kept so `Copy` and @@ -78,8 +78,8 @@ class FileStoreTable : public Table { /// @param dynamic_options The options a copy was given over the ones its schema stored, which /// `Copy` accumulates. See the member of the same name. static Result> Create( - const std::shared_ptr& file_system, const std::string& root_path, - const std::shared_ptr& schema, const Identifier& identifier, + const std::shared_ptr& specified_file_system, const std::string& root_path, + const std::shared_ptr& schema, const Identifier& identifier, const std::string& uuid, const std::string& branch, const std::map& fs_scheme_map, const std::map& dynamic_options); @@ -104,10 +104,16 @@ class FileStoreTable : public Table { return identifier_.GetFullName(); } - /// Returns a copy that serves `dynamic_options`. The root path and the identifier stay as they - /// are: they name this table, and an option does not rename it. The copy resolves the file - /// system the caller opened this table through, if any, together with its own merged options, - /// and builds its managers over that resolution rather than taking this table's over. + /// Returns a copy that serves `dynamic_options`. The root path stays as it is: it names this + /// table, and an option does not rename it. The `type` option is immutable the way Java + /// Paimon's is - a value naming the type the schema's primary keys already name, whatever its + /// case, is dropped, and any other value is refused - and the `branch` option names the branch + /// the copy serves: one on the branch this table serves keeps this table's schema, while one + /// on another branch reads that branch's own latest schema, carries the identifier of that + /// branch the way a load's routing does, and refuses a branch holding no schema. The copy + /// resolves the file system the caller opened this table through, if any, together with its + /// own merged options, and builds its managers over that resolution rather than taking this + /// table's over. Result> Copy( const std::map& dynamic_options) const override; @@ -130,15 +136,20 @@ class FileStoreTable : public Table { std::unique_ptr context) const override; private: - /// Checks the options a context carries against the ones the table was loaded with. A table - /// serves the options it was loaded with, so an option the context contradicts is refused: a - /// caller that means to change them copies the table. A table a load hands back carries the - /// context's own options merged in - `FileStoreTableFactory` merges them - so it agrees by - /// construction. The refusal belongs to a table the caller already held, whose load is over. A - /// key the table was not loaded with is no contradiction: it names a runtime setting, a scan - /// mode or a batch size, which is none of the table's business and is served as given. - static Status CheckOptionsAgreeWithTable( - const TableSchema& table_schema, const std::map& context_options); + /// Checks the options a context carries against the configuration the table serves: its + /// schema's options with its dynamic options on top. A table serves that configuration and + /// nothing else - the operators read it and add no context option to it - so an option the + /// context contradicts is refused, and so is one the table was not loaded with: serving it + /// would drop the context value in silence behind the table's own configuration. A caller + /// that means to change the configuration copies the table. A table a load hands back carries + /// the context's own options merged in - `FileStoreTableFactory` merges them, whatever the + /// key, as it merges a `header.` key the transport reads - so it agrees by construction; the + /// refusal belongs to a table the caller already held, whose load is over. The keys a load + /// routes with (`branch`, `type`) are folded into the table there - by the load or by `Copy` - + /// and checked against the identifier and the schema instead. + Status CheckOptionsAgreeWithTable( + const TableSchema& table_schema, + const std::map& context_options) const; /// Checks the identity a context carries - the path it names, the identifier it gives a /// catalog, the table id it records - against this table's own. The store below serves this @@ -166,8 +177,7 @@ class FileStoreTable : public Table { protected: /// The base constructor the kinds of managed table build through. The store carries the /// storage they hand down; `schema` is the schema the table was loaded with, which the `Table` - /// base holds and the operators read - null when the caller had none. See `Create` for what - /// each piece is. + /// base holds and the operators read. See `Create` for what each piece is. FileStoreTable(const std::shared_ptr& store, const std::shared_ptr& schema, const Identifier& identifier, const std::string& uuid); diff --git a/src/paimon/core/table/file_store_table_factory.cpp b/src/paimon/core/table/file_store_table_factory.cpp index 6231b791b..756273a91 100644 --- a/src/paimon/core/table/file_store_table_factory.cpp +++ b/src/paimon/core/table/file_store_table_factory.cpp @@ -69,52 +69,33 @@ Result> ManagedTableOf( latest_schema = *latest; } } - if (latest_schema != nullptr) { - PAIMON_ASSIGN_OR_RAISE(bool is_format_table, - TableTypeDefine::IsFormatTable(latest_schema->Options())); - if (is_format_table) { - return Status::Invalid(fmt::format( - "the schema under {} names a format table, which is not loaded from a path: a " - "catalog hands the table back through GetTable, and its operators come from " - "table->NewScan/NewRead/NewWrite/NewCommit on it", - path)); - } + // A path no schema was found under names no table: the failure belongs to the call that + // opens one, rather than to the first operator a handle without a schema would reach. + if (latest_schema == nullptr) { + return Status::Invalid(fmt::format("not found latest schema in branch {}", + BranchManager::NormalizeBranch(branch))); + } + PAIMON_ASSIGN_OR_RAISE(bool is_format_table, + TableTypeDefine::IsFormatTable(latest_schema->Options())); + if (is_format_table) { + return Status::Invalid(fmt::format( + "the schema under {} names a format table, which is not loaded from a path: a " + "catalog hands the table back through GetTable, and its operators come from " + "table->NewScan/NewRead/NewWrite/NewCommit on it", + path)); } Identifier table_identifier = identifier.value_or(CatalogUtils::IdentifierFromPath(path, branch)); - // Fold the context options into the schema before the store resolves them: resolving the - // schema's own options first would fail on a format plugin the context overrides away, - // before the override has a chance to take effect. Only options that differ from the - // schema's own restyle the load, the way `Copy` decides; the rest leave the schema and the - // dynamic options as they are. - std::shared_ptr loaded_schema = latest_schema; - std::map dynamic_options; - if (latest_schema != nullptr && !options.empty()) { - const std::map& schema_options = latest_schema->Options(); - bool options_differ = false; - for (const auto& [key, value] : options) { - auto schema_option = schema_options.find(key); - if (schema_option == schema_options.end() || schema_option->second != value) { - options_differ = true; - break; - } - } - if (options_differ) { - dynamic_options = options; - dynamic_options.erase(Options::TYPE); - dynamic_options.erase(Options::BRANCH); - PAIMON_RETURN_NOT_OK( - SchemaValidation::CheckImmutableOptions(*latest_schema, dynamic_options)); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merged_schema, - latest_schema->MergeOptions(dynamic_options)); - loaded_schema = std::shared_ptr(std::move(merged_schema)); - } - } - // The schema was read through a file system the context's options resolved, while the table is - // served through the one the schema's own options resolve: the store builds its managers over - // that resolution rather than taking over the reader's. - return FileStoreTable::Create(explicit_file_system, path, loaded_schema, table_identifier, uuid, - branch, fs_scheme_map, dynamic_options); + + std::map dynamic_options = options; + dynamic_options.erase(Options::TYPE); + dynamic_options.erase(Options::BRANCH); + PAIMON_RETURN_NOT_OK(SchemaValidation::CheckImmutableOptions(*latest_schema, dynamic_options)); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merged_schema, + latest_schema->MergeOptions(dynamic_options)); + return FileStoreTable::Create(explicit_file_system, path, + std::shared_ptr(std::move(merged_schema)), + table_identifier, uuid, branch, fs_scheme_map, dynamic_options); } Result> CatalogMainBranchSchemaJson( diff --git a/src/paimon/core/table/file_store_table_factory.h b/src/paimon/core/table/file_store_table_factory.h index e1090dc97..91113a96f 100644 --- a/src/paimon/core/table/file_store_table_factory.h +++ b/src/paimon/core/table/file_store_table_factory.h @@ -31,10 +31,10 @@ class Table; class WriteContext; /// Loads the data table a context names, as Java Paimon's `FileStoreTableFactory` does: the schema -/// under the context's path describes it. A system table behind a `T$system` path and a format -/// table are refused - a catalog hands those back through `GetTable`, and their operators come from -/// the table itself - so the static `Create` of all four operators answers the one question this -/// place asks and they cannot disagree. +/// under the context's path describes it, and a path holding none is refused. A system table +/// behind a `T$system` path and a format table are refused - a catalog hands those back through +/// `GetTable`, and their operators come from the table itself - so the static `Create` of all four +/// operators answers the one question this place asks and they cannot disagree. class FileStoreTableFactory { public: FileStoreTableFactory() = delete; diff --git a/src/paimon/core/table/format/format_table.cpp b/src/paimon/core/table/format/format_table.cpp index 846842cb7..141b31711 100644 --- a/src/paimon/core/table/format/format_table.cpp +++ b/src/paimon/core/table/format/format_table.cpp @@ -65,7 +65,7 @@ FormatTable::FormatTable(const std::shared_ptr& file_system, const std::string& file_compression, const std::string& partition_default_name, bool partition_only_value_in_path, bool location_carries_paimon_metadata) - : Table(file_system, location, schema, identifier), + : Table(file_system, location, schema, identifier, /*uuid=*/""), options_(options), format_(format), file_compression_(file_compression), diff --git a/src/paimon/core/table/system/audit_log_system_table.cpp b/src/paimon/core/table/system/audit_log_system_table.cpp index 2c3adec8c..e86ca34ca 100644 --- a/src/paimon/core/table/system/audit_log_system_table.cpp +++ b/src/paimon/core/table/system/audit_log_system_table.cpp @@ -358,9 +358,10 @@ std::unique_ptr CreateChangelogBatchReader( AuditLogSystemTable::AuditLogSystemTable(std::shared_ptr fs, std::string table_path, std::shared_ptr table_schema, + std::string data_table_name, std::map options) - : AuditLogSystemTable(std::move(fs), table_path, std::move(table_schema), std::move(options), - ComposeName(table_path, kName)) {} + : AuditLogSystemTable(std::move(fs), std::move(table_path), std::move(table_schema), + std::move(options), ComposeName(data_table_name, kName)) {} AuditLogSystemTable::AuditLogSystemTable(std::shared_ptr fs, std::string table_path, std::shared_ptr table_schema, diff --git a/src/paimon/core/table/system/audit_log_system_table.h b/src/paimon/core/table/system/audit_log_system_table.h index 4cc324b1d..3aad57a8e 100644 --- a/src/paimon/core/table/system/audit_log_system_table.h +++ b/src/paimon/core/table/system/audit_log_system_table.h @@ -54,8 +54,11 @@ class AuditLogSystemTable : public SystemTable { public: static constexpr const char* kName = "audit_log"; + /// @param data_table_name Logical object name of the data table underneath, branch included, + /// which this view answers `$audit_log` with - the name a catalog + /// knows the data table by, not the name of the directory it lives in. AuditLogSystemTable(std::shared_ptr fs, std::string table_path, - std::shared_ptr table_schema, + std::shared_ptr table_schema, std::string data_table_name, std::map options); Result> ArrowSchema() const override; diff --git a/src/paimon/core/table/system/binlog_system_table.cpp b/src/paimon/core/table/system/binlog_system_table.cpp index 8299f17f5..7b73ce1dc 100644 --- a/src/paimon/core/table/system/binlog_system_table.cpp +++ b/src/paimon/core/table/system/binlog_system_table.cpp @@ -73,9 +73,10 @@ std::shared_ptr CreateBinlogBatchConverter() { BinlogSystemTable::BinlogSystemTable(std::shared_ptr fs, std::string table_path, std::shared_ptr table_schema, + std::string data_table_name, std::map options) - : AuditLogSystemTable(std::move(fs), table_path, std::move(table_schema), std::move(options), - ComposeName(table_path, kName)) {} + : AuditLogSystemTable(std::move(fs), std::move(table_path), std::move(table_schema), + std::move(options), ComposeName(data_table_name, kName)) {} Result> BinlogSystemTable::ArrowSchema() const { std::shared_ptr rowkind_field = diff --git a/src/paimon/core/table/system/binlog_system_table.h b/src/paimon/core/table/system/binlog_system_table.h index 3a28ffa93..5d486e457 100644 --- a/src/paimon/core/table/system/binlog_system_table.h +++ b/src/paimon/core/table/system/binlog_system_table.h @@ -36,8 +36,11 @@ class BinlogSystemTable : public AuditLogSystemTable { public: static constexpr const char* kName = "binlog"; + /// @param data_table_name Logical object name of the data table underneath, branch included, + /// which this view answers `$binlog` with - the name a catalog knows + /// the data table by, not the name of the directory it lives in. BinlogSystemTable(std::shared_ptr fs, std::string table_path, - std::shared_ptr table_schema, + std::shared_ptr table_schema, std::string data_table_name, std::map options); Result> ArrowSchema() const override; diff --git a/src/paimon/core/table/system/metadata_system_tables.cpp b/src/paimon/core/table/system/metadata_system_tables.cpp index 1b5927004..3ccb488de 100644 --- a/src/paimon/core/table/system/metadata_system_tables.cpp +++ b/src/paimon/core/table/system/metadata_system_tables.cpp @@ -507,7 +507,7 @@ Result> WriteColsValue( OptionsSystemTable::OptionsSystemTable(const std::shared_ptr& fs, std::string table_path, std::shared_ptr table_schema) - : InMemorySystemTable(fs, table_path, ComposeName(table_path, kName)), + : InMemorySystemTable(fs, table_path, ComposeName(PathUtil::GetName(table_path), kName)), table_schema_(std::move(table_schema)) {} Result> OptionsSystemTable::ArrowSchema() const { @@ -530,7 +530,7 @@ Result> OptionsSystemTable::BuildRows() const { SnapshotsSystemTable::SnapshotsSystemTable(std::shared_ptr fs, std::string table_path, std::string branch) - : InMemorySystemTable(fs, table_path, ComposeName(table_path, kName)), + : InMemorySystemTable(fs, table_path, ComposeName(PathUtil::GetName(table_path), kName)), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch))) {} Result> SnapshotsSystemTable::ArrowSchema() const { @@ -588,7 +588,7 @@ Result> SnapshotsSystemTable::BuildRows() const { SchemasSystemTable::SchemasSystemTable(std::shared_ptr fs, std::string table_path, std::string branch) - : InMemorySystemTable(fs, table_path, ComposeName(table_path, kName)), + : InMemorySystemTable(fs, table_path, ComposeName(PathUtil::GetName(table_path), kName)), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch))) {} Result> SchemasSystemTable::ArrowSchema() const { @@ -640,7 +640,7 @@ Result> SchemasSystemTable::BuildRows() const { TagsSystemTable::TagsSystemTable(std::shared_ptr fs, std::string table_path, std::string branch) - : InMemorySystemTable(fs, table_path, ComposeName(table_path, kName)), + : InMemorySystemTable(fs, table_path, ComposeName(PathUtil::GetName(table_path), kName)), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch))) {} Result> TagsSystemTable::ArrowSchema() const { @@ -686,7 +686,7 @@ Result> TagsSystemTable::BuildRows() const { BranchesSystemTable::BranchesSystemTable(std::shared_ptr fs, std::string table_path, std::string branch) - : InMemorySystemTable(fs, table_path, ComposeName(table_path, kName)), + : InMemorySystemTable(fs, table_path, ComposeName(PathUtil::GetName(table_path), kName)), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch))) {} Result> BranchesSystemTable::ArrowSchema() const { @@ -721,7 +721,7 @@ Result> BranchesSystemTable::BuildRows() const { ConsumersSystemTable::ConsumersSystemTable(std::shared_ptr fs, std::string table_path, std::string branch) - : InMemorySystemTable(fs, table_path, ComposeName(table_path, kName)), + : InMemorySystemTable(fs, table_path, ComposeName(PathUtil::GetName(table_path), kName)), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch))) {} Result> ConsumersSystemTable::ArrowSchema() const { @@ -752,7 +752,7 @@ ManifestsSystemTable::ManifestsSystemTable(std::shared_ptr fs, std:: std::string branch, std::shared_ptr table_schema, std::map options) - : InMemorySystemTable(fs, table_path, ComposeName(table_path, kName)), + : InMemorySystemTable(fs, table_path, ComposeName(PathUtil::GetName(table_path), kName)), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch), std::move(table_schema), std::move(options))) {} @@ -817,7 +817,7 @@ Result> ManifestsSystemTable::BuildRows() const { FilesSystemTable::FilesSystemTable(std::shared_ptr fs, std::string table_path, std::string branch, std::shared_ptr table_schema, std::map options) - : InMemorySystemTable(fs, table_path, ComposeName(table_path, kName)), + : InMemorySystemTable(fs, table_path, ComposeName(PathUtil::GetName(table_path), kName)), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch), std::move(table_schema), std::move(options))) {} diff --git a/src/paimon/core/table/system/read_optimized_system_table.cpp b/src/paimon/core/table/system/read_optimized_system_table.cpp index ff6135178..4f529de13 100644 --- a/src/paimon/core/table/system/read_optimized_system_table.cpp +++ b/src/paimon/core/table/system/read_optimized_system_table.cpp @@ -39,8 +39,9 @@ namespace paimon { ReadOptimizedSystemTable::ReadOptimizedSystemTable(const std::shared_ptr& fs, std::string table_path, std::shared_ptr table_schema, + std::string data_table_name, std::map options) - : SystemTable(fs, table_path, ComposeName(table_path, kName)), + : SystemTable(fs, table_path, ComposeName(data_table_name, kName)), table_path_(std::move(table_path)), table_schema_(std::move(table_schema)), options_(std::move(options)) {} diff --git a/src/paimon/core/table/system/read_optimized_system_table.h b/src/paimon/core/table/system/read_optimized_system_table.h index 82fec3ef4..5efb0f289 100644 --- a/src/paimon/core/table/system/read_optimized_system_table.h +++ b/src/paimon/core/table/system/read_optimized_system_table.h @@ -34,8 +34,11 @@ class ReadOptimizedSystemTable : public SystemTable { public: static constexpr const char* kName = "ro"; + /// @param data_table_name Logical object name of the data table underneath, branch included, + /// which this view answers `$ro` with - the name a catalog knows the + /// data table by, not the name of the directory it lives in. ReadOptimizedSystemTable(const std::shared_ptr& fs, std::string table_path, - std::shared_ptr table_schema, + std::shared_ptr table_schema, std::string data_table_name, std::map options); Result> ArrowSchema() const override; diff --git a/src/paimon/core/table/system/system_table.cpp b/src/paimon/core/table/system/system_table.cpp index 872f6e502..a8be1c62d 100644 --- a/src/paimon/core/table/system/system_table.cpp +++ b/src/paimon/core/table/system/system_table.cpp @@ -52,7 +52,7 @@ namespace { using SystemTableFactory = std::function>( const std::shared_ptr&, const std::string&, const std::shared_ptr&, - const std::map&)>; + const std::map&, const std::string&)>; struct SystemTableRegistryEntry { std::string name; @@ -79,79 +79,82 @@ const std::vector& SystemTableRegistry() { {OptionsSystemTable::kName, [](const std::shared_ptr& fs, const std::string& table_path, const std::shared_ptr& table_schema, - const std::map& /*dynamic_options*/) - -> Result> { + const std::map& /*dynamic_options*/, + const std::string& /*data_table_name*/) -> Result> { return std::make_shared(fs, table_path, table_schema); }}, {AuditLogSystemTable::kName, [](const std::shared_ptr& fs, const std::string& table_path, const std::shared_ptr& table_schema, - const std::map& dynamic_options) - -> Result> { + const std::map& dynamic_options, + const std::string& data_table_name) -> Result> { return std::make_shared( - fs, table_path, table_schema, MergeOptions(table_schema, dynamic_options)); + fs, table_path, table_schema, data_table_name, + MergeOptions(table_schema, dynamic_options)); }}, {BinlogSystemTable::kName, [](const std::shared_ptr& fs, const std::string& table_path, const std::shared_ptr& table_schema, - const std::map& dynamic_options) - -> Result> { + const std::map& dynamic_options, + const std::string& data_table_name) -> Result> { return std::make_shared( - fs, table_path, table_schema, MergeOptions(table_schema, dynamic_options)); + fs, table_path, table_schema, data_table_name, + MergeOptions(table_schema, dynamic_options)); }}, {ReadOptimizedSystemTable::kName, [](const std::shared_ptr& fs, const std::string& table_path, const std::shared_ptr& table_schema, - const std::map& dynamic_options) - -> Result> { + const std::map& dynamic_options, + const std::string& data_table_name) -> Result> { return std::make_shared( - fs, table_path, table_schema, MergeOptions(table_schema, dynamic_options)); + fs, table_path, table_schema, data_table_name, + MergeOptions(table_schema, dynamic_options)); }}, {SnapshotsSystemTable::kName, [](const std::shared_ptr& fs, const std::string& table_path, const std::shared_ptr& table_schema, - const std::map& dynamic_options) - -> Result> { + const std::map& dynamic_options, + const std::string& /*data_table_name*/) -> Result> { auto options = MergeOptions(table_schema, dynamic_options); return std::make_shared(fs, table_path, LoadBranch(options)); }}, {SchemasSystemTable::kName, [](const std::shared_ptr& fs, const std::string& table_path, const std::shared_ptr& table_schema, - const std::map& dynamic_options) - -> Result> { + const std::map& dynamic_options, + const std::string& /*data_table_name*/) -> Result> { auto options = MergeOptions(table_schema, dynamic_options); return std::make_shared(fs, table_path, LoadBranch(options)); }}, {TagsSystemTable::kName, [](const std::shared_ptr& fs, const std::string& table_path, const std::shared_ptr& table_schema, - const std::map& dynamic_options) - -> Result> { + const std::map& dynamic_options, + const std::string& /*data_table_name*/) -> Result> { auto options = MergeOptions(table_schema, dynamic_options); return std::make_shared(fs, table_path, LoadBranch(options)); }}, {BranchesSystemTable::kName, [](const std::shared_ptr& fs, const std::string& table_path, const std::shared_ptr& table_schema, - const std::map& dynamic_options) - -> Result> { + const std::map& dynamic_options, + const std::string& /*data_table_name*/) -> Result> { auto options = MergeOptions(table_schema, dynamic_options); return std::make_shared(fs, table_path, LoadBranch(options)); }}, {ConsumersSystemTable::kName, [](const std::shared_ptr& fs, const std::string& table_path, const std::shared_ptr& table_schema, - const std::map& dynamic_options) - -> Result> { + const std::map& dynamic_options, + const std::string& /*data_table_name*/) -> Result> { auto options = MergeOptions(table_schema, dynamic_options); return std::make_shared(fs, table_path, LoadBranch(options)); }}, {ManifestsSystemTable::kName, [](const std::shared_ptr& fs, const std::string& table_path, const std::shared_ptr& table_schema, - const std::map& dynamic_options) - -> Result> { + const std::map& dynamic_options, + const std::string& /*data_table_name*/) -> Result> { auto options = MergeOptions(table_schema, dynamic_options); auto branch = LoadBranch(options); return std::make_shared(fs, table_path, std::move(branch), @@ -160,8 +163,8 @@ const std::vector& SystemTableRegistry() { {FilesSystemTable::kName, [](const std::shared_ptr& fs, const std::string& table_path, const std::shared_ptr& table_schema, - const std::map& dynamic_options) - -> Result> { + const std::map& dynamic_options, + const std::string& /*data_table_name*/) -> Result> { auto options = MergeOptions(table_schema, dynamic_options); auto branch = LoadBranch(options); return std::make_shared(fs, table_path, std::move(branch), @@ -189,8 +192,8 @@ SystemTable::SystemTable(const std::shared_ptr& fs, const std::strin Identifier(Identifier::kUnknownDatabase, "system_table"), /*uuid=*/""), name_(std::move(name)) {} -std::string SystemTable::ComposeName(const std::string& table_path, const std::string& kind) { - return PathUtil::GetName(table_path) + Catalog::SYSTEM_TABLE_SPLITTER + kind; +std::string SystemTable::ComposeName(const std::string& data_table_name, const std::string& kind) { + return data_table_name + Catalog::SYSTEM_TABLE_SPLITTER + kind; } Status SystemTable::CheckNoSearch(const std::shared_ptr& context) { @@ -233,11 +236,16 @@ bool SystemTableLoader::IsSupported(const std::string& system_table_name) { Result> SystemTableLoader::Load( const std::string& system_table_name, const std::shared_ptr& fs, const std::string& table_path, const std::shared_ptr& table_schema, - const std::map& dynamic_options) { + const std::map& dynamic_options, const std::string& data_table_name) { std::optional factory = FindSystemTableFactory(system_table_name); if (factory) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, - factory.value()(fs, table_path, table_schema, dynamic_options)); + // A caller that knows no logical name for the data table - a load from a bare path with no + // branch - leaves the views to answer with the directory name, as before. + const std::string effective_name = + data_table_name.empty() ? PathUtil::GetName(table_path) : data_table_name; + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr system_table, + factory.value()(fs, table_path, table_schema, dynamic_options, effective_name)); PAIMON_RETURN_NOT_OK(system_table->AdoptOwnSchema()); return system_table; } @@ -307,7 +315,14 @@ Result> SystemTableLoader::LoadFromPath( if (parsed.branch) { options[Options::BRANCH] = parsed.branch.value(); } - return Load(parsed.system_table_name, fs, parsed.table_path, latest_schema.value(), options); + // The logical name of the data table this path addresses, branch included: a `T$branch_dev$ro` + // path names the view `T$branch_dev$ro`, not the `T$ro` the bare directory name would give + // it. The branch constructor folds a spelling of the main branch into the bare name. + const Identifier data_identifier(Identifier::kUnknownDatabase, + PathUtil::GetName(parsed.table_path), + parsed.branch.value_or(std::string())); + return Load(parsed.system_table_name, fs, parsed.table_path, latest_schema.value(), options, + data_identifier.GetObjectName()); } } // namespace paimon diff --git a/src/paimon/core/table/system/system_table.h b/src/paimon/core/table/system/system_table.h index 640672236..3b93feb5a 100644 --- a/src/paimon/core/table/system/system_table.h +++ b/src/paimon/core/table/system/system_table.h @@ -85,14 +85,18 @@ class SystemTable : public Table, public std::enable_shared_from_this& fs, const std::string& table_path, std::string name); - /// Composes the name of a table-scoped system table. Java Paimon's `OptionsTable.name()` - /// composes it the same way: `location.getName() + SYSTEM_TABLE_SPLITTER + OPTIONS`. - static std::string ComposeName(const std::string& table_path, const std::string& kind); + /// Composes the name of a table-scoped system table out of the logical name of the data table + /// underneath - the object name a catalog knows it by, branch included - and the kind. Java + /// Paimon's `OptionsTable.name()` composes it the same way, from `location.getName()`; the + /// metadata tables answer with the directory name, while the views that serve the data table + /// answer with its logical name, the way Java's `ReadOptimizedTable` answers with the name of + /// the table it wraps. + static std::string ComposeName(const std::string& data_table_name, const std::string& kind); /// Refuses a read whose context asks for a search: a root TopK request or a `FullTextMatch` /// predicate. Neither is evaluated on the rows a system table serves, so answering one @@ -120,7 +124,8 @@ class SystemTableLoader { static Result> Load( const std::string& system_table_name, const std::shared_ptr& fs, const std::string& table_path, const std::shared_ptr& table_schema, - const std::map& dynamic_options); + const std::map& dynamic_options, + const std::string& data_table_name = ""); static Result> TryParsePath(const std::string& path); diff --git a/src/paimon/core/table/system/system_table_test.cpp b/src/paimon/core/table/system/system_table_test.cpp index 4e8a03516..79f4c6cca 100644 --- a/src/paimon/core/table/system/system_table_test.cpp +++ b/src/paimon/core/table/system/system_table_test.cpp @@ -89,11 +89,11 @@ TEST(SystemTableTest, TestChangelogArrowSchemaReturnsInvalidOptions) { ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, CreateTableSchemaForTest(options)); - AuditLogSystemTable audit_log(/*fs=*/nullptr, "/tmp/table", table_schema, options); + AuditLogSystemTable audit_log(/*fs=*/nullptr, "/tmp/table", table_schema, "table", options); ASSERT_NOK_WITH_MSG(audit_log.ArrowSchema(), "Invalid Config [table-read.sequence-number.enabled: invalid]"); - BinlogSystemTable binlog(/*fs=*/nullptr, "/tmp/table", table_schema, options); + BinlogSystemTable binlog(/*fs=*/nullptr, "/tmp/table", table_schema, "table", options); ASSERT_NOK_WITH_MSG(binlog.ArrowSchema(), "Invalid Config [table-read.sequence-number.enabled: invalid]"); } @@ -104,7 +104,7 @@ TEST(SystemTableTest, TestBinlogArrowSchemaWithSequenceNumber) { ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, CreateTableSchemaForTest(options)); - BinlogSystemTable binlog(/*fs=*/nullptr, "/tmp/table", table_schema, options); + BinlogSystemTable binlog(/*fs=*/nullptr, "/tmp/table", table_schema, "table", options); ASSERT_OK_AND_ASSIGN(std::shared_ptr schema, binlog.ArrowSchema()); ASSERT_EQ(schema->field_names(), @@ -233,6 +233,14 @@ TEST(SystemTableTest, TestReadOptimizedSystemTableRegistration) { /*dynamic_options=*/{})); ASSERT_EQ(system_table->Name(), "table$ro"); + // A view loaded with the logical name of its data table answers with it, branch included: + // reloading the name a branch view answers with addresses the same branch again. + ASSERT_OK_AND_ASSIGN(std::shared_ptr branch_view, + SystemTableLoader::Load(ReadOptimizedSystemTable::kName, /*fs=*/nullptr, + "/tmp/uuid-dir", table_schema, + /*dynamic_options=*/{}, "tbl$branch_dev")); + ASSERT_EQ(branch_view->Name(), "tbl$branch_dev$ro"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr arrow_schema, system_table->ArrowSchema()); ASSERT_EQ(arrow_schema->field_names(), (std::vector{"pk", "v"})); ASSERT_EQ(arrow_schema->field(0)->type()->id(), arrow::Type::STRING); @@ -271,9 +279,10 @@ TEST(SystemTableTest, TestNewReadPropagatesWarmupLevel) { ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, CreateTableSchemaForTest(options)); - AuditLogSystemTable audit_log(/*fs=*/nullptr, "/tmp/table", table_schema, options); - BinlogSystemTable binlog(/*fs=*/nullptr, "/tmp/table", table_schema, options); - ReadOptimizedSystemTable read_optimized(/*fs=*/nullptr, "/tmp/table", table_schema, options); + AuditLogSystemTable audit_log(/*fs=*/nullptr, "/tmp/table", table_schema, "table", options); + BinlogSystemTable binlog(/*fs=*/nullptr, "/tmp/table", table_schema, "table", options); + ReadOptimizedSystemTable read_optimized(/*fs=*/nullptr, "/tmp/table", table_schema, "table", + options); for (WarmupLevel level : {WarmupLevel::NONE, WarmupLevel::RAW, WarmupLevel::DECODED}) { ReadContextBuilder builder("/tmp/table"); @@ -317,9 +326,10 @@ TEST(SystemTableTest, TestOperatorEntriesRefuseANullContext) { ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, CreateTableSchemaForTest(options)); - AuditLogSystemTable audit_log(/*fs=*/nullptr, "/tmp/table", table_schema, options); - BinlogSystemTable binlog(/*fs=*/nullptr, "/tmp/table", table_schema, options); - ReadOptimizedSystemTable read_optimized(/*fs=*/nullptr, "/tmp/table", table_schema, options); + AuditLogSystemTable audit_log(/*fs=*/nullptr, "/tmp/table", table_schema, "table", options); + BinlogSystemTable binlog(/*fs=*/nullptr, "/tmp/table", table_schema, "table", options); + ReadOptimizedSystemTable read_optimized(/*fs=*/nullptr, "/tmp/table", table_schema, "table", + options); ASSERT_NOK_WITH_MSG(audit_log.NewScan(nullptr), "scan context is null pointer"); ASSERT_NOK_WITH_MSG(binlog.NewScan(nullptr), "scan context is null pointer"); @@ -355,9 +365,10 @@ TEST(SystemTableTest, TestNewReadRefusesASearch) { ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, CreateTableSchemaForTest(options)); - AuditLogSystemTable audit_log(/*fs=*/nullptr, "/tmp/table", table_schema, options); - BinlogSystemTable binlog(/*fs=*/nullptr, "/tmp/table", table_schema, options); - ReadOptimizedSystemTable read_optimized(/*fs=*/nullptr, "/tmp/table", table_schema, options); + AuditLogSystemTable audit_log(/*fs=*/nullptr, "/tmp/table", table_schema, "table", options); + BinlogSystemTable binlog(/*fs=*/nullptr, "/tmp/table", table_schema, "table", options); + ReadOptimizedSystemTable read_optimized(/*fs=*/nullptr, "/tmp/table", table_schema, "table", + options); auto make_context = [] { auto vector_search = std::make_shared( @@ -390,7 +401,8 @@ TEST(SystemTableTest, TestNewReadRefusesAFullTextMatchPredicate) { SystemTableLoader::Load(OptionsSystemTable::kName, /*fs=*/nullptr, "/tmp/table", table_schema, /*dynamic_options=*/{})); - ReadOptimizedSystemTable read_optimized(/*fs=*/nullptr, "/tmp/table", table_schema, options); + ReadOptimizedSystemTable read_optimized(/*fs=*/nullptr, "/tmp/table", table_schema, "table", + options); auto match = std::make_shared("pk", R"({"match":{"query":"apple"}})"); ASSERT_OK_AND_ASSIGN(std::shared_ptr predicate, diff --git a/src/paimon/core/table/table_test.cpp b/src/paimon/core/table/table_test.cpp index 43065e16b..ce896d83a 100644 --- a/src/paimon/core/table/table_test.cpp +++ b/src/paimon/core/table/table_test.cpp @@ -72,8 +72,7 @@ Result> CreateManagedTable( schema_manager.CreateTable(schema, /*partition_keys=*/{}, /*primary_keys=*/{}, options)); return FileStoreTable::Create( - file_system, path, - checked_pointer_cast(std::shared_ptr(std::move(created))), identifier, + file_system, path, std::shared_ptr(std::move(created)), identifier, /*uuid=*/"", branch, /*fs_scheme_map=*/{}, /*dynamic_options=*/{}); } @@ -181,10 +180,17 @@ TEST(TableTest, TestCopyKeepsTheBranchTheTableServes) { CreateManagedTable(dir->GetFileSystem(), dir->Str(), identifier, /*branch=*/"main", {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); - ASSERT_OK_AND_ASSIGN(std::shared_ptr
copy, table->Copy({{Options::BRANCH, "dev"}})); + // A copy whose `branch` option names the branch the table serves stays on it, the option + // folded away, and still takes the other dynamic options. + ASSERT_OK_AND_ASSIGN(std::shared_ptr
copy, + table->Copy({{Options::BRANCH, "main"}, {Options::BUCKET, "2"}})); auto file_store_copy = std::dynamic_pointer_cast(copy); ASSERT_TRUE(file_store_copy != nullptr); EXPECT_EQ(file_store_copy->CurrentBranch(), "main"); + std::shared_ptr copied_schema = + std::dynamic_pointer_cast(copy->LatestSchema()); + ASSERT_TRUE(copied_schema != nullptr); + EXPECT_EQ(copied_schema->Options().at(Options::BUCKET), "2"); { ScanContextBuilder scan_builder(dir->Str()); scan_builder.AddOption(Options::BRANCH, "main"); @@ -198,7 +204,7 @@ TEST(TableTest, TestCopyKeepsTheBranchTheTableServes) { identifier, /*branch=*/"dev", {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); ASSERT_OK_AND_ASSIGN(std::shared_ptr
branch_copy, - branch_table->Copy({{Options::BRANCH, "main"}})); + branch_table->Copy({{Options::BRANCH, "dev"}})); auto branch_file_store_copy = std::dynamic_pointer_cast(branch_copy); ASSERT_TRUE(branch_file_store_copy != nullptr); EXPECT_EQ(branch_file_store_copy->CurrentBranch(), "dev"); @@ -211,6 +217,50 @@ TEST(TableTest, TestCopyKeepsTheBranchTheTableServes) { } } +TEST(TableTest, TestCopySwitchesToTheBranchItNames) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::shared_ptr fs = dir->GetFileSystem(); + const std::string path = dir->Str(); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + CreateManagedTable(fs, path, identifier, /*branch=*/"main", + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + // The dev branch holds its own schema, told apart from the main branch's by its bucket. + SchemaManager dev_schema_manager(fs, path, "dev"); + auto schema = arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false)}); + ASSERT_OK_AND_ASSIGN( + [[maybe_unused]] std::unique_ptr dev_schema, + dev_schema_manager.CreateTable(schema, /*partition_keys=*/{}, /*primary_keys=*/{}, + {{Options::BUCKET, "4"}, {Options::BUCKET_KEY, "id"}})); + + // A copy on another branch serves that branch's own latest schema, addressed as its table. + ASSERT_OK_AND_ASSIGN(std::shared_ptr
dev_copy, table->Copy({{Options::BRANCH, "dev"}})); + auto dev_file_store_copy = std::dynamic_pointer_cast(dev_copy); + ASSERT_TRUE(dev_file_store_copy != nullptr); + EXPECT_EQ("dev", dev_file_store_copy->CurrentBranch()); + EXPECT_EQ("tbl$branch_dev", dev_copy->Name()); + std::shared_ptr dev_copied_schema = + std::dynamic_pointer_cast(dev_copy->LatestSchema()); + ASSERT_TRUE(dev_copied_schema != nullptr); + EXPECT_EQ(dev_copied_schema->Options().at(Options::BUCKET), "4"); + EXPECT_EQ(dev_copied_schema->Options().at(Options::BRANCH), "dev"); + EXPECT_EQ("dev", dev_file_store_copy->GetSnapshotManager()->Branch()); + + // The copy's other dynamic options land on the branch's schema on top. + ASSERT_OK_AND_ASSIGN(std::shared_ptr
dev_copy_restyled, + table->Copy({{Options::BRANCH, "dev"}, {Options::BUCKET, "2"}})); + std::shared_ptr restyled_schema = + std::dynamic_pointer_cast(dev_copy_restyled->LatestSchema()); + ASSERT_TRUE(restyled_schema != nullptr); + EXPECT_EQ(restyled_schema->Options().at(Options::BUCKET), "2"); + EXPECT_EQ(restyled_schema->Options().at(Options::BUCKET_KEY), "id"); + + // A branch holding no schema does not exist. + ASSERT_NOK_WITH_MSG(table->Copy({{Options::BRANCH, "missing"}}), + "branch missing does not exist"); +} + TEST(TableTest, TestACopyHandsItsDynamicOptionsToTheFallbackReader) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); @@ -232,13 +282,11 @@ TEST(TableTest, TestACopyHandsItsDynamicOptionsToTheFallbackReader) { fallback_schema_manager.CreateTable(schema, /*partition_keys=*/{}, /*primary_keys=*/{"id"}, {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); - ASSERT_OK_AND_ASSIGN( - std::shared_ptr
table, - FileStoreTable::Create( - fs, path, - checked_pointer_cast(std::shared_ptr(std::move(created))), - identifier, /*uuid=*/"", /*branch=*/"main", /*fs_scheme_map=*/{}, - /*dynamic_options=*/{})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + FileStoreTable::Create( + fs, path, std::shared_ptr(std::move(created)), identifier, + /*uuid=*/"", /*branch=*/"main", /*fs_scheme_map=*/{}, + /*dynamic_options=*/{})); ASSERT_OK_AND_ASSIGN(std::shared_ptr
copy, table->Copy({{Options::KEY_VALUE_SEQUENCE_NUMBER_ENABLED, "true"}})); @@ -249,6 +297,39 @@ TEST(TableTest, TestACopyHandsItsDynamicOptionsToTheFallbackReader) { copy->NewRead(std::move(read_context))); } +TEST(TableTest, TestAReadContextMatchingTheSchemaKeepsItsOptionsForTheFallbackReader) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::shared_ptr fs = dir->GetFileSystem(); + const std::string path = dir->Str() + "/tbl"; + + // The main schema already carries the sequence-number setting, so a read context repeating + // it with the same value does not restyle the load; the setting still has to reach the + // fallback branch, whose schema omits it, or that reader refuses the sequence number. + SchemaManager schema_manager(fs, path, "main"); + auto schema = arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false)}); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr created, + schema_manager.CreateTable(schema, /*partition_keys=*/{}, /*primary_keys=*/{"id"}, + {{Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "id"}, + {Options::SCAN_FALLBACK_BRANCH, "fb"}, + {Options::KEY_VALUE_SEQUENCE_NUMBER_ENABLED, "true"}})); + SchemaManager fallback_schema_manager(fs, path, "fb"); + ASSERT_OK_AND_ASSIGN( + [[maybe_unused]] std::unique_ptr fallback_schema, + fallback_schema_manager.CreateTable(schema, /*partition_keys=*/{}, + /*primary_keys=*/{"id"}, + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + + ReadContextBuilder read_builder(path); + read_builder.AddOption(Options::KEY_VALUE_SEQUENCE_NUMBER_ENABLED, "true"); + read_builder.SetReadFieldNames({"id", "_SEQUENCE_NUMBER"}); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr read, + TableRead::Create(std::move(read_context))); +} + TEST(TableTest, TestACopyOfACopyServesTheLatestValueOfARepeatedDynamicOption) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); @@ -270,13 +351,11 @@ TEST(TableTest, TestACopyOfACopyServesTheLatestValueOfARepeatedDynamicOption) { fallback_schema_manager.CreateTable(schema, /*partition_keys=*/{}, /*primary_keys=*/{"id"}, {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); - ASSERT_OK_AND_ASSIGN( - std::shared_ptr
table, - FileStoreTable::Create( - fs, path, - checked_pointer_cast(std::shared_ptr(std::move(created))), - identifier, /*uuid=*/"", /*branch=*/"main", /*fs_scheme_map=*/{}, - /*dynamic_options=*/{})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + FileStoreTable::Create( + fs, path, std::shared_ptr(std::move(created)), identifier, + /*uuid=*/"", /*branch=*/"main", /*fs_scheme_map=*/{}, + /*dynamic_options=*/{})); ASSERT_OK_AND_ASSIGN(std::shared_ptr
off, table->Copy({{Options::KEY_VALUE_SEQUENCE_NUMBER_ENABLED, "false"}})); @@ -307,11 +386,10 @@ TEST(TableTest, TestAFallbackReaderServesItsBranchsOwnSchema) { {Options::SCAN_FALLBACK_BRANCH, "fb"}})); ASSERT_OK_AND_ASSIGN( std::shared_ptr
main, - FileStoreTable::Create( - fs, path, - checked_pointer_cast(std::shared_ptr(std::move(main_created))), - identifier, /*uuid=*/"", /*branch=*/"main", /*fs_scheme_map=*/{}, - /*dynamic_options=*/{})); + FileStoreTable::Create(fs, path, std::shared_ptr(std::move(main_created)), + identifier, + /*uuid=*/"", /*branch=*/"main", /*fs_scheme_map=*/{}, + /*dynamic_options=*/{})); SchemaManager fallback_schema_manager(fs, path, "fb"); ASSERT_OK_AND_ASSIGN( @@ -321,11 +399,10 @@ TEST(TableTest, TestAFallbackReaderServesItsBranchsOwnSchema) { {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); ASSERT_OK_AND_ASSIGN( std::shared_ptr
fallback, - FileStoreTable::Create( - fs, path, - checked_pointer_cast(std::shared_ptr(std::move(fallback_created))), - identifier, /*uuid=*/"", /*branch=*/"fb", /*fs_scheme_map=*/{}, - /*dynamic_options=*/{})); + FileStoreTable::Create(fs, path, std::shared_ptr(std::move(fallback_created)), + identifier, + /*uuid=*/"", /*branch=*/"fb", /*fs_scheme_map=*/{}, + /*dynamic_options=*/{})); WriteAndCommitRow(fallback, path, /*id=*/1, /*v=*/10); WriteAndCommitRow(fallback, path, /*id=*/1, /*v=*/20); @@ -388,11 +465,10 @@ TEST(TableTest, TestAFallbackReaderServesItsBranchsOwnPartitionKeys) { {{Options::SCAN_FALLBACK_BRANCH, "fb"}})); ASSERT_OK_AND_ASSIGN( std::shared_ptr
main, - FileStoreTable::Create( - fs, path, - checked_pointer_cast(std::shared_ptr(std::move(main_created))), - identifier, /*uuid=*/"", /*branch=*/"main", /*fs_scheme_map=*/{}, - /*dynamic_options=*/{})); + FileStoreTable::Create(fs, path, std::shared_ptr(std::move(main_created)), + identifier, + /*uuid=*/"", /*branch=*/"main", /*fs_scheme_map=*/{}, + /*dynamic_options=*/{})); SchemaManager fallback_schema_manager(fs, path, "fb"); auto row_schema = arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false), @@ -404,11 +480,10 @@ TEST(TableTest, TestAFallbackReaderServesItsBranchsOwnPartitionKeys) { {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); ASSERT_OK_AND_ASSIGN( std::shared_ptr
fallback, - FileStoreTable::Create( - fs, path, - checked_pointer_cast(std::shared_ptr(std::move(fallback_created))), - identifier, /*uuid=*/"", /*branch=*/"fb", /*fs_scheme_map=*/{}, - /*dynamic_options=*/{})); + FileStoreTable::Create(fs, path, std::shared_ptr(std::move(fallback_created)), + identifier, + /*uuid=*/"", /*branch=*/"fb", /*fs_scheme_map=*/{}, + /*dynamic_options=*/{})); WriteAndCommitRow(fallback, path, /*id=*/1, /*v=*/20); ReadContextBuilder read_builder(path); @@ -452,7 +527,7 @@ TEST(TableTest, TestAFallbackReaderServesItsBranchsOwnPartitionKeys) { EXPECT_TRUE(BatchReader::IsEofBatch(eof_batch)); } -TEST(TableTest, TestAFallbackReaderIgnoresReadContextOptionsAbsentFromTheTable) { +TEST(TableTest, TestAFallbackReaderRefusesReadContextOptionsAbsentFromTheTable) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); const std::shared_ptr fs = dir->GetFileSystem(); @@ -472,11 +547,10 @@ TEST(TableTest, TestAFallbackReaderIgnoresReadContextOptionsAbsentFromTheTable) fallback_schema_manager.CreateTable(schema, /*partition_keys=*/{}, /*primary_keys=*/{}, {})); } - return FileStoreTable::Create( - fs, path, - checked_pointer_cast(std::shared_ptr(std::move(created))), - Identifier("db", "tbl"), /*uuid=*/"", /*branch=*/"main", /*fs_scheme_map=*/{}, - /*dynamic_options=*/{}); + return FileStoreTable::Create(fs, path, std::shared_ptr(std::move(created)), + Identifier("db", "tbl"), + /*uuid=*/"", /*branch=*/"main", /*fs_scheme_map=*/{}, + /*dynamic_options=*/{}); }; ASSERT_OK_AND_ASSIGN(std::shared_ptr
plain, make_table(dir->Str() + "/plain", {})); ASSERT_OK_AND_ASSIGN( @@ -488,8 +562,11 @@ TEST(TableTest, TestAFallbackReaderIgnoresReadContextOptionsAbsentFromTheTable) ReadContextBuilder read_builder(table->GetRootPath()); read_builder.AddOption(Options::FILE_FORMAT, "bogus"); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr read, - table->NewRead(std::move(read_context))); + // `file.format` is a key `CoreOptions` knows, so neither the plain reader nor the + // fallback one may drop it in silence: both refuse it rather than read the table's own + // format behind the caller's back. + ASSERT_NOK_WITH_MSG(table->NewRead(std::move(read_context)), + "the table was not loaded with it"); } } @@ -585,7 +662,7 @@ TEST(TableTest, TestSwitchToBranch) { ASSERT_NOK_WITH_MSG(main_file_store_table->SwitchToBranch("missing"), "branch"); } -TEST(TableTest, TestSwitchToBranchFromATableThatHasNoSchemaOfItsOwn) { +TEST(TableTest, TestCreateRefusesATableWithoutASchema) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); const std::string path = dir->Str() + "/tbl"; @@ -595,24 +672,13 @@ TEST(TableTest, TestSwitchToBranchFromATableThatHasNoSchemaOfItsOwn) { dev_schema_manager.CreateTable(arrow::schema({arrow::field("c0", arrow::int32())}), /*partition_keys=*/{}, /*primary_keys=*/{}, {})); - // A table opened over a path its own branch holds no schema under, and opened without a file - // system of its own: its operators report that no schema was found, while a branch that has one - // is read through the default file system the store falls back to and served on its own. - ASSERT_OK_AND_ASSIGN( - std::shared_ptr table, - FileStoreTable::Create(/*file_system=*/nullptr, path, /*schema=*/nullptr, - Identifier("db", "tbl"), /*uuid=*/"", - /*branch=*/BranchManager::DEFAULT_MAIN_BRANCH, /*fs_scheme_map=*/{}, - /*dynamic_options=*/{})); - ASSERT_OK_AND_ASSIGN(std::shared_ptr
dev_table, table->SwitchToBranch("dev")); - auto dev_file_store_table = std::dynamic_pointer_cast(dev_table); - ASSERT_TRUE(dev_file_store_table != nullptr); - EXPECT_EQ("dev", dev_file_store_table->CurrentBranch()); - EXPECT_EQ("tbl$branch_dev", dev_table->Name()); - ASSERT_EQ((std::vector{"c0"}), dev_table->LatestSchema()->FieldNames()); - ASSERT_TRUE(dev_file_store_table->GetSchemaManager() != nullptr); - ASSERT_TRUE(dev_file_store_table->GetSnapshotManager() != nullptr); - EXPECT_EQ("dev", dev_file_store_table->GetSnapshotManager()->Branch()); + // A path its own branch holds no schema under names no table: the call that opens one is + // refused, whatever another branch of it holds. + ASSERT_NOK_WITH_MSG(FileStoreTable::Create(/*file_system=*/nullptr, path, /*schema=*/nullptr, + Identifier("db", "tbl"), /*uuid=*/"", + /*branch=*/BranchManager::DEFAULT_MAIN_BRANCH, + /*fs_scheme_map=*/{}, /*dynamic_options=*/{}), + "not found latest schema of db.tbl"); } TEST(TableTest, TestNewWriteServesTheBranchTheTableWasLoadedWith) { @@ -790,7 +856,7 @@ TEST(TableTest, TestMainBranchAliasResolvesToTheMainBranch) { ASSERT_OK_AND_ASSIGN(std::unique_ptr created, schema_manager.CreateTable( arrow::schema({arrow::field("c0", arrow::int32())}), {}, {}, {})); - auto schema = checked_pointer_cast(std::shared_ptr(std::move(created))); + auto schema = std::shared_ptr(std::move(created)); for (const auto& main_identifier : {Identifier("db", "tbl"), Identifier("db", "tbl$branch_main"), @@ -833,7 +899,7 @@ TEST(TableTest, TestCatalogOpenedTableServesThePerTableFileSystem) { CreateManagedTable(dir->GetFileSystem(), dir->Str(), identifier)); auto catalog = std::make_shared(); - catalog->SetTableSchema(table->LatestSchema()); + catalog->SetTableSchema(checked_pointer_cast(table->LatestSchema())); ASSERT_OK_AND_ASSIGN(std::shared_ptr
opened, catalog->GetTable(identifier)); EXPECT_NE(opened->GetFileSystem(), nullptr); @@ -1057,7 +1123,7 @@ TEST(TableTest, TestNewWriteCreatesAFileStoreWrite) { auto catalog = std::make_shared(); catalog->SetFileSystem(dir->GetFileSystem()); // A catalog write reads the current schema from the catalog on the main branch. - catalog->SetTableSchema(table->LatestSchema()); + catalog->SetTableSchema(checked_pointer_cast(table->LatestSchema())); ASSERT_OK_AND_ASSIGN( std::unique_ptr ctx, @@ -1076,7 +1142,7 @@ TEST(TableTest, TestNewCommitCreatesAFileStoreCommit) { auto catalog = std::make_shared(); catalog->SetFileSystem(dir->GetFileSystem()); // Like a write, a catalog commit reads the current schema from the catalog. - catalog->SetTableSchema(table->LatestSchema()); + catalog->SetTableSchema(checked_pointer_cast(table->LatestSchema())); ASSERT_OK_AND_ASSIGN( std::unique_ptr ctx, @@ -1108,7 +1174,7 @@ TEST(TableTest, TestNewWriteAndCommitRefuseAContextThatNamesAnotherTable) { auto catalog = std::make_shared(); catalog->SetFileSystem(fs); - catalog->SetTableSchema(table->LatestSchema()); + catalog->SetTableSchema(checked_pointer_cast(table->LatestSchema())); { WriteContextBuilder write_builder(dir->Str(), "commit-user"); write_builder.WithCatalog(catalog, Identifier("db", "other")); @@ -1161,7 +1227,7 @@ TEST(TableTest, TestNewCommitRefusesABranchACatalogCannotAddress) { auto catalog = std::make_shared(); catalog->SetFileSystem(fs); - catalog->SetTableSchema(table->LatestSchema()); + catalog->SetTableSchema(checked_pointer_cast(table->LatestSchema())); WriteContextBuilder write_builder(dir->Str(), "commit-user"); write_builder.WithCatalog(catalog, identifier); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_ctx, write_builder.Finish()); @@ -1229,7 +1295,7 @@ TEST(TableTest, TestOperatorsRefuseAnOptionThatContradictsTheTable) { copied->NewScan(std::move(ctx))); } -TEST(TableTest, TestOperatorsIgnoreARuntimeKeyTheTableWasNotLoadedWith) { +TEST(TableTest, TestOperatorsRefuseAnOptionTheTableWasNotLoadedWith) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); const Identifier identifier("db", "tbl"); @@ -1239,22 +1305,51 @@ TEST(TableTest, TestOperatorsIgnoreARuntimeKeyTheTableWasNotLoadedWith) { {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); WriteAndCommitTwoRows(table, dir->Str()); - ScanContextBuilder scan_builder(dir->Str()); - scan_builder.AddOption(Options::SCAN_MODE, "from-snapshot") - .AddOption(Options::SCAN_SNAPSHOT_ID, "999"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_ctx, scan_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(scan_ctx))); - ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); - ASSERT_FALSE(plan->Splits().empty()); - ASSERT_EQ(plan->SnapshotId(), std::optional(1)); + // The operators serve the configuration the table was loaded with and nothing else, so a + // context option that configuration does not carry - a `CoreOptions` key the schema never + // saved, or a key no one reads - is dropped in silence if it is not refused: refused on every + // side, whatever the key. + const std::string dropped_in_silence = "the table was not loaded with it"; + { + ScanContextBuilder scan_builder(dir->Str()); + scan_builder.AddOption(Options::SCAN_MODE, "from-snapshot") + .AddOption(Options::SCAN_SNAPSHOT_ID, "999"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_ctx, scan_builder.Finish()); + ASSERT_NOK_WITH_MSG(table->NewScan(std::move(scan_ctx)), dropped_in_silence); + } + { + ReadContextBuilder read_builder(dir->Str()); + read_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "999"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_ctx, read_builder.Finish()); + ASSERT_NOK_WITH_MSG(table->NewRead(std::move(read_ctx)), dropped_in_silence); + } + { + // The commit-lease switch a writer sets on a table whose schema does not carry it is the + // same case: refused rather than committed without the lease. + CommitContextBuilder commit_builder(dir->Str(), "commit-user"); + commit_builder.AddOption(Options::REST_COMMIT_ACQUIRE_LOCK, "true"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_ctx, commit_builder.Finish()); + ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(commit_ctx)), dropped_in_silence); + } + { + // A key no `CoreOptions` reads is refused the same way: whether it names a setting the + // table would drop or one the caller only mistyped, the table was not loaded with it. + ReadContextBuilder unknown_builder(dir->Str()); + unknown_builder.AddOption("some.option-nothing-reads", "value"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr unknown_ctx, unknown_builder.Finish()); + ASSERT_NOK_WITH_MSG(table->NewRead(std::move(unknown_ctx)), dropped_in_silence); + } - ReadContextBuilder read_builder(dir->Str()); - read_builder.AddOption(Options::SCAN_MODE, "from-snapshot") - .AddOption(Options::SCAN_SNAPSHOT_ID, "999"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr read_ctx, read_builder.Finish()); - ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr read, - table->NewRead(std::move(read_ctx))); + // A `header.` key a one-step load folds into the table is the same case held against a table + // that was not loaded with it: the table was not, whatever the key names downstream. + ReadContextBuilder header_builder(dir->Str()); + header_builder.AddOption("header.X-Paimon-Read-Via", "table-view"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr header_ctx, header_builder.Finish()); + ASSERT_NOK_WITH_MSG(table->NewRead(std::move(header_ctx)), dropped_in_silence); + // The way to serve an option is a table loaded with it: `Copy()` merges it into the schema, + // and the copy applies it - which then fails on the snapshot it names, proving the option + // reached the scan instead of being dropped. ASSERT_OK_AND_ASSIGN( std::shared_ptr
copied, table->Copy({{Options::SCAN_MODE, "from-snapshot"}, {Options::SCAN_SNAPSHOT_ID, "999"}})); @@ -1275,7 +1370,7 @@ TEST(TableTest, TestCreateFailsWhenTheSchemaOptionsDoNotParse) { arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false)}), /*partition_keys=*/{}, /*primary_keys=*/{}, {{Options::TARGET_FILE_ROW_NUM, "0"}})); - auto schema = checked_pointer_cast(std::shared_ptr(std::move(created))); + auto schema = std::shared_ptr(std::move(created)); ASSERT_NOK_WITH_MSG( FileStoreTable::Create(dir->GetFileSystem(), dir->Str(), schema, Identifier("db", "tbl"), /*uuid=*/"", /*branch=*/BranchManager::DEFAULT_MAIN_BRANCH, @@ -1340,8 +1435,13 @@ TEST(TableTest, TestCopyServesTheTableWithTheGivenOptionsOnTop) { ASSERT_OK(commit->Commit(messages)); ASSERT_OK(write->Close()); + // The `type` option is immutable: a value naming another type is refused, the way Java + // Paimon's copy refuses it, and one naming the type the table already serves - whatever its + // case - is dropped. + ASSERT_NOK_WITH_MSG(table->Copy({{Options::BUCKET, "2"}, {Options::TYPE, "format-table"}}), + "Change 'type' is not supported yet."); ASSERT_OK_AND_ASSIGN(std::shared_ptr
copied, - table->Copy({{Options::BUCKET, "2"}, {Options::TYPE, "format-table"}})); + table->Copy({{Options::BUCKET, "2"}, {Options::TYPE, "TABLE"}})); std::shared_ptr copied_schema = std::dynamic_pointer_cast(copied->LatestSchema()); ASSERT_TRUE(copied_schema != nullptr); @@ -1483,7 +1583,7 @@ TEST(TableTest, TestCreateBuildsTheOperator) { auto catalog = std::make_shared(); catalog->SetFileSystem(dir->GetFileSystem()); // A catalog write and commit read the current schema from the catalog on the main branch. - catalog->SetTableSchema(table->LatestSchema()); + catalog->SetTableSchema(checked_pointer_cast(table->LatestSchema())); // `TableScan::Create()` and friends build the operator straight from the context, which // carries the catalog and identifier it resolves the table through. @@ -1519,7 +1619,7 @@ TEST(TableTest, TestCreateWithSpecificTableSchemaSkipsTheCatalogSchema) { auto catalog = std::make_shared(); catalog->SetFileSystem(dir->GetFileSystem()); - catalog->SetTableSchema(table->LatestSchema()); + catalog->SetTableSchema(checked_pointer_cast(table->LatestSchema())); // A schema the caller gave replaces the catalog's on the main branch, so neither the write // nor the commit asks the catalog for one. @@ -1551,7 +1651,7 @@ TEST(TableTest, TestCreateWithCatalogReadsTheMainBranchSchemaFromIt) { auto catalog = std::make_shared(); catalog->SetFileSystem(dir->GetFileSystem()); - catalog->SetTableSchema(table->LatestSchema()); + catalog->SetTableSchema(checked_pointer_cast(table->LatestSchema())); // All four loads take the schema the catalog holds on the main branch when the caller did // not set one, so each of them asks the catalog exactly once. diff --git a/src/paimon/rest/rest_catalog.cpp b/src/paimon/rest/rest_catalog.cpp index af144f8a0..af8c50623 100644 --- a/src/paimon/rest/rest_catalog.cpp +++ b/src/paimon/rest/rest_catalog.cpp @@ -428,8 +428,13 @@ Result> RestCatalog::LoadSystemTable( } PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table_file_system, GetTableFileSystem(identifier)); + // The data table's logical name, branch included, so that a view of a table the REST store + // keeps under a UUID directory still answers with the name the catalog knows it by. + PAIMON_ASSIGN_OR_RAISE( + Identifier data_identifier, + CatalogUtils::BranchIdentifier(identifier, branch.value_or(std::string()))); return SystemTableLoader::Load(system_table_name.value(), table_file_system, data_table_path, - latest_schema, dynamic_options); + latest_schema, dynamic_options, data_identifier.GetObjectName()); } Result> RestCatalog::GetTable(const Identifier& identifier) const { @@ -460,10 +465,16 @@ Result> RestCatalog::GetTable(const Identifier& identifie return LoadSystemTable(identifier, load_identifier, NormalizeBranch(std::move(branch))); } // Read UUID, path and schema together to avoid mixing table generations after recreation. + // The schema is read through the data-table path `LoadTableSchema` takes for one, so it + // arrives as the `TableSchema` the table below is built with, over the branch the identifier + // names - the same derivation that load makes. + PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); + branch = NormalizeBranch(std::move(branch)); + PAIMON_ASSIGN_OR_RAISE(Identifier load_identifier, ToLoadIdentifier(identifier)); std::string uuid; std::string table_path; - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, - LoadTableSchema(identifier, &uuid, &table_path)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, + LoadDataTableSchema(load_identifier, branch, &table_path, &uuid)); std::shared_ptr data_schema = std::dynamic_pointer_cast(schema); if (data_schema != nullptr) { PAIMON_ASSIGN_OR_RAISE(bool is_format_table, @@ -480,11 +491,9 @@ Result> RestCatalog::GetTable(const Identifier& identifie // tokens binds to this table. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table_file_system, GetTableFileSystem(identifier)); - PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); - return FileStoreTable::Create( - table_file_system, table_path, schema, identifier, uuid, - NormalizeBranch(std::move(branch)).value_or(Identifier::kDefaultMainBranch), - /*fs_scheme_map=*/{}, /*dynamic_options=*/{}); + return FileStoreTable::Create(table_file_system, table_path, schema, identifier, uuid, + branch.value_or(Identifier::kDefaultMainBranch), + /*fs_scheme_map=*/{}, /*dynamic_options=*/{}); } std::string RestCatalog::GetRootPath() const { diff --git a/src/paimon/testing/mock/mock_catalog.h b/src/paimon/testing/mock/mock_catalog.h index 2890330c3..8fa3b3fa8 100644 --- a/src/paimon/testing/mock/mock_catalog.h +++ b/src/paimon/testing/mock/mock_catalog.h @@ -34,6 +34,7 @@ #include "paimon/core/catalog/commit_lock.h" #include "paimon/core/catalog/version_managed_catalog.h" #include "paimon/core/partition/partition_statistics.h" +#include "paimon/core/schema/table_schema.h" #include "paimon/core/snapshot.h" #include "paimon/core/table/file_store_table.h" #include "paimon/result.h" @@ -113,16 +114,26 @@ class MockVersionManagedCatalog : public Catalog, public VersionManagedCatalog { Result> GetTable(const Identifier& identifier) const override { ++get_table_calls_; + // A managed table's schema is a `TableSchema`, while the one this mock was set may be a + // format table's `DataSchema` - served by `LoadTableSchema` alone - or none at all; + // `GetTable`, the managed kind's door, lets `FileStoreTable::Create` refuse both. + std::shared_ptr managed_schema = + std::dynamic_pointer_cast(table_schema_); + if (table_schema_ != nullptr && managed_schema == nullptr) { + return Status::Invalid(identifier.ToString() + " is not a managed table schema"); + } // Serves the per-table file system when one was set, matching `GetTableFileSystem` // without recording the request, so a test can still tell the two apart. No location: // this mock's `GetTableLocation` is the unsupported stub. return FileStoreTable::Create( table_file_system_ != nullptr ? table_file_system_ : file_system_, - /*root_path=*/"", table_schema_, identifier, table_uuid_, + /*root_path=*/"", managed_schema, identifier, table_uuid_, /*branch=*/BranchManager::DEFAULT_MAIN_BRANCH, /*fs_scheme_map=*/{}, /*dynamic_options=*/{}); } + /// Sets the table's schema, which `LoadTableSchema` serves either kind of: a managed table's + /// `TableSchema` - the one a load hands the table is - or a format table's `DataSchema`. void SetTableSchema(const std::shared_ptr& table_schema) { table_schema_ = table_schema; } From 76fc225b2ceb27d68dfd531eceaaff49f508a64d Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Fri, 30 Oct 2026 15:20:00 +0800 Subject: [PATCH 28/30] fix --- docs/source/api/format_table.rst | 4 +- include/paimon/commit_context.h | 24 ++- include/paimon/read_context.h | 16 +- include/paimon/scan_context.h | 16 +- include/paimon/write_context.h | 28 +++- .../core/catalog/file_system_catalog_test.cpp | 154 ++++++++++++++++++ src/paimon/core/operation/commit_context.cpp | 24 ++- .../core/operation/file_store_commit.cpp | 3 + .../core/operation/file_store_commit_test.cpp | 2 +- .../core/operation/file_store_write.cpp | 3 + src/paimon/core/operation/write_context.cpp | 23 ++- src/paimon/core/table/file_store_table.cpp | 21 +-- src/paimon/core/table/file_store_table.h | 23 +-- src/paimon/core/table/format/format_table.cpp | 20 ++- .../core/table/format/format_table_test.cpp | 35 ++-- src/paimon/core/table/source/table_read.cpp | 3 + src/paimon/core/table/source/table_scan.cpp | 3 + .../table/system/global_system_tables.cpp | 11 +- src/paimon/core/table/system/system_table.cpp | 88 +++++++++- src/paimon/core/table/system/system_table.h | 30 +++- .../core/table/system/system_table_test.cpp | 25 ++- src/paimon/core/table/table_test.cpp | 56 +++++++ src/paimon/core/utils/context_utils.h | 25 +++ 23 files changed, 542 insertions(+), 95 deletions(-) diff --git a/docs/source/api/format_table.rst b/docs/source/api/format_table.rst index 36295493f..cec5e848f 100644 --- a/docs/source/api/format_table.rst +++ b/docs/source/api/format_table.rst @@ -40,8 +40,8 @@ location, through the builder's usual constructor: * :cpp:class:`paimon::ScanContextBuilder`\ ``(const std::string&)`` * :cpp:class:`paimon::ReadContextBuilder`\ ``(const std::string&)`` -* :cpp:class:`paimon::WriteContextBuilder`\ ``(const std::string&)`` -* :cpp:class:`paimon::CommitContextBuilder`\ ``(const std::string&)`` +* :cpp:class:`paimon::WriteContextBuilder`\ ``(const std::string& root_path, const std::string& commit_user)`` +* :cpp:class:`paimon::CommitContextBuilder`\ ``(const std::string& root_path, const std::string& commit_user)`` The context then goes to the table's own operator - ``NewScan()``, ``NewRead()``, ``NewWrite()`` or ``NewCommit()`` - which serves a table whose schema does not live under its own location as well, diff --git a/include/paimon/commit_context.h b/include/paimon/commit_context.h index c76498e35..0c19ec5d6 100644 --- a/include/paimon/commit_context.h +++ b/include/paimon/commit_context.h @@ -59,6 +59,7 @@ class PAIMON_EXPORT CommitContext { const std::shared_ptr& executor, const std::shared_ptr& specific_file_system, const std::map& options, const std::string& branch, + const std::optional& named_branch, const std::optional& table_schema = std::nullopt, const std::map& fs_scheme_to_identifier_map = {}); ~CommitContext(); @@ -78,6 +79,14 @@ class PAIMON_EXPORT CommitContext { return branch_; } + /// Returns the branch the identifier of `WithCatalog()` or the `branch` option named, or + /// nullopt when neither did. An operator serves the branch the table it belongs to was loaded + /// with, so a table refuses a context naming a branch it does not serve - dropping the name + /// in silence would publish the changes on its own branch. + const std::optional& GetNamedBranch() const { + return named_branch_; + } + const std::string& GetCommitUser() const { return commit_user_; } @@ -134,9 +143,18 @@ class PAIMON_EXPORT CommitContext { return options_; } + /// Empties the options this context carries. A one-step `FileStoreCommit::Create()`-style + /// entry point folds them into the table it loads and clears them before handing the context + /// to `NewCommit`, which serves that table's own configuration alone; a caller holding a + /// table calls `NewCommit` with no options at all. + void ClearOptions() { + options_.clear(); + } + private: std::string root_path_; std::string branch_; + std::optional named_branch_; std::string commit_user_; bool ignore_empty_commit_; bool use_rest_catalog_commit_; @@ -171,9 +189,9 @@ class PAIMON_EXPORT CommitContextBuilder { /// @return Reference to this builder for method chaining. /// @deprecated These options reach an operator only through the deprecated /// `FileStoreCommit::Create()`-style entry points, which merge them into the table they - /// load. An operator created from a `Table` serves that table's own configuration, so pass - /// dynamic options through `Table::Copy()` instead. A key no table serves, such as a - /// `header.` key, is still read from the context. + /// load and then hand that table a context carrying none. An operator created from a `Table` + /// serves that table's own configuration alone and refuses a context carrying any option at + /// all, so pass dynamic options through `Table::Copy()` instead. PAIMON_DEPRECATED("Pass dynamic options through Table::Copy() instead") CommitContextBuilder& SetOptions(const std::map& options); diff --git a/include/paimon/read_context.h b/include/paimon/read_context.h index 0a926dc0c..b1d330c8d 100644 --- a/include/paimon/read_context.h +++ b/include/paimon/read_context.h @@ -90,6 +90,14 @@ class PAIMON_EXPORT ReadContext { return options_; } + /// Empties the options this context carries. A one-step `TableRead::Create()`-style entry point + /// folds them into the table it loads and clears them before handing the context to `NewRead`, + /// which serves that table's own configuration alone; a caller holding a table calls + /// `NewRead` with no options at all. + void ClearOptions() { + options_.clear(); + } + const std::vector& GetReadFieldNames() const { return read_field_names_; } @@ -325,10 +333,10 @@ class PAIMON_EXPORT ReadContextBuilder { /// @param options The configuration options map. /// @return Reference to this builder for method chaining. /// @deprecated These options reach an operator only through the deprecated - /// `TableRead::Create()`-style entry points, which merge them into the table they load. An - /// operator created from a `Table` serves that table's own configuration, so pass dynamic - /// options through `Table::Copy()` instead. A key no table serves, such as a `header.` key, - /// is still read from the context. + /// `TableRead::Create()`-style entry points, which merge them into the table they load and + /// then hand that table a context carrying none. An operator created from a `Table` serves + /// that table's own configuration alone and refuses a context carrying any option at all, so + /// pass dynamic options through `Table::Copy()` instead. PAIMON_DEPRECATED("Pass dynamic options through Table::Copy() instead") ReadContextBuilder& SetOptions(const std::map& options); diff --git a/include/paimon/scan_context.h b/include/paimon/scan_context.h index 52fddef73..b88b6037d 100644 --- a/include/paimon/scan_context.h +++ b/include/paimon/scan_context.h @@ -87,6 +87,14 @@ class PAIMON_EXPORT ScanContext { return options_; } + /// Empties the options this context carries. A one-step `TableScan::Create()`-style entry point + /// folds them into the table it loads and clears them before handing the context to `NewScan`, + /// which serves that table's own configuration alone; a caller holding a table calls + /// `NewScan` with no options at all. + void ClearOptions() { + options_.clear(); + } + std::shared_ptr GetMemoryPool() const { return memory_pool_; } @@ -223,10 +231,10 @@ class PAIMON_EXPORT ScanContextBuilder { /// @param options The configuration options map. /// @return Reference to this builder for method chaining. /// @deprecated These options reach an operator only through the deprecated - /// `TableScan::Create()`-style entry points, which merge them into the table they load. An - /// operator created from a `Table` serves that table's own configuration, so pass dynamic - /// options through `Table::Copy()` instead. A key no table serves, such as a `header.` key, - /// is still read from the context. + /// `TableScan::Create()`-style entry points, which merge them into the table they load and + /// then hand that table a context carrying none. An operator created from a `Table` serves + /// that table's own configuration alone and refuses a context carrying any option at all, so + /// pass dynamic options through `Table::Copy()` instead. PAIMON_DEPRECATED("Pass dynamic options through Table::Copy() instead") ScanContextBuilder& SetOptions(const std::map& options); diff --git a/include/paimon/write_context.h b/include/paimon/write_context.h index c3d3ea8b8..0d831991b 100644 --- a/include/paimon/write_context.h +++ b/include/paimon/write_context.h @@ -46,7 +46,8 @@ class PAIMON_EXPORT WriteContext { WriteContext(const std::string& root_path, const std::string& commit_user, bool is_streaming_mode, bool ignore_num_bucket_check, bool ignore_previous_files, bool enable_multi_thread_spill, const std::optional& write_id, - const std::string& branch, const std::vector& write_schema, + const std::string& branch, const std::optional& named_branch, + const std::vector& write_schema, const std::shared_ptr& memory_pool, const std::shared_ptr& executor, const std::string& temp_directory, const std::shared_ptr& specific_file_system, @@ -81,6 +82,14 @@ class PAIMON_EXPORT WriteContext { return options_; } + /// Empties the options this context carries. A one-step `FileStoreWrite::Create()`-style entry + /// point folds them into the table it loads and clears them before handing the context to + /// `NewWrite`, which serves that table's own configuration alone; a caller holding a table + /// calls `NewWrite` with no options at all. + void ClearOptions() { + options_.clear(); + } + bool IsStreamingMode() const { return is_streaming_mode_; } @@ -101,6 +110,14 @@ class PAIMON_EXPORT WriteContext { return branch_; } + /// Returns the branch the identifier of `WithCatalog()`, the `branch` option or + /// `WithBranch()` named, or nullopt when none of them did. An operator serves the branch the + /// table it belongs to was loaded with, so a table refuses a context naming a branch it does + /// not serve - dropping the name in silence would publish the changes on its own branch. + const std::optional& GetNamedBranch() const { + return named_branch_; + } + const std::vector& GetWriteSchema() const { return write_schema_; } @@ -156,6 +173,7 @@ class PAIMON_EXPORT WriteContext { std::string root_path_; std::string commit_user_; std::string branch_; + std::optional named_branch_; bool is_streaming_mode_; bool ignore_num_bucket_check_; bool ignore_previous_files_; @@ -192,10 +210,10 @@ class PAIMON_EXPORT WriteContextBuilder { /// @param options The configuration options map. /// @return Reference to this builder for method chaining. /// @deprecated These options reach an operator only through the deprecated - /// `FileStoreWrite::Create()`-style entry points, which merge them into the table they load. - /// An operator created from a `Table` serves that table's own configuration, so pass dynamic - /// options through `Table::Copy()` instead. A key no table serves, such as a `header.` key, - /// is still read from the context. + /// `FileStoreWrite::Create()`-style entry points, which merge them into the table they load + /// and then hand that table a context carrying none. An operator created from a `Table` + /// serves that table's own configuration alone and refuses a context carrying any option at + /// all, so pass dynamic options through `Table::Copy()` instead. PAIMON_DEPRECATED("Pass dynamic options through Table::Copy() instead") WriteContextBuilder& SetOptions(const std::map& options); diff --git a/src/paimon/core/catalog/file_system_catalog_test.cpp b/src/paimon/core/catalog/file_system_catalog_test.cpp index 7f63c8aef..b02ac50ec 100644 --- a/src/paimon/core/catalog/file_system_catalog_test.cpp +++ b/src/paimon/core/catalog/file_system_catalog_test.cpp @@ -27,6 +27,7 @@ #include "gtest/gtest.h" #include "paimon/catalog/identifier.h" #include "paimon/commit_context.h" +#include "paimon/commit_message.h" #include "paimon/common/data/blob_utils.h" #include "paimon/common/utils/path_util.h" #include "paimon/core/catalog/version_managed_catalog.h" @@ -42,11 +43,15 @@ #include "paimon/fs/file_system.h" #include "paimon/fs/file_system_factory.h" #include "paimon/read_context.h" +#include "paimon/reader/batch_reader.h" +#include "paimon/record_batch.h" #include "paimon/scan_context.h" #include "paimon/snapshot/snapshot_info.h" #include "paimon/table/format/format_table.h" +#include "paimon/table/source/plan.h" #include "paimon/table/source/table_read.h" #include "paimon/table/source/table_scan.h" +#include "paimon/testing/utils/read_result_collector.h" #include "paimon/testing/utils/testharness.h" #include "paimon/write_context.h" @@ -311,6 +316,128 @@ TEST(FileSystemCatalogTest, TestGetTableServesASystemTable) { "tbl1$options system table is read-only: committing for it is not supported"); } +// A system table a catalog hands out is copied the way a data table is. The dynamic options the +// copy is given are served by it - a `tbl$ro` view that selected a snapshot through `Copy` reads +// that snapshot's rows - and a `branch` option routes the copy to the branch it names, the way +// Java Paimon's system tables re-wrap `dataTable.copy(dynamicOptions)`. +TEST(FileSystemCatalogTest, TestACopyOfACatalogSystemTableAppliesItsOptions) { + std::map options; + options[Options::FILE_SYSTEM] = "local"; + options[Options::FILE_FORMAT] = "orc"; + ASSERT_OK_AND_ASSIGN(auto core_options, CoreOptions::FromMap(options)); + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str(), options); + ASSERT_OK(catalog.CreateDatabase("db1", options, /*ignore_if_exists=*/true)); + + auto typed_schema = arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false)}); + ::ArrowSchema schema; + ASSERT_TRUE(arrow::ExportSchema(*typed_schema, &schema).ok()); + const Identifier identifier("db1", "tbl1"); + ASSERT_OK(catalog.CreateTable(identifier, &schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*ignore_if_exists=*/false)); + ArrowSchemaRelease(&schema); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr
managed, catalog.GetTable(identifier)); + auto write_and_commit_two_rows = [](const std::shared_ptr
& table) { + WriteContextBuilder write_builder(table->GetRootPath(), "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write, + table->NewWrite(std::move(write_context))); + arrow::Int32Builder ids_builder; + ASSERT_TRUE(ids_builder.AppendValues({1, 2}).ok()); + std::shared_ptr ids_array; + ASSERT_TRUE(ids_builder.Finish(&ids_array).ok()); + auto struct_array = + arrow::StructArray::Make({ids_array}, + {arrow::field("id", arrow::int32(), /*nullable=*/false)}) + .ValueOrDie(); + ::ArrowArray arrow_array; + ASSERT_TRUE(arrow::ExportArray(*struct_array, &arrow_array).ok()); + RecordBatchBuilder batch_builder(&arrow_array); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, + batch_builder.SetBucket(0).Finish()); + ASSERT_OK(write->Write(std::move(batch))); + ASSERT_OK_AND_ASSIGN(std::vector> messages, + write->PrepareCommit()); + CommitContextBuilder commit_builder(table->GetRootPath(), "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + commit_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, + table->NewCommit(std::move(commit_context))); + ASSERT_OK(commit->Commit(messages)); + ASSERT_OK(write->Close()); + }; + // Two commits, so snapshot 1 holds the first two rows and snapshot 2 all four. + write_and_commit_two_rows(managed); + write_and_commit_two_rows(managed); + + auto count_rows = [](const std::shared_ptr
& table) -> Result { + ScanContextBuilder scan_builder(table->GetRootPath()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, + table->NewScan(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, scan->CreatePlan()); + ReadContextBuilder read_builder(table->GetRootPath()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, + table->NewRead(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, + read->CreateReader(plan->Splits())); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr rows, + ReadResultCollector::CollectResult(std::move(reader))); + return rows ? rows->length() : 0; + }; + + // A copy of `tbl$ro` selects the snapshot its options name: the view reads every row, the + // copy on snapshot 1 reads the two rows that commit wrote. + ASSERT_OK_AND_ASSIGN(std::shared_ptr
ro, catalog.GetTable(Identifier("db1", "tbl1$ro"))); + ASSERT_OK_AND_ASSIGN(int64_t all_rows, count_rows(ro)); + EXPECT_EQ(all_rows, 4); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
selected, + ro->Copy({{Options::SCAN_MODE, "from-snapshot"}, {Options::SCAN_SNAPSHOT_ID, "1"}})); + ASSERT_OK_AND_ASSIGN(int64_t selected_rows, count_rows(selected)); + EXPECT_EQ(selected_rows, 2); + + // The dev branch holds its own schema and one commit of two rows of its own. + SchemaManager dev_schema_manager(core_options.GetFileSystem(), managed->GetRootPath(), "dev"); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr dev_schema, + dev_schema_manager.CreateTable(typed_schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
dev_managed, + catalog.GetTable(Identifier("db1", "tbl1$branch_dev"))); + write_and_commit_two_rows(dev_managed); + + // A `branch` option routes the copy to the branch it names: the copy answers with the branch + // view's name and reads the rows that branch holds. + ASSERT_OK_AND_ASSIGN(std::shared_ptr
dev_selected, ro->Copy({{Options::BRANCH, "dev"}})); + EXPECT_EQ(dev_selected->Name(), "tbl1$branch_dev$ro"); + ASSERT_OK_AND_ASSIGN(int64_t dev_rows, count_rows(dev_selected)); + EXPECT_EQ(dev_rows, 2); + + // A view of a branch keeps serving that branch through a copy. + ASSERT_OK_AND_ASSIGN(std::shared_ptr
dev_ro, + catalog.GetTable(Identifier("db1", "tbl1$branch_dev$ro"))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
dev_ro_copy, dev_ro->Copy({})); + EXPECT_EQ(dev_ro_copy->Name(), "tbl1$branch_dev$ro"); + ASSERT_OK_AND_ASSIGN(int64_t dev_ro_rows, count_rows(dev_ro_copy)); + EXPECT_EQ(dev_ro_rows, 2); + + // The metadata tables copy the same way: the snapshots view serves the branch its copy names, + // and a branch holding no schema does not exist. + ASSERT_OK_AND_ASSIGN(std::shared_ptr
snapshots, + catalog.GetTable(Identifier("db1", "tbl1$snapshots"))); + ASSERT_OK_AND_ASSIGN(int64_t main_snapshot_rows, count_rows(snapshots)); + EXPECT_EQ(main_snapshot_rows, 2); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
dev_snapshots, + snapshots->Copy({{Options::BRANCH, "dev"}})); + ASSERT_OK_AND_ASSIGN(int64_t dev_snapshot_rows, count_rows(dev_snapshots)); + EXPECT_EQ(dev_snapshot_rows, 1); + ASSERT_NOK_WITH_MSG(snapshots->Copy({{Options::BRANCH, "missing"}}), + "branch missing does not exist"); +} + TEST(FileSystemCatalogTest, TestGetTableServesTheSysDatabase) { std::map options; options[Options::FILE_SYSTEM] = "local"; @@ -342,6 +469,33 @@ TEST(FileSystemCatalogTest, TestGetTableServesTheSysDatabase) { catalog->GetTable(Identifier("sys", "catalog_options"))); ASSERT_TRUE(std::dynamic_pointer_cast(catalog_options) != nullptr); + // A global system table copies over the options it already carries and ignores the dynamic + // ones, as Java Paimon's `CatalogOptionsTable#copy` does, so its copy reads the same rows. + ASSERT_OK_AND_ASSIGN(std::shared_ptr
copied_catalog_options, + catalog_options->Copy({{"ignored", "on-top"}})); + EXPECT_NE(copied_catalog_options, catalog_options); + ScanContextBuilder copied_scan_builder(copied_catalog_options->GetRootPath()); + copied_scan_builder.SetOptions(options); + ASSERT_OK_AND_ASSIGN(std::unique_ptr copied_scan_context, + copied_scan_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr copied_scan, + copied_catalog_options->NewScan(std::move(copied_scan_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr copied_plan, copied_scan->CreatePlan()); + ReadContextBuilder copied_read_builder(copied_catalog_options->GetRootPath()); + copied_read_builder.SetOptions(options); + ASSERT_OK_AND_ASSIGN(std::unique_ptr copied_read_context, + copied_read_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr copied_read, + copied_catalog_options->NewRead(std::move(copied_read_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr copied_reader, + copied_read->CreateReader(copied_plan->Splits())); + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch copied_result, copied_reader->NextBatch()); + auto& [copied_array, copied_c_schema] = copied_result; + auto maybe_copied_batch = arrow::ImportRecordBatch(copied_array.get(), copied_c_schema.get()); + ASSERT_TRUE(maybe_copied_batch.ok()) << maybe_copied_batch.status().ToString(); + auto copied_batch = std::move(maybe_copied_batch).ValueUnsafe(); + ASSERT_GE(copied_batch->num_rows(), 1); + ASSERT_NOK_WITH_MSG(catalog->GetTable(Identifier("sys", "nope")), "not exist"); ASSERT_OK_AND_ASSIGN(auto tables, catalog->GetTable(Identifier("sys", "tables"))); diff --git a/src/paimon/core/operation/commit_context.cpp b/src/paimon/core/operation/commit_context.cpp index ba0713ec0..3bbbab5d9 100644 --- a/src/paimon/core/operation/commit_context.cpp +++ b/src/paimon/core/operation/commit_context.cpp @@ -50,6 +50,12 @@ CommitContext::CommitContext(const std::string& root_path, const std::string& co // Without an identifier the option is the only name a branch can have. BranchManager::NormalizeBranch( options.count(Options::BRANCH) ? options.at(Options::BRANCH) : ""), + // The option names the branch, and nothing else can when there is no + // identifier. + options.count(Options::BRANCH) + ? std::optional(BranchManager::NormalizeBranch( + options.at(Options::BRANCH))) + : std::nullopt, /*table_schema=*/std::nullopt, /*fs_scheme_to_identifier_map=*/{}) {} CommitContext::CommitContext( @@ -60,10 +66,12 @@ CommitContext::CommitContext( const std::shared_ptr& executor, const std::shared_ptr& specific_file_system, const std::map& options, const std::string& branch, + const std::optional& named_branch, const std::optional& table_schema, const std::map& fs_scheme_to_identifier_map) : root_path_(root_path), branch_(branch), + named_branch_(named_branch), commit_user_(commit_user), ignore_empty_commit_(ignore_empty_commit), use_rest_catalog_commit_(use_rest_catalog_commit), @@ -208,6 +216,10 @@ Result> CommitContextBuilder::Finish() { PAIMON_RETURN_NOT_OK( CatalogUtils::CheckNotSystemTable(impl_->identifier_.value(), "commit")); } + std::optional identifier_branch; + if (impl_->identifier_) { + PAIMON_ASSIGN_OR_RAISE(identifier_branch, impl_->identifier_->GetBranchName()); + } PAIMON_ASSIGN_OR_RAISE(std::string branch, BranchManager::ResolveBranch( impl_->identifier_, impl_->options_, /*explicit_branch=*/std::nullopt, "commit")); @@ -219,8 +231,6 @@ Result> CommitContextBuilder::Finish() { PAIMON_RETURN_NOT_OK(BranchManager::CheckCatalogAddressableBranch(branch)); } if (impl_->catalog_ != nullptr) { - PAIMON_ASSIGN_OR_RAISE(std::optional identifier_branch, - impl_->identifier_.value().GetBranchName()); if (!identifier_branch) { PAIMON_ASSIGN_OR_RAISE(std::string table_name, impl_->identifier_.value().GetDataTableName()); @@ -248,12 +258,18 @@ Result> CommitContextBuilder::Finish() { if (impl_->root_path_.empty()) { return Status::Invalid("root path is empty"); } + // The branch the naming settings named, kept beside the resolved one so that a table can + // tell a context naming no branch - which it serves on its own - from one naming another. + std::optional named_branch; + if (identifier_branch || impl_->options_.find(Options::BRANCH) != impl_->options_.end()) { + named_branch = branch; + } auto ctx = std::make_unique( impl_->root_path_, impl_->commit_user_, impl_->ignore_empty_commit_, impl_->use_rest_catalog_commit_, impl_->catalog_, impl_->identifier_, impl_->table_id_, impl_->append_commit_check_conflict_, impl_->memory_pool_, impl_->executor_, - impl_->specific_file_system_, impl_->options_, branch, impl_->table_schema_, - impl_->fs_scheme_to_identifier_map_); + impl_->specific_file_system_, impl_->options_, branch, named_branch, + impl_->table_schema_, impl_->fs_scheme_to_identifier_map_); impl_->Reset(); return ctx; } diff --git a/src/paimon/core/operation/file_store_commit.cpp b/src/paimon/core/operation/file_store_commit.cpp index 9b6c3975c..af818e868 100644 --- a/src/paimon/core/operation/file_store_commit.cpp +++ b/src/paimon/core/operation/file_store_commit.cpp @@ -30,6 +30,9 @@ Result> FileStoreCommit::Create( std::unique_ptr ctx) { PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "commit")); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, FileStoreTableFactory::Create(*ctx)); + // The load folded the context's options into the table, so the operator below serves them + // from there and the context carries none of its own. + ctx->ClearOptions(); return table->NewCommit(std::move(ctx)); } diff --git a/src/paimon/core/operation/file_store_commit_test.cpp b/src/paimon/core/operation/file_store_commit_test.cpp index 732f83bd0..753f2bba7 100644 --- a/src/paimon/core/operation/file_store_commit_test.cpp +++ b/src/paimon/core/operation/file_store_commit_test.cpp @@ -101,7 +101,7 @@ TEST(FileStoreCommitTest, TestCreateWithCatalogRequiresIdentifier) { /*use_rest_catalog_commit=*/false, catalog, /*identifier=*/std::nullopt, /*table_id=*/std::nullopt, /*append_commit_check_conflict=*/false, GetDefaultPool(), CreateDefaultExecutor(), /*specific_file_system=*/nullptr, options, - BranchManager::DEFAULT_MAIN_BRANCH); + BranchManager::DEFAULT_MAIN_BRANCH, /*named_branch=*/std::nullopt); ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(commit_context)), "a catalog commit requires a table identifier"); } diff --git a/src/paimon/core/operation/file_store_write.cpp b/src/paimon/core/operation/file_store_write.cpp index ea73b2246..36ea1dea1 100644 --- a/src/paimon/core/operation/file_store_write.cpp +++ b/src/paimon/core/operation/file_store_write.cpp @@ -28,6 +28,9 @@ namespace paimon { Result> FileStoreWrite::Create(std::unique_ptr ctx) { PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "write")); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, FileStoreTableFactory::Create(*ctx)); + // The load folded the context's options into the table, so the operator below serves them + // from there and the context carries none of its own. + ctx->ClearOptions(); return table->NewWrite(std::move(ctx)); } diff --git a/src/paimon/core/operation/write_context.cpp b/src/paimon/core/operation/write_context.cpp index fbcbfe368..ad20baf0a 100644 --- a/src/paimon/core/operation/write_context.cpp +++ b/src/paimon/core/operation/write_context.cpp @@ -38,6 +38,7 @@ WriteContext::WriteContext( const std::string& root_path, const std::string& commit_user, bool is_streaming_mode, bool ignore_num_bucket_check, bool ignore_previous_files, bool enable_multi_thread_spill, const std::optional& write_id, const std::string& branch, + const std::optional& named_branch, const std::vector& write_schema, const std::shared_ptr& memory_pool, const std::shared_ptr& executor, const std::string& temp_directory, const std::shared_ptr& specific_file_system, @@ -49,6 +50,7 @@ WriteContext::WriteContext( : root_path_(root_path), commit_user_(commit_user), branch_(branch), + named_branch_(named_branch), is_streaming_mode_(is_streaming_mode), ignore_num_bucket_check_(ignore_num_bucket_check), ignore_previous_files_(ignore_previous_files), @@ -233,14 +235,16 @@ Result> WriteContextBuilder::Finish() { PAIMON_RETURN_NOT_OK( CatalogUtils::CheckNotSystemTable(impl_->identifier_.value(), "write")); } + std::optional identifier_branch; + if (impl_->identifier_) { + PAIMON_ASSIGN_OR_RAISE(identifier_branch, impl_->identifier_->GetBranchName()); + } PAIMON_ASSIGN_OR_RAISE( std::string branch, BranchManager::ResolveBranch(impl_->identifier_, impl_->options_, impl_->branch_, "write")); if (impl_->catalog_ != nullptr && !BranchManager::IsMainBranch(branch)) { // Refused here rather than at the commit of what is written to the branch. PAIMON_RETURN_NOT_OK(BranchManager::CheckCatalogAddressableBranch(branch)); - PAIMON_ASSIGN_OR_RAISE(std::optional identifier_branch, - impl_->identifier_->GetBranchName()); if (!identifier_branch) { PAIMON_ASSIGN_OR_RAISE(std::string table_name, impl_->identifier_->GetDataTableName()); return Status::Invalid(fmt::format( @@ -262,13 +266,20 @@ Result> WriteContextBuilder::Finish() { PAIMON_RETURN_NOT_OK_FROM_ARROW( arrow::SetCpuThreadPoolCapacity(impl_->spill_thread_number_)); } + // The branch the naming settings named, kept beside the resolved one so that a table can + // tell a context naming no branch - which it serves on its own - from one naming another. + std::optional named_branch; + if (impl_->branch_ || identifier_branch || + impl_->options_.find(Options::BRANCH) != impl_->options_.end()) { + named_branch = branch; + } auto ctx = std::make_unique( impl_->root_path_, impl_->commit_user_, impl_->is_streaming_mode_, impl_->ignore_num_bucket_check_, impl_->ignore_previous_files_, enable_multi_thread_spill, - impl_->write_id_, branch, impl_->write_schema_, impl_->memory_pool_, impl_->executor_, - impl_->temp_directory_, impl_->specific_file_system_, impl_->fs_scheme_to_identifier_map_, - impl_->realtime_context_, impl_->options_, impl_->table_schema_, impl_->catalog_, - impl_->identifier_); + impl_->write_id_, branch, named_branch, impl_->write_schema_, impl_->memory_pool_, + impl_->executor_, impl_->temp_directory_, impl_->specific_file_system_, + impl_->fs_scheme_to_identifier_map_, impl_->realtime_context_, impl_->options_, + impl_->table_schema_, impl_->catalog_, impl_->identifier_); impl_->Reset(); return ctx; } diff --git a/src/paimon/core/table/file_store_table.cpp b/src/paimon/core/table/file_store_table.cpp index da853a43a..1b5dae4fa 100644 --- a/src/paimon/core/table/file_store_table.cpp +++ b/src/paimon/core/table/file_store_table.cpp @@ -190,22 +190,19 @@ Result> FileStoreTable::Copy( CatalogUtils::BranchIdentifier(identifier_, target_branch)); branch_identifier.emplace(std::move(branch_named)); } - // The schema options name the branch the copy serves, the way `SwitchToBranch` styles them: - // another branch names itself in them, the main branch carries no `branch` option. The key is - // folded into the schema alone, not into the dynamic options the copy accumulates. - std::map schema_overridable = overridable; - if (target_branch != CurrentBranch() && target_branch != BranchManager::DEFAULT_MAIN_BRANCH) { - schema_overridable[Options::BRANCH] = target_branch; - } - PAIMON_RETURN_NOT_OK(SchemaValidation::CheckImmutableOptions(*base_schema, overridable)); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merged_schema, - base_schema->MergeOptions(schema_overridable)); std::map accumulated = dynamic_options_; accumulated.erase(Options::TYPE); accumulated.erase(Options::BRANCH); for (const auto& [key, value] : overridable) { accumulated.insert_or_assign(key, value); } + std::map schema_overridable = accumulated; + if (target_branch != CurrentBranch() && target_branch != BranchManager::DEFAULT_MAIN_BRANCH) { + schema_overridable[Options::BRANCH] = target_branch; + } + PAIMON_RETURN_NOT_OK(SchemaValidation::CheckImmutableOptions(*base_schema, accumulated)); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merged_schema, + base_schema->MergeOptions(schema_overridable)); // A copy resolves its file system from its own merged options, so it takes over no manager of // the table it was copied from: the store builds both over the resolution the copy names. return FileStoreTable::Create( @@ -266,10 +263,6 @@ Status FileStoreTable::RestoreRealtimeCommittedProgress( Status FileStoreTable::CheckOptionsAgreeWithTable( const TableSchema& table_schema, const std::map& context_options) const { - // The configuration the operators serve: the schema's own options as the load styled them, - // with the dynamic options of the load on top. `NewXxx` reads it through `store_->GetOptions()` - // and adds nothing from a context, so a context option that is not among these keys is one the - // operators cannot apply. std::map table_options = table_schema.Options(); for (const auto& [key, value] : dynamic_options_) { table_options.insert_or_assign(key, value); diff --git a/src/paimon/core/table/file_store_table.h b/src/paimon/core/table/file_store_table.h index 64be124ef..9737dada0 100644 --- a/src/paimon/core/table/file_store_table.h +++ b/src/paimon/core/table/file_store_table.h @@ -136,29 +136,14 @@ class FileStoreTable : public Table { std::unique_ptr context) const override; private: - /// Checks the options a context carries against the configuration the table serves: its - /// schema's options with its dynamic options on top. A table serves that configuration and - /// nothing else - the operators read it and add no context option to it - so an option the - /// context contradicts is refused, and so is one the table was not loaded with: serving it - /// would drop the context value in silence behind the table's own configuration. A caller - /// that means to change the configuration copies the table. A table a load hands back carries - /// the context's own options merged in - `FileStoreTableFactory` merges them, whatever the - /// key, as it merges a `header.` key the transport reads - so it agrees by construction; the - /// refusal belongs to a table the caller already held, whose load is over. The keys a load - /// routes with (`branch`, `type`) are folded into the table there - by the load or by `Copy` - - /// and checked against the identifier and the schema instead. - Status CheckOptionsAgreeWithTable( - const TableSchema& table_schema, - const std::map& context_options) const; - /// Checks the identity a context carries - the path it names, the identifier it gives a /// catalog, the table id it records - against this table's own. The store below serves this /// table's files, so a context naming another table would write this table's metadata and /// publish it as the other's; a field that contradicts the table is refused by name, the way - /// `CheckOptionsAgreeWithTable` refuses a contradicting option. A field the context does not - /// carry is no contradiction: a write or a commit from a bare path names no catalog object. - /// An identifier that names a branch this table does not serve names a branch of another - /// table, as the operator serves the branch the table was loaded with alone. + /// `CheckContextCarriesNoOptions` refuses an option a context carries. A field the context + /// does not carry is no contradiction: a write or a commit from a bare path names no catalog + /// object. An identifier that names a branch this table does not serve names a branch of + /// another table, as the operator serves the branch the table was loaded with alone. /// @param kind The operator the context was handed to, which the error names. /// @param context_table_id The id `CommitContext` records for a catalog commit; a write /// context carries none, so `NewWrite` passes null. diff --git a/src/paimon/core/table/format/format_table.cpp b/src/paimon/core/table/format/format_table.cpp index 141b31711..b5b41ff5e 100644 --- a/src/paimon/core/table/format/format_table.cpp +++ b/src/paimon/core/table/format/format_table.cpp @@ -228,8 +228,9 @@ Result> FormatTable::NewScan( std::unique_ptr context) const { std::shared_ptr shared_context = std::move(context); PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(shared_context, "scan")); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, - CopySelf(shared_context->GetOptions())); + PAIMON_RETURN_NOT_OK( + ContextUtils::CheckContextCarriesNoOptions("scan", shared_context->GetOptions())); + std::shared_ptr table = SharedSelf(); if (shared_context->IsStreamingMode()) { return Status::Invalid( "a format table has no snapshots, so there is nothing for a streaming scan to follow"); @@ -272,6 +273,8 @@ Result> FormatTable::NewRead( std::unique_ptr context) const { std::shared_ptr read_context = std::move(context); PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(read_context, "read")); + PAIMON_RETURN_NOT_OK( + ContextUtils::CheckContextCarriesNoOptions("read", read_context->GetOptions())); if (read_context->HasVectorOrFullTextSearch()) { return Status::NotImplemented( "VectorSearch or FullTextSearch is not supported for format tables"); @@ -279,17 +282,17 @@ Result> FormatTable::NewRead( if (PredicateUtils::ContainsFullTextMatch(read_context->GetPredicate())) { return Status::NotImplemented("FullTextMatch predicate execution is not implemented yet"); } - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, - CopySelf(read_context->GetOptions())); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, - FormatTableRead::Create(table, read_context)); + FormatTableRead::Create(SharedSelf(), read_context)); return std::unique_ptr(std::move(read)); } Result> FormatTable::NewWrite( std::unique_ptr context) const { PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "write")); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, CopySelf(context->GetOptions())); + PAIMON_RETURN_NOT_OK( + ContextUtils::CheckContextCarriesNoOptions("write", context->GetOptions())); + std::shared_ptr table = SharedSelf(); if (context->GetCatalog() != nullptr) { return Status::Invalid( "WithCatalog() requires a native table; a format table takes a WriteContextBuilder " @@ -319,8 +322,9 @@ Result> FormatTable::NewWrite( Result> FormatTable::NewCommit( std::unique_ptr context) const { PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "commit")); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr given_table, - CopySelf(context->GetOptions())); + PAIMON_RETURN_NOT_OK( + ContextUtils::CheckContextCarriesNoOptions("commit", context->GetOptions())); + std::shared_ptr given_table = SharedSelf(); if (!context->IgnoreEmptyCommit()) { return Status::Invalid( "a format table cannot record an empty commit: keeping one means writing a " diff --git a/src/paimon/core/table/format/format_table_test.cpp b/src/paimon/core/table/format/format_table_test.cpp index c7456d005..f4c7e6651 100644 --- a/src/paimon/core/table/format/format_table_test.cpp +++ b/src/paimon/core/table/format/format_table_test.cpp @@ -763,7 +763,7 @@ TEST(FormatTableTest, TestCatalogManagedPartitionsAreRejected) { ASSERT_TRUE(dynamic_table.status().IsNotImplemented()) << dynamic_table.status().ToString(); } -TEST(FormatTableTest, TestTheTableOperatorsValidateOptionsGivenAtTheCall) { +TEST(FormatTableTest, TestTheTableOperatorsRefuseAContextOption) { std::unique_ptr dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); ASSERT_OK_AND_ASSIGN(std::shared_ptr table, @@ -774,22 +774,26 @@ TEST(FormatTableTest, TestTheTableOperatorsValidateOptionsGivenAtTheCall) { ScanContextBuilder scan_builder(dir->Str()); scan_builder.SetOptions(refused); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); - ASSERT_NOK_WITH_MSG(table->NewScan(std::move(scan_context)), "metastore.partitioned-table"); + ASSERT_NOK_WITH_MSG(table->NewScan(std::move(scan_context)), + "the scan context carries option 'metastore.partitioned-table'"); ReadContextBuilder read_builder(dir->Str()); read_builder.SetOptions(refused); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); - ASSERT_NOK_WITH_MSG(table->NewRead(std::move(read_context)), "metastore.partitioned-table"); + ASSERT_NOK_WITH_MSG(table->NewRead(std::move(read_context)), + "the read context carries option 'metastore.partitioned-table'"); WriteContextBuilder write_builder(dir->Str(), "test-user"); write_builder.SetOptions(refused); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); - ASSERT_NOK_WITH_MSG(table->NewWrite(std::move(write_context)), "metastore.partitioned-table"); + ASSERT_NOK_WITH_MSG(table->NewWrite(std::move(write_context)), + "the write context carries option 'metastore.partitioned-table'"); CommitContextBuilder commit_builder(dir->Str(), "test-user"); commit_builder.SetOptions(refused); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); - ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(commit_context)), "metastore.partitioned-table"); + ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(commit_context)), + "the commit context carries option 'metastore.partitioned-table'"); } TEST(FormatTableTest, TestValueOnlyPartitionCannotBeNamedAfterThisTablesMetadata) { @@ -2805,13 +2809,14 @@ TEST(FormatTableTest, TestALoadedFormatTableIsServedThroughItsOwnOperators) { ASSERT_OK_AND_ASSIGN(std::unique_ptr path_context, path_builder.Finish()); ASSERT_NOK(TableScan::Create(std::move(path_context))); - // Options given at the call still win over the ones the schema stored, so a file is rolled - // per row rather than one file holding all three. + // The way to serve a different value is a copy that carries it, so a file is rolled per + // row rather than one file holding all three. + ASSERT_OK_AND_ASSIGN(std::shared_ptr
rolled, + external->Copy({{Options::TARGET_FILE_ROW_NUM, "1"}})); WriteContextBuilder write_builder(external->Location(), "test-user"); - write_builder.SetOptions({{Options::TARGET_FILE_ROW_NUM, "1"}}); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr write, - external->NewWrite(std::move(write_context))); + rolled->NewWrite(std::move(write_context))); for (int32_t i = 1; i <= 3; i++) { ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, MakeBatch({i}, {"name"}, "20240101", {{"dt", "20240101"}})); @@ -3035,24 +3040,28 @@ TEST(FormatTableTest, TestABranchNamedThroughOptionsIsRefusedByTheTableOperators ScanContextBuilder scan_builder(dir->Str()); scan_builder.SetOptions(branch); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); - ASSERT_NOK_WITH_MSG(table->NewScan(std::move(scan_context)), "has no branches"); + ASSERT_NOK_WITH_MSG(table->NewScan(std::move(scan_context)), + "the scan context carries option 'branch'"); ReadContextBuilder read_builder(dir->Str()); read_builder.SetOptions(branch); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); - ASSERT_NOK_WITH_MSG(table->NewRead(std::move(read_context)), "has no branches"); + ASSERT_NOK_WITH_MSG(table->NewRead(std::move(read_context)), + "the read context carries option 'branch'"); WriteContextBuilder write_builder(dir->Str(), "test-user"); write_builder.SetOptions(branch); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); - ASSERT_NOK_WITH_MSG(table->NewWrite(std::move(write_context)), "has no branches"); + ASSERT_NOK_WITH_MSG(table->NewWrite(std::move(write_context)), + "the write context carries option 'branch'"); // The commit is the one that matters most: an overwrite clears what it replaces before it // publishes anything, so the refusal has to come before the deleting does. CommitContextBuilder commit_builder(dir->Str(), "test-user"); commit_builder.SetOptions(branch); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); - ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(commit_context)), "has no branches"); + ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(commit_context)), + "the commit context carries option 'branch'"); ASSERT_OK_AND_ASSIGN( std::unique_ptr scan, diff --git a/src/paimon/core/table/source/table_read.cpp b/src/paimon/core/table/source/table_read.cpp index 5688c7650..2ccbe142b 100644 --- a/src/paimon/core/table/source/table_read.cpp +++ b/src/paimon/core/table/source/table_read.cpp @@ -28,6 +28,9 @@ namespace paimon { Result> TableRead::Create(std::unique_ptr ctx) { PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "read")); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, FileStoreTableFactory::Create(*ctx)); + // The load folded the context's options into the table, so the operator below serves them + // from there and the context carries none of its own. + ctx->ClearOptions(); return table->NewRead(std::move(ctx)); } diff --git a/src/paimon/core/table/source/table_scan.cpp b/src/paimon/core/table/source/table_scan.cpp index d3dd794b4..14c78ab3b 100644 --- a/src/paimon/core/table/source/table_scan.cpp +++ b/src/paimon/core/table/source/table_scan.cpp @@ -30,6 +30,9 @@ namespace paimon { Result> TableScan::Create(std::unique_ptr context) { PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "scan")); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, FileStoreTableFactory::Create(*context)); + // The load folded the context's options into the table, so the operator below serves them + // from there and the context carries none of its own. + context->ClearOptions(); return table->NewScan(std::move(context)); } diff --git a/src/paimon/core/table/system/global_system_tables.cpp b/src/paimon/core/table/system/global_system_tables.cpp index e420125a7..d86849d47 100644 --- a/src/paimon/core/table/system/global_system_tables.cpp +++ b/src/paimon/core/table/system/global_system_tables.cpp @@ -306,7 +306,16 @@ Result> GlobalSystemTableLoader::Load( if (context.catalog == nullptr && RequiresCatalog(table_name)) { return Status::NotImplemented("global system table requires catalog context: ", table_name); } - return LoadForSchema(table_name, context); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, + LoadForSchema(table_name, context)); + // The inputs this table was built from are recorded for `Copy`, which re-enters this loader + // with the context unchanged: Java Paimon's global system tables copy over the options they + // already carry and ignore the dynamic ones a copy is given. + SystemTable::Origin origin; + origin.kind = StringUtils::ToLowerCase(table_name); + origin.context = std::make_shared(context); + system_table->origin_.emplace(std::move(origin)); + return system_table; } Result> GlobalSystemTableLoader::LoadForSchema( diff --git a/src/paimon/core/table/system/system_table.cpp b/src/paimon/core/table/system/system_table.cpp index a8be1c62d..a75aa1a2b 100644 --- a/src/paimon/core/table/system/system_table.cpp +++ b/src/paimon/core/table/system/system_table.cpp @@ -26,12 +26,15 @@ #include #include +#include "fmt/format.h" #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" #include "paimon/commit_context.h" #include "paimon/common/utils/path_util.h" #include "paimon/common/utils/string_utils.h" +#include "paimon/core/options/table_type.h" #include "paimon/core/schema/schema_manager.h" +#include "paimon/core/schema/schema_validation.h" #include "paimon/core/schema/table_schema.h" #include "paimon/core/table/system/audit_log_system_table.h" #include "paimon/core/table/system/binlog_system_table.h" @@ -214,8 +217,80 @@ Status SystemTable::AdoptOwnSchema() { } Result> SystemTable::Copy( - const std::map& /*dynamic_options*/) const { - return Status::NotImplemented(Name(), " system table is read-only: copying is not supported"); + const std::map& dynamic_options) const { + if (!origin_) { + // A table no loader built - one of the schema-only handles `LoadForSchema` hands out - has + // no recorded inputs to rebuild from, the same defensive case the refused write and + // commit below guard. + return Status::Invalid(Name(), + " was not loaded by a system table loader: copying it is " + "not supported"); + } + if (origin_->context) { + // Java Paimon's global system tables copy over the options they already carry and ignore + // the dynamic ones (`CatalogOptionsTable#copy`), so the copy re-enters the same loader. + return GlobalSystemTableLoader::Load(origin_->kind, *origin_->context); + } + const std::shared_ptr& loaded_schema = origin_->table_schema; + // `type` is immutable the way `FileStoreTable::Copy` holds it: the kind of a copy is named by + // the primary keys of the schema it serves, never by this option. A value naming another + // type is refused, and one naming the same type is dropped. + auto type_option = dynamic_options.find(Options::TYPE); + if (type_option != dynamic_options.end()) { + PAIMON_ASSIGN_OR_RAISE(TableType schema_type, + TableTypeDefine::FromOptions(loaded_schema->Options())); + std::map named{{Options::TYPE, type_option->second}}; + PAIMON_ASSIGN_OR_RAISE(TableType named_type, TableTypeDefine::FromOptions(named)); + if (named_type != schema_type) { + return Status::Invalid(fmt::format("Change '{}' is not supported yet.", Options::TYPE)); + } + } + // `branch` names the branch the copy serves, the way it routes a load: a copy on the branch + // this view serves keeps the data table's schema, while one on another branch reads that + // branch's own latest schema and refuses a branch that holds none. + std::string current_branch = LoadBranch(MergeOptions(loaded_schema, origin_->dynamic_options)); + std::string target_branch = current_branch; + auto branch_option = dynamic_options.find(Options::BRANCH); + if (branch_option != dynamic_options.end()) { + target_branch = BranchManager::NormalizeBranch(branch_option->second); + } + std::map accumulated = origin_->dynamic_options; + accumulated.erase(Options::TYPE); + accumulated.erase(Options::BRANCH); + for (const auto& [key, value] : dynamic_options) { + if (key != Options::TYPE && key != Options::BRANCH) { + accumulated.insert_or_assign(key, value); + } + } + std::shared_ptr base_schema = loaded_schema; + std::string data_table_name = origin_->data_table_name; + if (target_branch != current_branch) { + // The branch is read through the file system this table serves, and then served through + // the schema that branch holds, the same read `FileStoreTable::Copy` makes. + SchemaManager schema_manager(GetFileSystem(), GetRootPath(), target_branch); + PAIMON_ASSIGN_OR_RAISE(std::optional> latest, + schema_manager.Latest()); + if (!latest) { + return Status::Invalid(fmt::format("branch {} does not exist", target_branch)); + } + base_schema = *latest; + // The copy answers with the logical name of the branch it now serves, composed the way + // `LoadFromPath` composes one: the data table's name with the branch it addresses. + const std::string& recorded_name = + data_table_name.empty() ? PathUtil::GetName(GetRootPath()) : data_table_name; + PAIMON_ASSIGN_OR_RAISE( + std::string data_table_base, + Identifier(Identifier::kUnknownDatabase, recorded_name).GetDataTableName()); + data_table_name = Identifier(Identifier::kUnknownDatabase, data_table_base, target_branch) + .GetObjectName(); + } + std::map schema_overridable = accumulated; + if (target_branch != BranchManager::DEFAULT_MAIN_BRANCH) { + schema_overridable[Options::BRANCH] = target_branch; + } + PAIMON_RETURN_NOT_OK(SchemaValidation::CheckImmutableOptions(*base_schema, accumulated)); + return SystemTableLoader::Load(origin_->kind, GetFileSystem(), GetRootPath(), base_schema, + schema_overridable, data_table_name); } Result> SystemTable::NewWrite( @@ -246,6 +321,15 @@ Result> SystemTableLoader::Load( PAIMON_ASSIGN_OR_RAISE( std::shared_ptr system_table, factory.value()(fs, table_path, table_schema, dynamic_options, effective_name)); + // The inputs this table was built from are recorded for `Copy`, which merges new dynamic + // options into them and rebuilds through this same loader, the way Java Paimon's system + // tables re-wrap `dataTable.copy(dynamicOptions)`. + SystemTable::Origin origin; + origin.kind = StringUtils::ToLowerCase(system_table_name); + origin.table_schema = table_schema; + origin.dynamic_options = dynamic_options; + origin.data_table_name = data_table_name; + system_table->origin_.emplace(std::move(origin)); PAIMON_RETURN_NOT_OK(system_table->AdoptOwnSchema()); return system_table; } diff --git a/src/paimon/core/table/system/system_table.h b/src/paimon/core/table/system/system_table.h index 3b93feb5a..92281ad07 100644 --- a/src/paimon/core/table/system/system_table.h +++ b/src/paimon/core/table/system/system_table.h @@ -37,6 +37,7 @@ class ReadContext; class ScanContext; class TableSchema; class SystemTableSchema; +struct GlobalSystemTableContext; /// Parsed information for a table-scoped system table path. struct SystemTablePath { @@ -70,9 +71,12 @@ class SystemTable : public Table, public std::enable_shared_from_this> ArrowSchema() const = 0; - /// A system table is read-only, so copying it, writing it and committing for it are refused - /// instead of routed to the data table underneath. That table serves them through its own - /// `Table`. + /// Copies this system table with `dynamic_options` on top of the ones it was loaded with, + /// rebuilding it through the loader that created it - the way Java Paimon's system tables + /// re-wrap `dataTable.copy(dynamicOptions)`. A global system table of the `sys` database + /// copies over the options it already carries and ignores the dynamic ones, as Java's do. + /// Writing and committing stay refused: this view is read-only, and the data table + /// underneath serves them through its own `Table`. Result> Copy( const std::map& dynamic_options) const override; @@ -111,7 +115,25 @@ class SystemTable : public Table, public std::enable_shared_from_this table_schema; + /// Dynamic options the table was loaded with, on top of the schema's own. + std::map dynamic_options; + /// Logical name of the data table underneath, branch included. May be empty for a + /// table-scoped table, whose loader then falls back to the directory name. + std::string data_table_name; + /// Context a global system table was loaded with. Null for a table-scoped table. + std::shared_ptr context; + }; + std::string name_; + std::optional origin_; }; /// Loads system table implementations from parsed table identifiers or table paths. @@ -125,7 +147,7 @@ class SystemTableLoader { const std::string& system_table_name, const std::shared_ptr& fs, const std::string& table_path, const std::shared_ptr& table_schema, const std::map& dynamic_options, - const std::string& data_table_name = ""); + const std::string& data_table_name); static Result> TryParsePath(const std::string& path); diff --git a/src/paimon/core/table/system/system_table_test.cpp b/src/paimon/core/table/system/system_table_test.cpp index 79f4c6cca..d46f2426e 100644 --- a/src/paimon/core/table/system/system_table_test.cpp +++ b/src/paimon/core/table/system/system_table_test.cpp @@ -230,7 +230,7 @@ TEST(SystemTableTest, TestReadOptimizedSystemTableRegistration) { ASSERT_OK_AND_ASSIGN(std::shared_ptr system_table, SystemTableLoader::Load(ReadOptimizedSystemTable::kName, /*fs=*/nullptr, "/tmp/table", table_schema, - /*dynamic_options=*/{})); + /*dynamic_options=*/{}, /*data_table_name=*/"")); ASSERT_EQ(system_table->Name(), "table$ro"); // A view loaded with the logical name of its data table answers with it, branch included: @@ -247,7 +247,11 @@ TEST(SystemTableTest, TestReadOptimizedSystemTableRegistration) { ASSERT_EQ(arrow_schema->field(1)->type()->id(), arrow::Type::INT32); } -TEST(SystemTableTest, TestCopyIsRefused) { +// A copy of a system table merges the dynamic options into the ones it was loaded with and +// rebuilds through the same loader, the way Java Paimon's system tables re-wrap +// `dataTable.copy(dynamicOptions)`: the copy keeps the view's identity while being a table of +// its own, and an option the data table's own `Copy` would refuse is refused here too. +TEST(SystemTableTest, TestCopyRebuildsThroughTheLoader) { std::map options = {{Options::FILE_SYSTEM, "local"}, {Options::FILE_FORMAT, "orc"}}; ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, @@ -255,8 +259,19 @@ TEST(SystemTableTest, TestCopyIsRefused) { ASSERT_OK_AND_ASSIGN(std::shared_ptr system_table, SystemTableLoader::Load(ReadOptimizedSystemTable::kName, /*fs=*/nullptr, "/tmp/table", table_schema, - /*dynamic_options=*/{})); - ASSERT_NOK_WITH_MSG(system_table->Copy({{"probe", "on-top"}}), "copying is not supported"); + /*dynamic_options=*/{}, /*data_table_name=*/"")); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
copy, system_table->Copy({})); + EXPECT_EQ(copy->Name(), system_table->Name()); + EXPECT_NE(copy, system_table); + EXPECT_TRUE(std::dynamic_pointer_cast(copy) != nullptr); + + // The dynamic options a copy is given reach the data table underneath it, so an immutable one + // is refused the way the data table's own `Copy` refuses it, and `type` names the kind of that + // table, which no copy changes. + ASSERT_NOK_WITH_MSG(system_table->Copy({{Options::BUCKET_KEY, "other"}}), + "Change 'bucket-key' is not supported yet."); + ASSERT_NOK_WITH_MSG(system_table->Copy({{Options::TYPE, "format-table"}}), + "Change 'type' is not supported yet."); } TEST(SystemTableTest, TestReadOptimizedSystemTablePathParsing) { @@ -400,7 +415,7 @@ TEST(SystemTableTest, TestNewReadRefusesAFullTextMatchPredicate) { ASSERT_OK_AND_ASSIGN(std::shared_ptr in_memory, SystemTableLoader::Load(OptionsSystemTable::kName, /*fs=*/nullptr, "/tmp/table", table_schema, - /*dynamic_options=*/{})); + /*dynamic_options=*/{}, /*data_table_name=*/"")); ReadOptimizedSystemTable read_optimized(/*fs=*/nullptr, "/tmp/table", table_schema, "table", options); diff --git a/src/paimon/core/table/table_test.cpp b/src/paimon/core/table/table_test.cpp index ce896d83a..bc1dbcf9b 100644 --- a/src/paimon/core/table/table_test.cpp +++ b/src/paimon/core/table/table_test.cpp @@ -368,6 +368,62 @@ TEST(TableTest, TestACopyOfACopyServesTheLatestValueOfARepeatedDynamicOption) { copy->NewRead(std::move(read_context))); } +TEST(TableTest, TestACopySwitchingBranchesKeepsItsAccumulatedDynamicOptions) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::shared_ptr fs = dir->GetFileSystem(); + const std::string path = dir->Str() + "/tbl"; + const Identifier identifier("db", "tbl"); + + SchemaManager schema_manager(fs, path, "main"); + auto schema = arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false)}); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr created, + schema_manager.CreateTable(schema, /*partition_keys=*/{}, /*primary_keys=*/{"id"}, + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + // The dev branch holds its own schema, one the sequence-number switch never reached. + SchemaManager dev_schema_manager(fs, path, "dev"); + ASSERT_OK_AND_ASSIGN( + [[maybe_unused]] std::unique_ptr dev_schema, + dev_schema_manager.CreateTable(schema, /*partition_keys=*/{}, /*primary_keys=*/{"id"}, + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + FileStoreTable::Create( + fs, path, std::shared_ptr(std::move(created)), identifier, + /*uuid=*/"", /*branch=*/"main", /*fs_scheme_map=*/{}, + /*dynamic_options=*/{})); + + // A copy onto another branch still applies the options a previous copy accumulated - the dev + // schema omits the sequence-number switch, yet the copy serves it enabled, or a read of + // _SEQUENCE_NUMBER through it fails while the record names the switch as effective. + ASSERT_OK_AND_ASSIGN(std::shared_ptr
with_sequence, + table->Copy({{Options::KEY_VALUE_SEQUENCE_NUMBER_ENABLED, "true"}})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
dev_copy, + with_sequence->Copy({{Options::BRANCH, "dev"}})); + auto dev_file_store_copy = std::dynamic_pointer_cast(dev_copy); + ASSERT_TRUE(dev_file_store_copy != nullptr); + EXPECT_EQ("dev", dev_file_store_copy->CurrentBranch()); + std::shared_ptr dev_copied_schema = + std::dynamic_pointer_cast(dev_copy->LatestSchema()); + ASSERT_TRUE(dev_copied_schema != nullptr); + EXPECT_EQ("true", dev_copied_schema->Options().at(Options::KEY_VALUE_SEQUENCE_NUMBER_ENABLED)); + ReadContextBuilder read_builder(path); + read_builder.SetReadFieldNames({"id", "_SEQUENCE_NUMBER"}); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr read, + dev_copy->NewRead(std::move(read_context))); + + // A key both the accumulated options and this call name takes this call's value. + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
off, + with_sequence->Copy( + {{Options::BRANCH, "dev"}, {Options::KEY_VALUE_SEQUENCE_NUMBER_ENABLED, "false"}})); + std::shared_ptr off_schema = + std::dynamic_pointer_cast(off->LatestSchema()); + ASSERT_TRUE(off_schema != nullptr); + EXPECT_EQ("false", off_schema->Options().at(Options::KEY_VALUE_SEQUENCE_NUMBER_ENABLED)); +} + TEST(TableTest, TestAFallbackReaderServesItsBranchsOwnSchema) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); diff --git a/src/paimon/core/utils/context_utils.h b/src/paimon/core/utils/context_utils.h index 57e337904..428bc32d2 100644 --- a/src/paimon/core/utils/context_utils.h +++ b/src/paimon/core/utils/context_utils.h @@ -19,6 +19,11 @@ #pragma once #include "fmt/format.h" + +#include +#include + +#include "paimon/common/utils/sensitive_config_utils.h" #include "paimon/status.h" namespace paimon { @@ -46,6 +51,26 @@ class ContextUtils { } return Status::OK(); } + + /// An operator created from a table serves that table's own configuration and adds nothing + /// from the context a caller hands it, so a context carrying any option at all is refused: + /// dynamic configuration belongs to `Table::Copy()`, and a branch is named through + /// `WithBranch()` or the identifier of `WithCatalog()`, never through an option here. The + /// one-step `Create()` entry points fold the options they are given into the table they load + /// and clear them, so their contexts arrive empty. `kind` names the operator the message + /// speaks of. + static Status CheckContextCarriesNoOptions(const std::string& kind, + const std::map& options) { + if (options.empty()) { + return Status::OK(); + } + const auto& [key, value] = *options.begin(); + return Status::Invalid(fmt::format( + "the {} context carries option '{}' as '{}', but an operator created from a table " + "serves that table's own configuration and adds nothing from the context: apply " + "dynamic configuration through Table::Copy()", + kind, key, SensitiveConfigUtils::RedactValue(key, value))); + } }; } // namespace paimon From 940e507eb10bc46b6b98a2936aaaf6adcc3e25e6 Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Tue, 3 Nov 2026 22:41:15 +0800 Subject: [PATCH 29/30] refactor(table): separate context options and align copy semantics Pass dynamic options explicitly to operator factories and use Table::Copy for table overrides. Align managed and system table copy behavior with Java, preserve historical schemas for time travel, guard incompatible options, and simplify new comments. BREAKING CHANGE: operator contexts no longer accept dynamic options or WithBranch. Static operator Create calls require an options argument. --- benchmark/benchmark_suite.cpp | 40 +- docs/source/user_guide/catalog.rst | 47 +- docs/source/user_guide/format_table.rst | 7 +- docs/source/user_guide/global_index.rst | 11 +- .../user_guide/manifest_entry_cache.rst | 5 +- examples/clean_demo.cpp | 12 +- examples/read_write_demo.cpp | 14 +- include/paimon/catalog/catalog.h | 19 +- include/paimon/catalog/table.h | 45 +- include/paimon/commit_context.h | 95 +- include/paimon/file_store_commit.h | 17 +- include/paimon/file_store_write.h | 17 +- include/paimon/read_context.h | 95 +- include/paimon/scan_context.h | 69 +- include/paimon/table/format/format_table.h | 57 +- include/paimon/table/source/table_read.h | 18 +- include/paimon/table/source/table_scan.h | 17 +- include/paimon/write_context.h | 83 +- .../append_compact_coordinator_test.cpp | 4 +- src/paimon/core/catalog/catalog_utils.h | 25 +- .../core/catalog/file_system_catalog.cpp | 2 - src/paimon/core/catalog/file_system_catalog.h | 3 +- .../core/catalog/file_system_catalog_test.cpp | 136 +- src/paimon/core/catalog/snapshot_commit.h | 25 +- .../global_index/global_index_write_task.cpp | 10 +- ...eted_primary_key_index_maintainer_test.cpp | 42 +- .../pksorted/pk_sorted_data_file_reader.cpp | 10 +- ..._changelog_merge_tree_compact_rewriter.cpp | 11 +- .../lookup_merge_tree_compact_rewriter.cpp | 11 +- ...erge_tree_compact_manager_factory_test.cpp | 5 +- .../compact/merge_tree_compact_rewriter.cpp | 11 +- .../merge_tree_compact_rewriter_test.cpp | 2 +- src/paimon/core/mergetree/lookup_levels.cpp | 11 +- .../append_only_file_store_scan_test.cpp | 34 +- .../append_only_file_store_write.cpp | 10 +- .../append_only_file_store_write_test.cpp | 64 +- src/paimon/core/operation/commit_context.cpp | 90 +- .../core/operation/commit_context_test.cpp | 114 +- .../data_evolution_split_read_test.cpp | 2 +- .../core/operation/file_store_commit.cpp | 8 +- .../operation/file_store_commit_impl_test.cpp | 1007 ++++++++----- .../core/operation/file_store_commit_test.cpp | 29 +- .../core/operation/file_store_write.cpp | 9 +- .../core/operation/file_store_write_test.cpp | 47 +- .../core/operation/internal_read_context.cpp | 28 +- .../core/operation/internal_read_context.h | 47 +- .../operation/internal_read_context_test.cpp | 108 +- .../key_value_file_store_write_test.cpp | 105 +- .../operation/merge_file_split_read_test.cpp | 184 ++- .../operation/raw_file_split_read_test.cpp | 63 +- src/paimon/core/operation/read_context.cpp | 43 +- .../core/operation/read_context_test.cpp | 101 +- src/paimon/core/operation/scan_context.cpp | 51 +- .../core/operation/scan_context_test.cpp | 39 +- src/paimon/core/operation/write_context.cpp | 71 +- .../core/operation/write_context_test.cpp | 103 +- src/paimon/core/schema/schema_validation.cpp | 132 +- src/paimon/core/schema/schema_validation.h | 17 +- src/paimon/core/schema/table_schema.cpp | 5 + src/paimon/core/schema/table_schema.h | 15 +- .../core/table/append_only_file_store_table.h | 8 +- src/paimon/core/table/file_store.h | 54 +- src/paimon/core/table/file_store_table.cpp | 231 +-- src/paimon/core/table/file_store_table.h | 124 +- .../core/table/file_store_table_factory.cpp | 59 +- .../core/table/file_store_table_factory.h | 21 +- src/paimon/core/table/format/format_table.cpp | 18 +- .../core/table/format/format_table_test.cpp | 169 +-- .../core/table/primary_key_file_store_table.h | 8 +- .../core/table/sink/commit_message_test.cpp | 10 +- .../table/source/append_count_reader_test.cpp | 10 +- .../table/source/pk_count_reader_test.cpp | 10 +- .../core/table/source/realtime_table_scan.h | 4 +- .../static_from_tag_starting_scanner.h | 3 +- src/paimon/core/table/source/table_read.cpp | 9 +- .../core/table/source/table_read_test.cpp | 69 +- src/paimon/core/table/source/table_scan.cpp | 9 +- .../core/table/source/table_scan_test.cpp | 46 +- .../table/system/audit_log_system_table.cpp | 20 +- .../table/system/audit_log_system_table.h | 7 +- .../core/table/system/binlog_system_table.h | 4 +- .../table/system/global_system_tables.cpp | 6 - .../core/table/system/global_system_tables.h | 12 +- .../system/read_optimized_system_table.cpp | 27 +- .../system/read_optimized_system_table.h | 4 +- src/paimon/core/table/system/system_table.cpp | 112 +- src/paimon/core/table/system/system_table.h | 58 +- .../core/table/system/system_table_test.cpp | 55 +- src/paimon/core/table/table_test.cpp | 620 +++++--- src/paimon/core/utils/blob_view_lookup.cpp | 7 +- src/paimon/core/utils/branch_manager.cpp | 4 +- src/paimon/core/utils/branch_manager.h | 11 +- src/paimon/core/utils/branch_manager_test.cpp | 56 +- src/paimon/core/utils/context_utils.h | 32 +- src/paimon/rest/rest_catalog.cpp | 8 +- src/paimon/rest/rest_catalog.h | 3 +- src/paimon/rest/rest_catalog_test.cpp | 9 +- src/paimon/testing/mock/mock_catalog.h | 5 - src/paimon/testing/utils/test_helper.h | 36 +- test/inte/append_compaction_inte_test.cpp | 14 +- test/inte/blob_table_inte_test.cpp | 26 +- test/inte/clean_inte_test.cpp | 171 ++- test/inte/data_evolution_table_test.cpp | 88 +- test/inte/global_index_test.cpp | 14 +- test/inte/nested_column_pruning_inte_test.cpp | 73 +- test/inte/paimon_read_compat_inte_test.cpp | 18 +- test/inte/pk_compaction_inte_test.cpp | 113 +- .../primary_key_sorted_index_inte_test.cpp | 18 +- test/inte/read_inte_test.cpp | 591 +++++--- test/inte/realtime_write_inte_test.cpp | 295 ++-- test/inte/scan_and_read_inte_test.cpp | 1260 +++++++++++------ test/inte/scan_inte_test.cpp | 533 ++++--- test/inte/write_and_read_inte_test.cpp | 200 ++- .../write_and_read_inte_with_index_test.cpp | 60 +- test/inte/write_inte_test.cpp | 271 ++-- 115 files changed, 5047 insertions(+), 4158 deletions(-) diff --git a/benchmark/benchmark_suite.cpp b/benchmark/benchmark_suite.cpp index 6ec44f0d2..6eef62371 100644 --- a/benchmark/benchmark_suite.cpp +++ b/benchmark/benchmark_suite.cpp @@ -434,25 +434,29 @@ Status WriteSourceDataToWriter(paimon::FileStoreWrite* writer, const SourceDataS Status WriteAndCommit(const std::string& table_path, const std::map& options, const SourceDataSpec& source_spec) { + std::map write_builder_options; paimon::WriteContextBuilder write_builder(table_path, "benchmark-writer"); + write_builder_options = options; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_ctx, + AddContext(write_builder.Finish(), "create write context")); PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr write_ctx, - AddContext(write_builder.SetOptions(options).Finish(), "create write context")); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr writer, - AddContext(paimon::FileStoreWrite::Create(std::move(write_ctx)), - "create file store writer")); + std::unique_ptr writer, + AddContext(paimon::FileStoreWrite::Create(std::move(write_ctx), write_builder_options), + "create file store writer")); PAIMON_RETURN_NOT_OK(WriteSourceDataToWriter(writer.get(), source_spec)); PAIMON_ASSIGN_OR_RAISE(std::vector> messages, AddContext(writer->PrepareCommit(), "prepare commit")); + std::map commit_builder_options; paimon::CommitContextBuilder commit_builder(table_path, "benchmark-writer"); - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr commit_ctx, - AddContext(commit_builder.SetOptions(options).Finish(), "create commit context")); + commit_builder_options = options; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_ctx, + AddContext(commit_builder.Finish(), "create commit context")); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr committer, - AddContext(paimon::FileStoreCommit::Create(std::move(commit_ctx)), "create committer")); + AddContext(paimon::FileStoreCommit::Create(std::move(commit_ctx), commit_builder_options), + "create committer")); PAIMON_RETURN_NOT_OK(AddContext(committer->Commit(messages), "commit write")); return Status::OK(); } @@ -562,20 +566,23 @@ Result GetOrCreateSharedReadTable(const std::string Result ReadRows(const std::string& table_path, const std::map& options, int32_t prefetch_parallel_num) { + std::map scan_builder_options; paimon::ScanContextBuilder scan_builder(table_path); - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr scan_ctx, - AddContext(scan_builder.SetOptions(options).Finish(), "create scan context")); + scan_builder_options = options; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_ctx, + AddContext(scan_builder.Finish(), "create scan context")); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr scanner, - AddContext(paimon::TableScan::Create(std::move(scan_ctx)), "create scanner")); + AddContext(paimon::TableScan::Create(std::move(scan_ctx), scan_builder_options), + "create scanner")); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, AddContext(scanner->CreatePlan(), "create plan")); + std::map read_builder_options; paimon::ReadContextBuilder read_builder(table_path); constexpr int32_t kPrefetchBatchCount = 600; - read_builder.SetOptions(options) - .EnablePrefetch(true) + read_builder_options = options; + read_builder.EnablePrefetch(true) .SetPrefetchBatchCount(kPrefetchBatchCount) .SetPrefetchMaxParallelNum(prefetch_parallel_num) .EnableMultiThreadRowToBatch(GetRowToBatchThreadNumber() > 1) @@ -584,7 +591,8 @@ Result ReadRows(const std::string& table_path, AddContext(read_builder.Finish(), "create read context")); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr reader, - AddContext(paimon::TableRead::Create(std::move(read_ctx)), "create table reader")); + AddContext(paimon::TableRead::Create(std::move(read_ctx), read_builder_options), + "create table reader")); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch_reader, AddContext(reader->CreateReader(plan->Splits()), "create batch reader")); diff --git a/docs/source/user_guide/catalog.rst b/docs/source/user_guide/catalog.rst index a7a9ccdc4..0b0ac6f20 100644 --- a/docs/source/user_guide/catalog.rst +++ b/docs/source/user_guide/catalog.rst @@ -26,6 +26,45 @@ recommend that you use Catalog to access the Paimon table. Paimon C++ supports two metastores, selected with the catalog option ``metastore``: the filesystem metastore (default) and the REST metastore. +Dynamic options and time travel +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Use ``table->Copy(options)`` to apply dynamic options before creating operators. +Managed table copies retain the current schema and identifier when ``branch`` +changes. ``FileStoreTable::SwitchToBranch`` loads the target branch's latest schema. +Physical layout options such as ``merge-engine``, ``row-tracking.enabled`` and +``data-evolution.enabled`` cannot be changed by a copy. +Changing ``deletion-vectors.enabled`` requires the table's existing options to +contain ``deletion-vectors.modifiable=true``. Set that option before copying with +a different deletion-vectors mode. + +For time travel, specify one of ``scan.snapshot-id``, ``scan.tag-name``, +``scan.timestamp-millis`` or ``scan.timestamp``. The copy selects the historical +snapshot's schema, preserving runtime options. Create both the scan and read from +that copied table so they use the same historical columns. + +Selectors are mutually exclusive and must match an explicit ``scan.mode``. +Conflicting selectors, including ones accumulated across multiple copies, are +rejected. Data views such as ``$ro``, ``$audit_log`` and ``$binlog`` also retain +the selected historical schema when creating their scan and read operators. + +For example: + +.. code-block:: cpp + + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr historical, + table->Copy({{paimon::Options::SCAN_SNAPSHOT_ID, "1"}})); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, + historical->NewScan(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, + historical->NewRead(std::move(read_context))); + +If the snapshot cannot be resolved, the copy keeps its current schema and the +scan reports the snapshot error. A snapshot whose schema is missing makes the +copy fail. The string-valued C++ option map cannot express Java's ``null`` option +deletion. ``scan.version`` and watermark-based time travel are not supported. +Existing scan-specific restrictions, including tag and timestamp restrictions +for global-index scans on data-evolution tables, still apply. + Filesystem Catalog ~~~~~~~~~~~~~~~~~~ The filesystem metastore (``metastore=filesystem``, the default) stores both @@ -218,8 +257,12 @@ keep it across recovery. } paimon::CommitContextBuilder builder(table_path, "writer-1"); builder.WithCatalog(catalog, paimon::Identifier("db", "tbl")) - .WithTableId(table_uuid.value()) - .AddOption(paimon::Options::REST_COMMIT_ACQUIRE_LOCK, "true"); + .WithTableId(table_uuid.value()); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr locked_table, + table->Copy({{paimon::Options::REST_COMMIT_ACQUIRE_LOCK, "true"}})); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit, + locked_table->NewCommit(std::move(context))); Each attempt acquires its own lease before reading the latest snapshot, waits while another writer holds it until ``commit.timeout`` without using up diff --git a/docs/source/user_guide/format_table.rst b/docs/source/user_guide/format_table.rst index 037300ca1..db002f101 100644 --- a/docs/source/user_guide/format_table.rst +++ b/docs/source/user_guide/format_table.rst @@ -122,6 +122,10 @@ cannot decide what kind of table this is. The merged result is validated, not th options, so an option a format table refuses - ``metastore.partitioned-table``, a file format nothing here can read - is refused wherever it comes from rather than dropped in silence. +Like Java, ``Copy()`` retains the table's original physical file format even if +``file.format`` changes in the copied option map. Read and write operators +continue using that original format. + A setting the format path cannot act on is refused by name rather than quietly dropped: * ``ReadContextBuilder::SetReadSchema()``. A projected read schema can rename a column, prune a @@ -143,8 +147,7 @@ A setting the format path cannot act on is refused by name rather than quietly d * the ``branch`` option, whatever names it: a format table keeps no metadata to branch, and its data is the files under its one location, which is where a read or a write would go whichever branch was asked for. It is refused wherever it comes from - the schema, a catalog that parsed - ``tbl$branch_dev``, or the options of a single call - and so is ``WithBranch()`` on a context - built from a format table. A branch quietly ignored would be the wrong answer everywhere and a + ``tbl$branch_dev``, or the options passed to ``Copy()``. A branch quietly ignored would be the wrong answer everywhere and a destructive one for an overwrite, which would clear the data it was told not to touch. The value naming the main branch is the default and is left alone. * A scan predicate or bucket filter, as above, and more than one partition filter: a scan descends diff --git a/docs/source/user_guide/global_index.rst b/docs/source/user_guide/global_index.rst index 897cfebbb..d9e4a6b7c 100644 --- a/docs/source/user_guide/global_index.rst +++ b/docs/source/user_guide/global_index.rst @@ -218,9 +218,14 @@ to both the index scan and the data scan: "f0", R"({"match":{"query":"paimon lake"}})", /*limit=*/10))); paimon::ScanContextBuilder scan_builder(table_path); - scan_builder.AddOption(paimon::Options::SCAN_MODE, "from-snapshot") - .AddOption(paimon::Options::SCAN_SNAPSHOT_ID, std::to_string(snapshot_id)) - .SetGlobalIndexResult(result); + scan_builder.SetGlobalIndexResult(result); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, + scan_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr scan, + paimon::TableScan::Create(std::move(scan_context), + {{paimon::Options::SCAN_MODE, "from-snapshot"}, + {paimon::Options::SCAN_SNAPSHOT_ID, std::to_string(snapshot_id)}})); **Configuration:** diff --git a/docs/source/user_guide/manifest_entry_cache.rst b/docs/source/user_guide/manifest_entry_cache.rst index 995d77e38..a3c9f7227 100644 --- a/docs/source/user_guide/manifest_entry_cache.rst +++ b/docs/source/user_guide/manifest_entry_cache.rst @@ -83,8 +83,11 @@ under std::unique_ptr scan_context, context_builder .WithCache(cache) - .AddOption(Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, "3") .Finish()); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr scan, + TableScan::Create(std::move(scan_context), + {{Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, "3"}})); Cache entries are scoped by table path, branch, and bucket, so they can be reused across newly created ``TableScan`` and ``FileStoreScan`` instances as diff --git a/examples/clean_demo.cpp b/examples/clean_demo.cpp index b2888a86a..c7e91cb2a 100644 --- a/examples/clean_demo.cpp +++ b/examples/clean_demo.cpp @@ -51,23 +51,27 @@ Status DropPartition(const std::string& table_path, PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, commit_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr committer, - FileStoreCommit::Create(std::move(commit_context))); + FileStoreCommit::Create(std::move(commit_context), {})); PAIMON_RETURN_NOT_OK(committer->DropPartition(partitions, /*commit_identifier=*/10)); return Status::OK(); } Status ExpireSnapshot(const std::string& table_path) { + std::map commit_context_builder_options; CommitContextBuilder commit_context_builder(table_path, /*commit_user=*/"commit_user_1"); std::map commit_options = { {Options::SNAPSHOT_NUM_RETAINED_MAX, "2"}, {Options::SNAPSHOT_NUM_RETAINED_MIN, "1"}, {Options::SNAPSHOT_TIME_RETAINED, "1ms"}, {Options::SNAPSHOT_CLEAN_EMPTY_DIRECTORIES, "true"}}; + commit_context_builder_options = commit_options; PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, - commit_context_builder.SetOptions(commit_options).Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr committer, - FileStoreCommit::Create(std::move(commit_context))); + + commit_context_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr committer, + FileStoreCommit::Create(std::move(commit_context), commit_context_builder_options)); PAIMON_RETURN_NOT_OK(committer->Expire()); return Status::OK(); diff --git a/examples/read_write_demo.cpp b/examples/read_write_demo.cpp index e911bedb1..4113bb83e 100644 --- a/examples/read_write_demo.cpp +++ b/examples/read_write_demo.cpp @@ -87,13 +87,16 @@ paimon::Status Run(const std::string& root_path, const std::string& db_name, PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, catalog->GetTable(paimon::Identifier(db_name, table_name))); + PAIMON_ASSIGN_OR_RAISE(table, table->Copy(options)); + std::string table_path = root_path + "/" + db_name + ".db/" + table_name; std::string commit_user = "some_commit_user"; // write paimon::WriteContextBuilder context_builder(table_path, commit_user); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, - context_builder.SetOptions(options).Finish()); + + context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr writer, table->NewWrite(std::move(write_context))); @@ -117,7 +120,8 @@ paimon::Status Run(const std::string& root_path, const std::string& db_name, // commit paimon::CommitContextBuilder commit_context_builder(table_path, commit_user); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, - commit_context_builder.SetOptions(options).Finish()); + + commit_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr committer, table->NewCommit(std::move(commit_context))); PAIMON_RETURN_NOT_OK(committer->Commit(commit_message)); @@ -125,7 +129,8 @@ paimon::Status Run(const std::string& root_path, const std::string& db_name, // scan paimon::ScanContextBuilder scan_context_builder(table_path); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, - scan_context_builder.SetOptions(options).Finish()); + + scan_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scanner, table->NewScan(std::move(scan_context))); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, scanner->CreatePlan()); @@ -133,7 +138,8 @@ paimon::Status Run(const std::string& root_path, const std::string& db_name, // read paimon::ReadContextBuilder read_context_builder(table_path); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, - read_context_builder.SetOptions(options).Finish()); + + read_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, table->NewRead(std::move(read_context))); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch_reader, diff --git a/include/paimon/catalog/catalog.h b/include/paimon/catalog/catalog.h index 3898a1967..9c4aef683 100644 --- a/include/paimon/catalog/catalog.h +++ b/include/paimon/catalog/catalog.h @@ -69,9 +69,7 @@ class PAIMON_EXPORT Catalog { /// tokens, keeps routing each scheme to its backend. Ignored when /// `file_system` is supplied, which serves every scheme itself. /// @return A result containing a shared pointer to a `Catalog` instance, or an error status. - /// @note Shared rather than unique ownership: a global system table of the `sys` database - /// builds its rows lazily through the catalog it was handed, so the table keeps the - /// catalog alive. A catalog built by this factory always satisfies that. + /// @note Global system tables retain shared ownership of the catalog. static Result> Create( const std::string& root_path, const std::map& options, const std::shared_ptr& file_system = nullptr, @@ -148,21 +146,12 @@ class PAIMON_EXPORT Catalog { virtual Status RenameTable(const Identifier& from_table, const Identifier& to_table, bool ignore_if_not_exists) = 0; - /// Gets a table. + /// Gets a managed, format or system table through the common Table interface. /// - /// The table's `type` option decides what is handed back: a `format-table` comes back as the - /// `FormatTable` it is - a directory of data files, with no snapshot and no manifest - and - /// every other implemented type as the managed table it is, while a `T$system` identifier names - /// a read-only `SystemTable`. All go through this one `Table` interface: `NewScan()` and - /// `NewRead()` serve any of them, `NewWrite()` and `NewCommit()` the writable ones. - /// - /// A global system table of the "sys" database reads its rows lazily through the catalog it - /// came from, so the table keeps shared ownership of that catalog. `Catalog::Create()` hands - /// back a `std::shared_ptr`, which satisfies this. A catalog held some other way, such as - /// through a `std::unique_ptr`, cannot be kept and is refused by name. + /// Global views that enumerate tables require shared catalog ownership. /// /// @param identifier Identifier of the table to get. - /// @return A result containing the table, or an error status. + /// @return The table, or an error status. virtual Result> GetTable(const Identifier& identifier) const = 0; /// Checks whether a database with the specified name exists in the catalog. diff --git a/include/paimon/catalog/table.h b/include/paimon/catalog/table.h index 6e401ff6b..9bdd306ea 100644 --- a/include/paimon/catalog/table.h +++ b/include/paimon/catalog/table.h @@ -42,35 +42,26 @@ class TableRead; class TableScan; class WriteContext; -/// A table provides basic abstraction for table type. -/// -/// The state every kind of table carries - file system, root path, schema and identifier - lives -/// here, and each kind fills in the operations. The `NewXxx` operators below are the table-first -/// entry points: the deprecated `TableScan::Create()` and its friends load the table a context -/// names and hand the context to it. A table refuses by name whatever a context asks for that it -/// cannot honour, rather than dropping it silently. +/// Common interface for managed, format and system tables. +/// Create operators with NewScan/NewRead/NewWrite/NewCommit; unsupported operations return errors. class PAIMON_EXPORT Table { public: virtual ~Table() = default; - /// Returns the file system the table was opened with, null when the opener had none. A - /// catalog that hands out per-table file systems opens a table with the one for that table. + /// Returns the table file system, or null if none was supplied. std::shared_ptr GetFileSystem() const { return file_system_; } - /// Returns the root path of the table in its file system. It is empty when the opener had no - /// location for the table, which a mock in a test is one case of. + /// Returns the table root path, or an empty string if unavailable. const std::string& GetRootPath() const { return root_path_; } - /// A name to identify this table. A managed or format table answers the object name of its - /// identifier, a system table its own name. + /// Returns the object name for data tables, or the system table name. virtual std::string Name() const = 0; - /// Full name of the table: `database.tableName` for a managed or format table, the same name - /// as `Name()` for a system table. + /// Returns database.tableName for data tables, or Name() for system tables. virtual std::string FullName() const = 0; /// Returns the metastore UUID, falling back to `FullName()`. @@ -90,36 +81,24 @@ class PAIMON_EXPORT Table { return schema_; } - /// Copies this table with `dynamic_options` on top of the options it carries. An option given - /// at the call wins over the one the schema stored, the precedence every context builder - /// promises. `type` is not overridable: it is structural and read from the schema alone. - /// - /// @param dynamic_options Options given at the call. - /// @return A result containing the copied table, or the error copying it reports. + /// Copies this table with dynamic_options overriding its current options. + /// The table type is structural and cannot be changed by these options. virtual Result> Copy( const std::map& dynamic_options) const = 0; - /// Creates a scan of this table from a finished scan context. - /// @param context The finished scan context. - /// @return The scan, or the error building it reports. + /// Creates a scan using this table's schema and options. virtual Result> NewScan( std::unique_ptr context) const = 0; - /// Creates a read of this table from a finished read context. - /// @param context The finished read context. - /// @return The read, or the error building it reports. + /// Creates a reader using this table's schema and options. virtual Result> NewRead( std::unique_ptr context) const = 0; - /// Creates a write of this table from a finished write context. - /// @param context The finished write context. - /// @return The write, or the error building it reports. + /// Creates a writer using this table's schema and options. virtual Result> NewWrite( std::unique_ptr context) const = 0; - /// Creates a commit of this table from a finished commit context. - /// @param context The finished commit context. - /// @return The commit, or the error building it reports. + /// Creates a commit using this table's schema and options. virtual Result> NewCommit( std::unique_ptr context) const = 0; diff --git a/include/paimon/commit_context.h b/include/paimon/commit_context.h index 0c19ec5d6..ee0fc517a 100644 --- a/include/paimon/commit_context.h +++ b/include/paimon/commit_context.h @@ -47,8 +47,7 @@ class PAIMON_EXPORT CommitContext { bool ignore_empty_commit, bool use_rest_catalog_commit, bool append_commit_check_conflict, const std::shared_ptr& memory_pool, const std::shared_ptr& executor, - const std::shared_ptr& specific_file_system, - const std::map& options); + const std::shared_ptr& specific_file_system); CommitContext(const std::string& root_path, const std::string& commit_user, bool ignore_empty_commit, bool use_rest_catalog_commit, @@ -58,8 +57,6 @@ class PAIMON_EXPORT CommitContext { const std::shared_ptr& memory_pool, const std::shared_ptr& executor, const std::shared_ptr& specific_file_system, - const std::map& options, const std::string& branch, - const std::optional& named_branch, const std::optional& table_schema = std::nullopt, const std::map& fs_scheme_to_identifier_map = {}); ~CommitContext(); @@ -68,25 +65,11 @@ class PAIMON_EXPORT CommitContext { return root_path_; } - /// Returns the table root path. An alias of `GetRootPath()`, so that every context - /// exposes the table location under the same name. + /// Alias for GetRootPath(). const std::string& GetPath() const { return GetRootPath(); } - /// Returns the branch the builder resolved out of the identifier and the options. - const std::string& GetBranch() const { - return branch_; - } - - /// Returns the branch the identifier of `WithCatalog()` or the `branch` option named, or - /// nullopt when neither did. An operator serves the branch the table it belongs to was loaded - /// with, so a table refuses a context naming a branch it does not serve - dropping the name - /// in silence would publish the changes on its own branch. - const std::optional& GetNamedBranch() const { - return named_branch_; - } - const std::string& GetCommitUser() const { return commit_user_; } @@ -99,7 +82,7 @@ class PAIMON_EXPORT CommitContext { return use_rest_catalog_commit_; } - /// Returns the table schema the caller set, or nullopt when the commit loads it. + /// Returns the optional schema supplied by the caller. const std::optional& GetSpecificTableSchema() const { return table_schema_; } @@ -139,22 +122,8 @@ class PAIMON_EXPORT CommitContext { return fs_scheme_to_identifier_map_; } - const std::map& GetOptions() const { - return options_; - } - - /// Empties the options this context carries. A one-step `FileStoreCommit::Create()`-style - /// entry point folds them into the table it loads and clears them before handing the context - /// to `NewCommit`, which serves that table's own configuration alone; a caller holding a - /// table calls `NewCommit` with no options at all. - void ClearOptions() { - options_.clear(); - } - private: std::string root_path_; - std::string branch_; - std::optional named_branch_; std::string commit_user_; bool ignore_empty_commit_; bool use_rest_catalog_commit_; @@ -166,7 +135,6 @@ class PAIMON_EXPORT CommitContext { std::shared_ptr executor_; std::shared_ptr specific_file_system_; std::map fs_scheme_to_identifier_map_; - std::map options_; std::optional table_schema_; }; @@ -180,33 +148,6 @@ class PAIMON_EXPORT CommitContextBuilder { ~CommitContextBuilder(); - /// Set a configuration options map to set some option entries which are not defined in the - /// table schema or whose values you want to overwrite. A load merges them into the table it - /// hands back; an option contradicting a table you loaded already is refused, so change that - /// one through `Table::Copy()`. - /// @note The options map will clear the options added by `AddOption()` before. - /// @param options The configuration options map. - /// @return Reference to this builder for method chaining. - /// @deprecated These options reach an operator only through the deprecated - /// `FileStoreCommit::Create()`-style entry points, which merge them into the table they - /// load and then hand that table a context carrying none. An operator created from a `Table` - /// serves that table's own configuration alone and refuses a context carrying any option at - /// all, so pass dynamic options through `Table::Copy()` instead. - PAIMON_DEPRECATED("Pass dynamic options through Table::Copy() instead") - CommitContextBuilder& SetOptions(const std::map& options); - - /// Add a single configuration option, served the way `SetOptions()` describes. - /// - /// If you want to add multiple options, call `AddOption()` multiple times or use `SetOptions()` - /// instead. - /// @param key The option key. - /// @param value The option value. - /// @return Reference to this builder for method chaining. - /// @deprecated Use `Table::Copy()` to pass dynamic options, for the reason `SetOptions()` - /// gives. - PAIMON_DEPRECATED("Pass dynamic options through Table::Copy() instead") - CommitContextBuilder& AddOption(const std::string& key, const std::string& value); - /// Sets whether to ignore empty commits (default is true). /// When set to true, commits that don't contain any actual data changes will be ignored. /// @param ignore_empty_commit True to ignore empty commits, false otherwise. @@ -271,32 +212,16 @@ class PAIMON_EXPORT CommitContextBuilder { /// @note If not set, use default file system (configured in `Options::FILE_SYSTEM`) CommitContextBuilder& WithFileSystem(const std::shared_ptr& file_system); - /// Sets a mapping from URI schemes (e.g., "file", "oss") to registered file system - /// identifiers. This allows selecting different pre-registered file system implementations - /// based on the URI scheme at runtime. - /// - /// @param fs_scheme_to_identifier_map Map from URI scheme (like "oss") to the corresponding - /// file system identifier. - /// @return Reference to this builder for method chaining. - /// @note - /// - This method is intended for environments where multiple file systems are pre-registered. - /// - The specified identifiers must correspond to file systems that have been registered at - /// compile time or initialization. - /// - Cannot be used together with `WithFileSystem()`. - /// - If not set, use default file system (configured in `Options::FILE_SYSTEM`). - /// Example: - /// builder.WithFileSystemSchemeToIdentifierMap({{"oss", "jindo"}, {"file", "local"}}); - /// + /// Maps URI schemes to registered file system identifiers. + /// Mutually exclusive with WithFileSystem(); defaults to Options::FILE_SYSTEM when unset. + /// @param fs_scheme_to_identifier_map Mapping such as {{"oss", "jindo"}, {"file", "local"}}. + /// @return This builder. CommitContextBuilder& WithFileSystemSchemeToIdentifierMap( const std::map& fs_scheme_to_identifier_map); - /// Set the table schema as a string to avoid schema loading I/O operations. - /// - /// @param table_schema String representation of the table schema. - /// @return Reference to this builder for method chaining. - /// @note The user must ensure that the schema string is valid and matches the table. - /// @note If not set, the schema is read from the catalog of `WithCatalog()` on the main - /// branch, and loaded from the table path otherwise. + /// Supplies a serialized table schema to skip schema loading. + /// The caller must ensure it matches the table. Without it, main uses the catalog schema + /// when WithCatalog() is set; other branches load from the table path. CommitContextBuilder& SetTableSchema(const std::string& table_schema); /// Build and return a `CommitContext` instance with input validation. diff --git a/include/paimon/file_store_commit.h b/include/paimon/file_store_commit.h index c495cf281..7e0c2bbb4 100644 --- a/include/paimon/file_store_commit.h +++ b/include/paimon/file_store_commit.h @@ -51,16 +51,15 @@ class CommitMessage; /// manages versions. `UseRESTCatalogCommit()` only prepares a request for the caller to send. class PAIMON_EXPORT FileStoreCommit { public: - /// Create an instance of `FileStoreCommit`. The data table the context names is loaded from its - /// root path - a format table is refused, served by the table a catalog hands back through its - /// own `NewCommit` instead. - /// - /// @param context A unique pointer to the `CommitContext` used for commit operations. - /// - /// @return A Result containing a unique pointer to the `FileStoreCommit` instance. - /// @deprecated Use `Table::NewCommit()` on the table a catalog hands back instead. + /// Creates a managed-table commit from the context and table options. + /// @param context Commit context. + /// @param options Overrides for the stored schema options. Catalog branch commits require a + /// branch-qualified identifier. + /// @return The commit, or an error status. + /// @deprecated Use Table::NewCommit(). PAIMON_DEPRECATED("Use Table::NewCommit() instead") - static Result> Create(std::unique_ptr context); + static Result> Create( + std::unique_ptr context, const std::map& options); virtual ~FileStoreCommit() = default; diff --git a/include/paimon/file_store_write.h b/include/paimon/file_store_write.h index 697541aac..ecb945e96 100644 --- a/include/paimon/file_store_write.h +++ b/include/paimon/file_store_write.h @@ -43,16 +43,15 @@ class WriteContext; /// Interface for write operations in a file store. class PAIMON_EXPORT FileStoreWrite { public: - /// Create an instance of `FileStoreWrite`. The data table the context names is loaded from its - /// root path - a format table is refused, served by the table a catalog hands back through its - /// own `NewWrite` instead. - /// - /// @param context A unique pointer to the `WriteContext` used for write operations. - /// - /// @return A Result containing a unique pointer to the `FileStoreWrite` instance. - /// @deprecated Use `Table::NewWrite()` on the table a catalog hands back instead. + /// Creates a managed-table writer from the context and table options. + /// @param context Write context. + /// @param options Overrides for the stored schema options. Catalog branch writes require a + /// branch-qualified identifier. + /// @return The writer, or an error status. + /// @deprecated Use Table::NewWrite(). PAIMON_DEPRECATED("Use Table::NewWrite() instead") - static Result> Create(std::unique_ptr context); + static Result> Create( + std::unique_ptr context, const std::map& options); virtual ~FileStoreWrite() = default; diff --git a/include/paimon/read_context.h b/include/paimon/read_context.h index b1d330c8d..7eb3aa17f 100644 --- a/include/paimon/read_context.h +++ b/include/paimon/read_context.h @@ -52,25 +52,21 @@ struct VectorSearch; /// @see ReadContextBuilder class PAIMON_EXPORT ReadContext { public: - ReadContext(const std::string& path, const std::string& branch, - const std::vector& read_field_names, - const std::vector& read_field_ids, - const std::shared_ptr& predicate, - const std::shared_ptr& vector_search, - const std::shared_ptr& full_text_search, - bool enable_predicate_filter, bool enable_prefetch, bool enable_late_materializing, - uint32_t prefetch_batch_count, uint32_t prefetch_max_parallel_num, - bool enable_multi_thread_row_to_batch, uint32_t row_to_batch_thread_number, - const std::optional& table_schema, - const std::shared_ptr& memory_pool, - const std::shared_ptr& executor, - const std::shared_ptr& specific_file_system, - const std::map& fs_scheme_to_identifier_map, - const std::shared_ptr& realtime_context, - const std::map& options, bool read_ahead_cache_enabled, - const CacheConfig& cache_config, const std::shared_ptr& cache, - WarmupLevel warmup_level, const std::shared_ptr& catalog, - const std::optional& identifier); + ReadContext( + const std::string& path, const std::vector& read_field_names, + const std::vector& read_field_ids, const std::shared_ptr& predicate, + const std::shared_ptr& vector_search, + const std::shared_ptr& full_text_search, bool enable_predicate_filter, + bool enable_prefetch, bool enable_late_materializing, uint32_t prefetch_batch_count, + uint32_t prefetch_max_parallel_num, bool enable_multi_thread_row_to_batch, + uint32_t row_to_batch_thread_number, const std::optional& table_schema, + const std::shared_ptr& memory_pool, const std::shared_ptr& executor, + const std::shared_ptr& specific_file_system, + const std::map& fs_scheme_to_identifier_map, + const std::shared_ptr& realtime_context, bool read_ahead_cache_enabled, + const CacheConfig& cache_config, const std::shared_ptr& cache, + WarmupLevel warmup_level, const std::shared_ptr& catalog, + const std::optional& identifier); ~ReadContext(); @@ -78,26 +74,10 @@ class PAIMON_EXPORT ReadContext { return path_; } - const std::string& GetBranch() const { - return branch_; - } - const std::map& GetFileSystemSchemeToIdentifierMap() const { return fs_scheme_to_identifier_map_; } - const std::map& GetOptions() const { - return options_; - } - - /// Empties the options this context carries. A one-step `TableRead::Create()`-style entry point - /// folds them into the table it loads and clears them before handing the context to `NewRead`, - /// which serves that table's own configuration alone; a caller holding a table calls - /// `NewRead` with no options at all. - void ClearOptions() { - options_.clear(); - } - const std::vector& GetReadFieldNames() const { return read_field_names_; } @@ -188,8 +168,7 @@ class PAIMON_EXPORT ReadContext { return identifier_; } - /// Returns the catalog table UUID, so that every context answers `GetTableId()` the way - /// `CommitContext` does; a read carries none. + /// Returns nullopt; read contexts do not carry a table UUID. const std::optional& GetTableId() const { static const std::optional kNoTableId; return kNoTableId; @@ -213,7 +192,6 @@ class PAIMON_EXPORT ReadContext { private: std::string path_; - std::string branch_; std::vector read_field_names_; std::vector read_field_ids_; std::shared_ptr predicate_; @@ -232,7 +210,6 @@ class PAIMON_EXPORT ReadContext { std::shared_ptr specific_file_system_; std::map fs_scheme_to_identifier_map_; std::shared_ptr realtime_context_; - std::map options_; bool read_ahead_cache_enabled_; CacheConfig cache_config_; std::shared_ptr cache_; @@ -325,33 +302,6 @@ class PAIMON_EXPORT ReadContextBuilder { /// When set, read_field_ids and read_field_names are ignored. ReadContextBuilder& SetReadSchema(std::unique_ptr read_schema); - /// Set a configuration options map to set some option entries which are not defined in the - /// table schema or whose values you want to overwrite. A load merges them into the table it - /// hands back; an option contradicting a table you loaded already is refused, so change that - /// one through `Table::Copy()`. - /// @note The options map will clear the options added by `AddOption()` before. - /// @param options The configuration options map. - /// @return Reference to this builder for method chaining. - /// @deprecated These options reach an operator only through the deprecated - /// `TableRead::Create()`-style entry points, which merge them into the table they load and - /// then hand that table a context carrying none. An operator created from a `Table` serves - /// that table's own configuration alone and refuses a context carrying any option at all, so - /// pass dynamic options through `Table::Copy()` instead. - PAIMON_DEPRECATED("Pass dynamic options through Table::Copy() instead") - ReadContextBuilder& SetOptions(const std::map& options); - - /// Add a single configuration option, served the way `SetOptions()` describes. - /// - /// If you want to add multiple options, call `AddOption()` multiple times or use `SetOptions()` - /// instead. - /// @param key The option key. - /// @param value The option value. - /// @return Reference to this builder for method chaining. - /// @deprecated Use `Table::Copy()` to pass dynamic options, for the reason `SetOptions()` - /// gives. - PAIMON_DEPRECATED("Pass dynamic options through Table::Copy() instead") - ReadContextBuilder& AddOption(const std::string& key, const std::string& value); - /// Set a predicate for filtering data during reading. /// /// The predicate is used for both partition pruning and data filtering. @@ -513,19 +463,6 @@ class PAIMON_EXPORT ReadContextBuilder { /// @note If not set, the schema will be loaded from the table path. ReadContextBuilder& SetTableSchema(const std::string& table_schema); - /// Set the specific branch to read from in a versioned table. - /// - /// Paimon supports branching for data versioning and time travel queries. - /// This method allows reading from a specific branch instead of the main branch. - /// - /// The `branch` option names the branch too, as it does for a scan of one branch; naming two - /// different branches is refused rather than silently resolved. - /// - /// @param branch Name of the branch to read from. - /// @return Reference to this builder for method chaining. - /// @note Default branch is "main" if not specified. An empty name is the main branch. - ReadContextBuilder& WithBranch(const std::string& branch); - /// Sets a mapping from URI schemes (e.g., "file", "oss") to registered file system /// identifiers. This allows selecting different pre-registered file system implementations /// based on the URI scheme at runtime. diff --git a/include/paimon/scan_context.h b/include/paimon/scan_context.h index b88b6037d..aacb40c86 100644 --- a/include/paimon/scan_context.h +++ b/include/paimon/scan_context.h @@ -55,10 +55,9 @@ class PAIMON_EXPORT ScanContext { const std::shared_ptr& memory_pool, const std::shared_ptr& executor, const std::shared_ptr& specific_file_system, - const std::optional& table_schema, - const std::map& options, - const std::shared_ptr& cache, const std::shared_ptr& catalog, - const std::optional& identifier, const std::string& branch, + const std::optional& table_schema, const std::shared_ptr& cache, + const std::shared_ptr& catalog, + const std::optional& identifier, const std::map& fs_scheme_to_identifier_map = {}); ~ScanContext(); @@ -67,11 +66,6 @@ class PAIMON_EXPORT ScanContext { return path_; } - /// Returns the branch the builder resolved out of the identifier and the options. - const std::string& GetBranch() const { - return branch_; - } - bool IsStreamingMode() const { return is_streaming_mode_; } @@ -83,18 +77,6 @@ class PAIMON_EXPORT ScanContext { std::shared_ptr GetScanFilters() const { return scan_filters_; } - const std::map& GetOptions() const { - return options_; - } - - /// Empties the options this context carries. A one-step `TableScan::Create()`-style entry point - /// folds them into the table it loads and clears them before handing the context to `NewScan`, - /// which serves that table's own configuration alone; a caller holding a table calls - /// `NewScan` with no options at all. - void ClearOptions() { - options_.clear(); - } - std::shared_ptr GetMemoryPool() const { return memory_pool_; } @@ -137,8 +119,7 @@ class PAIMON_EXPORT ScanContext { return identifier_; } - /// Returns the catalog table UUID, so that every context answers `GetTableId()` the way - /// `CommitContext` does; a scan carries none. + /// Returns nullopt; scan contexts do not carry a table UUID. const std::optional& GetTableId() const { static const std::optional kNoTableId; return kNoTableId; @@ -146,7 +127,6 @@ class PAIMON_EXPORT ScanContext { private: std::string path_; - std::string branch_; bool is_streaming_mode_; std::optional limit_; std::shared_ptr scan_filters_; @@ -157,7 +137,6 @@ class PAIMON_EXPORT ScanContext { std::shared_ptr specific_file_system_; std::map fs_scheme_to_identifier_map_; std::optional table_schema_; - std::map options_; std::shared_ptr cache_; std::shared_ptr catalog_; std::optional identifier_; @@ -218,26 +197,6 @@ class PAIMON_EXPORT ScanContextBuilder { ScanContextBuilder& WithRealtimeContext( const std::shared_ptr& realtime_context); - /// Add a single configuration option, served the way `SetOptions()` describes. - /// @deprecated Use `Table::Copy()` to pass dynamic options, for the reason `SetOptions()` - /// gives. - PAIMON_DEPRECATED("Pass dynamic options through Table::Copy() instead") - ScanContextBuilder& AddOption(const std::string& key, const std::string& value); - /// Set a configuration options map to set some option entries which are not defined in the - /// table schema or whose values you want to overwrite. A load merges them into the table it - /// hands back; an option contradicting a table you loaded already is refused, so change that - /// one through `Table::Copy()`. - /// @note The options map will clear the options added by `AddOption()` before. - /// @param options The configuration options map. - /// @return Reference to this builder for method chaining. - /// @deprecated These options reach an operator only through the deprecated - /// `TableScan::Create()`-style entry points, which merge them into the table they load and - /// then hand that table a context carrying none. An operator created from a `Table` serves - /// that table's own configuration alone and refuses a context carrying any option at all, so - /// pass dynamic options through `Table::Copy()` instead. - PAIMON_DEPRECATED("Pass dynamic options through Table::Copy() instead") - ScanContextBuilder& SetOptions(const std::map& options); - /// Set whether the scan is in streaming mode. /// @note if not set, is_streaming_mode = false ScanContextBuilder& WithStreamingMode(bool is_streaming_mode); @@ -258,22 +217,10 @@ class PAIMON_EXPORT ScanContextBuilder { /// @note If not set, use default file system (configured in `Options::FILE_SYSTEM`) ScanContextBuilder& WithFileSystem(const std::shared_ptr& file_system); - /// Sets a mapping from URI schemes (e.g., "file", "oss") to registered file system - /// identifiers. This allows selecting different pre-registered file system implementations - /// based on the URI scheme at runtime. - /// - /// @param fs_scheme_to_identifier_map Map from URI scheme (like "oss") to the corresponding - /// file system identifier. - /// @return Reference to this builder for method chaining. - /// @note - /// - This method is intended for environments where multiple file systems are pre-registered. - /// - The specified identifiers must correspond to file systems that have been registered at - /// compile time or initialization. - /// - Cannot be used together with `WithFileSystem()`. - /// - If not set, use default file system (configured in `Options::FILE_SYSTEM`). - /// Example: - /// builder.WithFileSystemSchemeToIdentifierMap({{"oss", "jindo"}, {"file", "local"}}); - /// + /// Maps URI schemes to registered file system identifiers. + /// Mutually exclusive with WithFileSystem(); defaults to Options::FILE_SYSTEM when unset. + /// @param fs_scheme_to_identifier_map Mapping such as {{"oss", "jindo"}, {"file", "local"}}. + /// @return This builder. ScanContextBuilder& WithFileSystemSchemeToIdentifierMap( const std::map& fs_scheme_to_identifier_map); diff --git a/include/paimon/table/format/format_table.h b/include/paimon/table/format/format_table.h index c5bfe60d3..3082eea6a 100644 --- a/include/paimon/table/format/format_table.h +++ b/include/paimon/table/format/format_table.h @@ -37,26 +37,16 @@ namespace paimon { class FileSystem; -/// A table that is a directory of data files of one format, laid out like a standard Hive table. +/// A directory of data files in one format, without snapshots or manifests. +/// Partitions use key=value directories, or bare values with +/// format-table.partition-path-only-value. The type option must be format-table; file.format +/// defaults to parquet. /// -/// It carries no snapshots and no manifests: the files under its location are the table, and a -/// partitioned table's partitions are the `key=value` directories below that location, or the -/// bare-value ones under `format-table.partition-path-only-value`. A table is a format table when -/// its `type` option is `format-table`; `file.format` then names the format of every file in it, -/// defaulting to `parquet`. +/// Writes accept inserts only, and reads expose _VALUE_KIND as INSERT. Load through +/// Catalog::GetTable() and create operators through the Table interface. Instances must be owned by +/// shared_ptr. /// -/// Writes only insert - there is nowhere to record an update or a delete - so a read fills -/// `_VALUE_KIND` with inserts throughout. -/// -/// A format table is a `Table`, so it is reached both ways: through the `NewXxx` operators below, -/// and through the entry points every other table uses - `TableScan::Create()` and friends - which -/// recognise one from the schema under the table path, or take one already loaded through the -/// `FormatTable` constructor every context builder has, how a table whose schema lives in a -/// metastore is reached. -/// -/// Every instance lives in a `shared_ptr`: a copy that adds nothing hands the same one back. -/// -/// See `docs/source/user_guide/format_table.rst` for what is not supported yet. +/// See docs/source/user_guide/format_table.rst for supported features. class PAIMON_EXPORT FormatTable : public Table, public std::enable_shared_from_this { public: /// Formats a format table's files can be in. @@ -72,18 +62,14 @@ class PAIMON_EXPORT FormatTable : public Table, public std::enable_shared_from_t /// The identifier of a format, as it appears in `file.format` and as a file extension. static std::string FormatToString(Format format); - /// Loads a format table from its directory, reading the schema stored under it. - /// - /// This needs a schema file under the table directory, which a table created through - /// `SchemaManager` or a file system catalog has. A table whose schema lives in a metastore has - /// none, and is loaded through `Catalog::GetTable()` instead. + /// Loads a format table from a schema stored under its data directory. + /// Use Catalog::GetTable() for tables whose schema is stored in a metastore. /// /// @param file_system File system holding the table directory. - /// @param table_path Root path of the table, which is also its data location. - /// @param identifier Logical table identifier, used for naming and error messages. - /// @param dynamic_options Options given at the call, which win over the ones stored in the - /// schema. Empty when the caller has none. - /// @return A result containing the format table, or an error status. + /// @param table_path Table directory and data location. + /// @param identifier Logical table identifier. + /// @param dynamic_options Overrides for the stored schema options. + /// @return The table, or an error status. static Result> Create( const std::shared_ptr& file_system, const std::string& table_path, const Identifier& identifier, const std::map& dynamic_options); @@ -116,7 +102,7 @@ class PAIMON_EXPORT FormatTable : public Table, public std::enable_shared_from_t return identifier_.GetFullName(); } - /// The same table comes back when there is nothing to add. + /// Returns the same instance when dynamic_options is empty. Result> Copy( const std::map& dynamic_options) const override; @@ -158,8 +144,7 @@ class PAIMON_EXPORT FormatTable : public Table, public std::enable_shared_from_t return options_; } - /// Schema of the table, including its partition fields. Hides `Table::LatestSchema()` to hand - /// back the same schema as the `DataSchema` a format table is always created with. + /// Returns the data schema, including partition columns. std::shared_ptr LatestSchema() const { return std::static_pointer_cast(schema_); } @@ -176,9 +161,7 @@ class PAIMON_EXPORT FormatTable : public Table, public std::enable_shared_from_t return location_carries_paimon_metadata_; } - /// The four operators below create this table's scan, read, write and commit, applying the - /// context's options on top of the ones this table carries and refusing by name what a - /// format table cannot honour. + /// Creates operators using this table's options and the supplied operation context. Result> NewScan(std::unique_ptr context) const override; Result> NewRead(std::unique_ptr context) const override; @@ -196,14 +179,12 @@ class PAIMON_EXPORT FormatTable : public Table, public std::enable_shared_from_t const std::string& file_compression, const std::string& partition_default_name, bool partition_only_value_in_path, bool location_carries_paimon_metadata); - /// This table as a `shared_ptr`. `shared_from_this()` on a const method hands back a - /// `shared_ptr`, while the copy below needs a mutable one; the cast is - /// safe because the table is never written to. + /// Returns a mutable shared_ptr from this const instance; the table itself is not modified. std::shared_ptr SharedSelf() const { return std::const_pointer_cast(shared_from_this()); } - /// `Copy()` as a `FormatTable` rather than a `Table`, which is what its operators need. + /// Typed implementation of Copy() for internal callers. Result> CopySelf( const std::map& dynamic_options) const; diff --git a/include/paimon/table/source/table_read.h b/include/paimon/table/source/table_read.h index 36de26880..14f168851 100644 --- a/include/paimon/table/source/table_read.h +++ b/include/paimon/table/source/table_read.h @@ -19,7 +19,9 @@ #pragma once +#include #include +#include #include #include "paimon/executor.h" @@ -43,15 +45,15 @@ class PAIMON_EXPORT TableRead { public: virtual ~TableRead() = default; - /// Create an instance of `TableRead`. The data table the context names is loaded from its path - /// - a system table behind a `T$system` path and a format table are refused, served by the - /// table a catalog hands back through its own `NewRead` instead. - /// - /// @param context A unique pointer to the `ReadContext` used for read operations. - /// @return A Result containing a unique pointer to the `TableRead` instance. - /// @deprecated Use `Table::NewRead()` on the table a catalog hands back instead. + /// Creates a managed-table reader from the context and table options. + /// System and format tables require Table::NewRead(). + /// @param context Read context. + /// @param options Overrides for the stored schema options. + /// @return The reader, or an error status. + /// @deprecated Use Table::NewRead(). PAIMON_DEPRECATED("Use Table::NewRead() instead") - static Result> Create(std::unique_ptr context); + static Result> Create( + std::unique_ptr context, const std::map& options); /// Creates a `BatchReader` instance for reading data. /// diff --git a/include/paimon/table/source/table_scan.h b/include/paimon/table/source/table_scan.h index ea14f4fb8..4f68a84c4 100644 --- a/include/paimon/table/source/table_scan.h +++ b/include/paimon/table/source/table_scan.h @@ -37,15 +37,16 @@ class ScanContext; /// A scanner interface for reading table's meta and create a plan. class PAIMON_EXPORT TableScan { public: - /// Create an instance of `TableScan`. The data table the context names is loaded from its path - /// - a system table behind a `T$system` path and a format table are refused, served by the - /// table a catalog hands back through its own `NewScan` instead. - /// - /// @param context A unique pointer to the `ScanContext` used for scan operations. - /// @return A Result containing a unique pointer to the `TableScan` instance. - /// @deprecated Use `Table::NewScan()` on the table a catalog hands back instead. + /// Creates a managed-table scan from the context and table options. + /// System and format tables require Table::NewScan(). + /// @param context Scan context. + /// @param options Overrides for the stored schema options. The branch option must agree with + /// any branch in the identifier. + /// @return The scan, or an error status. + /// @deprecated Use Table::NewScan(). PAIMON_DEPRECATED("Use Table::NewScan() instead") - static Result> Create(std::unique_ptr context); + static Result> Create( + std::unique_ptr context, const std::map& options); virtual ~TableScan() = default; diff --git a/include/paimon/write_context.h b/include/paimon/write_context.h index 0d831991b..14717b284 100644 --- a/include/paimon/write_context.h +++ b/include/paimon/write_context.h @@ -46,14 +46,12 @@ class PAIMON_EXPORT WriteContext { WriteContext(const std::string& root_path, const std::string& commit_user, bool is_streaming_mode, bool ignore_num_bucket_check, bool ignore_previous_files, bool enable_multi_thread_spill, const std::optional& write_id, - const std::string& branch, const std::optional& named_branch, const std::vector& write_schema, const std::shared_ptr& memory_pool, const std::shared_ptr& executor, const std::string& temp_directory, const std::shared_ptr& specific_file_system, const std::map& fs_scheme_to_identifier_map, const std::shared_ptr& realtime_context, - const std::map& options, const std::optional& table_schema = std::nullopt, const std::shared_ptr& catalog = nullptr, const std::optional& identifier = std::nullopt); @@ -64,8 +62,7 @@ class PAIMON_EXPORT WriteContext { return root_path_; } - /// Returns the table root path. An alias of `GetRootPath()`, so that every context - /// exposes the table location under the same name. + /// Alias for GetRootPath(). const std::string& GetPath() const { return GetRootPath(); } @@ -78,18 +75,6 @@ class PAIMON_EXPORT WriteContext { return fs_scheme_to_identifier_map_; } - const std::map& GetOptions() const { - return options_; - } - - /// Empties the options this context carries. A one-step `FileStoreWrite::Create()`-style entry - /// point folds them into the table it loads and clears them before handing the context to - /// `NewWrite`, which serves that table's own configuration alone; a caller holding a table - /// calls `NewWrite` with no options at all. - void ClearOptions() { - options_.clear(); - } - bool IsStreamingMode() const { return is_streaming_mode_; } @@ -106,18 +91,6 @@ class PAIMON_EXPORT WriteContext { return write_id_; } - const std::string& GetBranch() const { - return branch_; - } - - /// Returns the branch the identifier of `WithCatalog()`, the `branch` option or - /// `WithBranch()` named, or nullopt when none of them did. An operator serves the branch the - /// table it belongs to was loaded with, so a table refuses a context naming a branch it does - /// not serve - dropping the name in silence would publish the changes on its own branch. - const std::optional& GetNamedBranch() const { - return named_branch_; - } - const std::vector& GetWriteSchema() const { return write_schema_; } @@ -147,7 +120,7 @@ class PAIMON_EXPORT WriteContext { return realtime_context_; } - /// Returns the table schema the caller set, or nullopt when the write loads it. + /// Returns the optional schema supplied by the caller. const std::optional& GetSpecificTableSchema() const { return table_schema_; } @@ -162,8 +135,7 @@ class PAIMON_EXPORT WriteContext { return identifier_; } - /// Returns the catalog table UUID, so that every context answers `GetTableId()` the way - /// `CommitContext` does; a write carries none. + /// Returns nullopt; write contexts do not carry a table UUID. const std::optional& GetTableId() const { static const std::optional kNoTableId; return kNoTableId; @@ -172,8 +144,6 @@ class PAIMON_EXPORT WriteContext { private: std::string root_path_; std::string commit_user_; - std::string branch_; - std::optional named_branch_; bool is_streaming_mode_; bool ignore_num_bucket_check_; bool ignore_previous_files_; @@ -186,7 +156,6 @@ class PAIMON_EXPORT WriteContext { std::shared_ptr specific_file_system_; std::map fs_scheme_to_identifier_map_; std::shared_ptr realtime_context_; - std::map options_; std::optional table_schema_; std::shared_ptr catalog_; std::optional identifier_; @@ -202,33 +171,6 @@ class PAIMON_EXPORT WriteContextBuilder { ~WriteContextBuilder(); - /// Set a configuration options map to set some option entries which are not defined in the - /// table schema or whose values you want to overwrite. A load merges them into the table it - /// hands back; an option contradicting a table you loaded already is refused, so change that - /// one through `Table::Copy()`. - /// @note The options map will clear the options added by `AddOption()` before. - /// @param options The configuration options map. - /// @return Reference to this builder for method chaining. - /// @deprecated These options reach an operator only through the deprecated - /// `FileStoreWrite::Create()`-style entry points, which merge them into the table they load - /// and then hand that table a context carrying none. An operator created from a `Table` - /// serves that table's own configuration alone and refuses a context carrying any option at - /// all, so pass dynamic options through `Table::Copy()` instead. - PAIMON_DEPRECATED("Pass dynamic options through Table::Copy() instead") - WriteContextBuilder& SetOptions(const std::map& options); - - /// Add a single configuration option, served the way `SetOptions()` describes. - /// - /// If you want to add multiple options, call `AddOption()` multiple times or use `SetOptions()` - /// instead. - /// @param key The option key. - /// @param value The option value. - /// @return Reference to this builder for method chaining. - /// @deprecated Use `Table::Copy()` to pass dynamic options, for the reason `SetOptions()` - /// gives. - PAIMON_DEPRECATED("Pass dynamic options through Table::Copy() instead") - WriteContextBuilder& AddOption(const std::string& key, const std::string& value); - /// Set whether to enable streaming mode (default is false) /// @return Reference to this builder for method chaining. WriteContextBuilder& WithStreamingMode(bool is_streaming_mode); @@ -266,15 +208,6 @@ class PAIMON_EXPORT WriteContextBuilder { /// @return Reference to this builder for method chaining. WriteContextBuilder& WithWriteId(int32_t write_id); - /// Write to specific branch, default is main. - /// - /// The identifier of `WithCatalog()` and the `branch` option may name the branch too; naming - /// two different ones is refused rather than silently resolved. An empty name is the main - /// branch. - /// @param branch Name of the branch to write to. - /// @return Reference to this builder for method chaining. - WriteContextBuilder& WithBranch(const std::string& branch); - /// For data evolution, user can write partial specific fields from table schema. /// If not set, write all fields in table. /// @return Reference to this builder for method chaining. @@ -329,13 +262,9 @@ class PAIMON_EXPORT WriteContextBuilder { WriteContextBuilder& WithFileSystemSchemeToIdentifierMap( const std::map& fs_scheme_to_identifier_map); - /// Set the table schema as a string to avoid schema loading I/O operations. - /// - /// @param table_schema String representation of the table schema. - /// @return Reference to this builder for method chaining. - /// @note The user must ensure that the schema string is valid and matches the table. - /// @note If not set, the schema is read from the catalog of `WithCatalog()` on the main - /// branch, and loaded from the table path otherwise. + /// Supplies a serialized table schema to skip schema loading. + /// The caller must ensure it matches the table. Without it, main uses the catalog schema + /// when WithCatalog() is set; other branches load from the table path. WriteContextBuilder& SetTableSchema(const std::string& table_schema); /// Set the thread number for write buffer spill operations. (default is 0) diff --git a/src/paimon/core/append/append_compact_coordinator_test.cpp b/src/paimon/core/append/append_compact_coordinator_test.cpp index acdf59cb0..be52d0b07 100644 --- a/src/paimon/core/append/append_compact_coordinator_test.cpp +++ b/src/paimon/core/append/append_compact_coordinator_test.cpp @@ -83,7 +83,7 @@ class AppendCompactCoordinatorTest : public ::testing::Test { write_builder.WithStreamingMode(false); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, write_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(auto file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(std::move(write_context), {})); ArrowArray c_array; PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*write_array, &c_array)); auto record_batch = std::make_unique( @@ -101,7 +101,7 @@ class AppendCompactCoordinatorTest : public ::testing::Test { PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, commit_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_store_commit, - FileStoreCommit::Create(std::move(commit_context))); + FileStoreCommit::Create(std::move(commit_context), {})); return file_store_commit->Commit(commit_msgs); } diff --git a/src/paimon/core/catalog/catalog_utils.h b/src/paimon/core/catalog/catalog_utils.h index 58775d4df..3630d4820 100644 --- a/src/paimon/core/catalog/catalog_utils.h +++ b/src/paimon/core/catalog/catalog_utils.h @@ -32,17 +32,13 @@ class Catalog; class FormatTable; class Schema; -/// The identifier tools and the checks shared by the catalog implementations. Every check takes an -/// `action` naming the rejected operation in the error message, e.g. "dropTable". +/// Identifier utilities shared by catalog implementations. class CatalogUtils { public: CatalogUtils() = delete; ~CatalogUtils() = delete; - /// The identifier a table path names when no catalog name goes with it, the C++ reading of the - /// Java `SchemaManagerUtils#identifierFromPath` over `ignoreIfUnknownDatabase = true`: the - /// directory above the table's names the database when it carries the `.db` suffix a catalog - /// lays its databases out with, and a path that names no database leaves it unknown. + /// Derives the database from the parent directory's .db suffix; otherwise leaves it unknown. static Identifier IdentifierFromPath(const std::string& table_path, const std::string& branch); /// Returns whether `db_name` is the reserved system database "sys". @@ -65,9 +61,7 @@ class CatalogUtils { static Result BranchIdentifier(const Identifier& identifier, const std::string& branch); - /// The identifier a catalog knows the table by, on `branch`, when there is one to fold a - /// branch into: a caller without an identifier has none to ask the catalog about, and gets - /// none back rather than an error. + /// Returns an identifier qualified with branch, or nullopt if no identifier was supplied. static Result> BranchIdentifier( const std::optional& identifier, const std::string& branch); @@ -79,16 +73,9 @@ class CatalogUtils { /// branch name, system table name) cannot be used as a single path component. static Status CheckValidTableName(const Identifier& identifier); - /// Fails when `schema` is not a table a managed `Table` can describe. - /// - /// Only a managed table is one: the remaining table types are stored differently and are - /// not implemented here, so handing one back as a `Table` would promise snapshots it never - /// had. A schema that is not a data table's, such as a system table's, passes. A format - /// table is served by the caller - `GetTable()` hands one back as the `FormatTable` it is - - /// so its schema only reaches this check by mistake and falls to the refusal below. - /// - /// @param action The call being refused, qualified as a caller would write it, e.g. - /// `Catalog::GetTable`. It names the entry point in the message. + /// Rejects unsupported managed-table types; system-table schemas pass through. + /// Format tables must be handled by the caller. + /// @param action Operation name used in the error message, e.g. Catalog::GetTable. static Status CheckManagedTableType(const Identifier& identifier, const std::shared_ptr& schema, const std::string& action); diff --git a/src/paimon/core/catalog/file_system_catalog.cpp b/src/paimon/core/catalog/file_system_catalog.cpp index a7dcd01ed..3be877c1e 100644 --- a/src/paimon/core/catalog/file_system_catalog.cpp +++ b/src/paimon/core/catalog/file_system_catalog.cpp @@ -385,8 +385,6 @@ Result> FileSystemCatalog::LoadSystemTable( dynamic_options[Options::BRANCH] = branch.value(); } PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(data_identifier)); - // The data table's logical name, branch included, so that a view of a branch answers with the - // name the branch is addressed by rather than the bare directory name. return SystemTableLoader::Load(system_table_name.value(), fs_, table_path, latest_schema.value(), dynamic_options, data_identifier.GetObjectName()); diff --git a/src/paimon/core/catalog/file_system_catalog.h b/src/paimon/core/catalog/file_system_catalog.h index 0f3123238..47e6a179d 100644 --- a/src/paimon/core/catalog/file_system_catalog.h +++ b/src/paimon/core/catalog/file_system_catalog.h @@ -86,8 +86,7 @@ class FileSystemCatalog : public Catalog, public std::enable_shared_from_this>> TableSchemaExists( const Identifier& identifier) const; - /// Loads the system table a `T$system` identifier names, on the branch the identifier - /// carries. The base table it belongs to must exist and carry a schema. + /// Loads a T$system view from the identifier's branch. The base table must have a schema. Result> LoadSystemTable(const Identifier& identifier) const; Status CreateDatabaseImpl(const std::string& db_name, diff --git a/src/paimon/core/catalog/file_system_catalog_test.cpp b/src/paimon/core/catalog/file_system_catalog_test.cpp index b02ac50ec..a4a9a3104 100644 --- a/src/paimon/core/catalog/file_system_catalog_test.cpp +++ b/src/paimon/core/catalog/file_system_catalog_test.cpp @@ -282,16 +282,24 @@ TEST(FileSystemCatalogTest, TestGetTableServesASystemTable) { ASSERT_TRUE(std::dynamic_pointer_cast(system_table) != nullptr); ASSERT_EQ("tbl1$options", system_table->Name()); + std::map scan_builder_options; ScanContextBuilder scan_builder(system_table->GetRootPath()); - scan_builder.SetOptions(options); + scan_builder_options = options; + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto scan, system_table->NewScan(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_1, + system_table->Copy(scan_builder_options)); + ASSERT_OK_AND_ASSIGN(auto scan, configured_table_1->NewScan(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto plan, scan->CreatePlan()); + std::map read_builder_options; ReadContextBuilder read_builder(system_table->GetRootPath()); - read_builder.SetOptions(options); + read_builder_options = options; + ASSERT_OK_AND_ASSIGN(auto read_context, read_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto read, system_table->NewRead(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_2, + system_table->Copy(read_builder_options)); + ASSERT_OK_AND_ASSIGN(auto read, configured_table_2->NewRead(std::move(read_context))); ASSERT_OK_AND_ASSIGN(auto reader, read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(auto table_result, reader->NextBatch()); @@ -302,24 +310,28 @@ TEST(FileSystemCatalogTest, TestGetTableServesASystemTable) { batch = std::move(maybe_batch).ValueUnsafe(); ASSERT_GE(batch->num_rows(), 2); + std::map write_builder_options; WriteContextBuilder write_builder(system_table->GetRootPath(), "commit_user"); - write_builder.SetOptions(options); + write_builder_options = options; + ASSERT_OK_AND_ASSIGN(auto write_context, write_builder.Finish()); - ASSERT_NOK_WITH_MSG(system_table->NewWrite(std::move(write_context)), + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_3, + system_table->Copy(write_builder_options)); + ASSERT_NOK_WITH_MSG(configured_table_3->NewWrite(std::move(write_context)), "tbl1$options system table is read-only: writing is not supported"); + std::map commit_builder_options; CommitContextBuilder commit_builder(system_table->GetRootPath(), "commit_user"); - commit_builder.SetOptions(options); + commit_builder_options = options; + ASSERT_OK_AND_ASSIGN(auto commit_context, commit_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_4, + system_table->Copy(commit_builder_options)); ASSERT_NOK_WITH_MSG( - system_table->NewCommit(std::move(commit_context)), + configured_table_4->NewCommit(std::move(commit_context)), "tbl1$options system table is read-only: committing for it is not supported"); } -// A system table a catalog hands out is copied the way a data table is. The dynamic options the -// copy is given are served by it - a `tbl$ro` view that selected a snapshot through `Copy` reads -// that snapshot's rows - and a `branch` option routes the copy to the branch it names, the way -// Java Paimon's system tables re-wrap `dataTable.copy(dynamicOptions)`. TEST(FileSystemCatalogTest, TestACopyOfACatalogSystemTableAppliesItsOptions) { std::map options; options[Options::FILE_SYSTEM] = "local"; @@ -339,13 +351,13 @@ TEST(FileSystemCatalogTest, TestACopyOfACatalogSystemTableAppliesItsOptions) { ArrowSchemaRelease(&schema); ASSERT_OK_AND_ASSIGN(std::shared_ptr
managed, catalog.GetTable(identifier)); - auto write_and_commit_two_rows = [](const std::shared_ptr
& table) { + auto write_and_commit_two_rows = [](const std::shared_ptr
& table, int32_t first_id = 1) { WriteContextBuilder write_builder(table->GetRootPath(), "commit-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr write, table->NewWrite(std::move(write_context))); arrow::Int32Builder ids_builder; - ASSERT_TRUE(ids_builder.AppendValues({1, 2}).ok()); + ASSERT_TRUE(ids_builder.AppendValues({first_id, first_id + 1}).ok()); std::shared_ptr ids_array; ASSERT_TRUE(ids_builder.Finish(&ids_array).ok()); auto struct_array = @@ -368,7 +380,7 @@ TEST(FileSystemCatalogTest, TestACopyOfACatalogSystemTableAppliesItsOptions) { ASSERT_OK(commit->Commit(messages)); ASSERT_OK(write->Close()); }; - // Two commits, so snapshot 1 holds the first two rows and snapshot 2 all four. + // Snapshot 1 has two rows; snapshot 2 has four. write_and_commit_two_rows(managed); write_and_commit_two_rows(managed); @@ -389,8 +401,6 @@ TEST(FileSystemCatalogTest, TestACopyOfACatalogSystemTableAppliesItsOptions) { return rows ? rows->length() : 0; }; - // A copy of `tbl$ro` selects the snapshot its options name: the view reads every row, the - // copy on snapshot 1 reads the two rows that commit wrote. ASSERT_OK_AND_ASSIGN(std::shared_ptr
ro, catalog.GetTable(Identifier("db1", "tbl1$ro"))); ASSERT_OK_AND_ASSIGN(int64_t all_rows, count_rows(ro)); EXPECT_EQ(all_rows, 4); @@ -400,7 +410,32 @@ TEST(FileSystemCatalogTest, TestACopyOfACatalogSystemTableAppliesItsOptions) { ASSERT_OK_AND_ASSIGN(int64_t selected_rows, count_rows(selected)); EXPECT_EQ(selected_rows, 2); - // The dev branch holds its own schema and one commit of two rows of its own. + const std::map pk_options = {{Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "orc"}, + {Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "id"}}; + ASSERT_TRUE(arrow::ExportSchema(*typed_schema, &schema).ok()); + ASSERT_OK(catalog.CreateTable(Identifier("db1", "pk_tbl"), &schema, /*partition_keys=*/{}, + /*primary_keys=*/{"id"}, pk_options, /*ignore_if_exists=*/false)); + ArrowSchemaRelease(&schema); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
pk_table, + catalog.GetTable(Identifier("db1", "pk_tbl"))); + write_and_commit_two_rows(pk_table, 1); + write_and_commit_two_rows(pk_table, 3); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
audit, + catalog.GetTable(Identifier("db1", "pk_tbl$audit_log"))); + ASSERT_OK_AND_ASSIGN(int64_t audit_rows, count_rows(audit)); + EXPECT_EQ(audit_rows, 4); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
audit_selected, + audit->Copy({{Options::SCAN_MODE, "from-snapshot"}, {Options::SCAN_SNAPSHOT_ID, "1"}})); + EXPECT_EQ(audit_selected->Name(), "pk_tbl$audit_log"); + ASSERT_OK_AND_ASSIGN(int64_t audit_selected_rows, count_rows(audit_selected)); + EXPECT_EQ(audit_selected_rows, 2); + + ASSERT_OK_AND_ASSIGN(int64_t original_audit_rows, count_rows(audit)); + EXPECT_EQ(original_audit_rows, 4); + SchemaManager dev_schema_manager(core_options.GetFileSystem(), managed->GetRootPath(), "dev"); ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr dev_schema, dev_schema_manager.CreateTable(typed_schema, /*partition_keys=*/{}, @@ -409,14 +444,12 @@ TEST(FileSystemCatalogTest, TestACopyOfACatalogSystemTableAppliesItsOptions) { catalog.GetTable(Identifier("db1", "tbl1$branch_dev"))); write_and_commit_two_rows(dev_managed); - // A `branch` option routes the copy to the branch it names: the copy answers with the branch - // view's name and reads the rows that branch holds. + // Copy retains the identifier while routing reads to dev. ASSERT_OK_AND_ASSIGN(std::shared_ptr
dev_selected, ro->Copy({{Options::BRANCH, "dev"}})); - EXPECT_EQ(dev_selected->Name(), "tbl1$branch_dev$ro"); + EXPECT_EQ(dev_selected->Name(), ro->Name()); ASSERT_OK_AND_ASSIGN(int64_t dev_rows, count_rows(dev_selected)); EXPECT_EQ(dev_rows, 2); - // A view of a branch keeps serving that branch through a copy. ASSERT_OK_AND_ASSIGN(std::shared_ptr
dev_ro, catalog.GetTable(Identifier("db1", "tbl1$branch_dev$ro"))); ASSERT_OK_AND_ASSIGN(std::shared_ptr
dev_ro_copy, dev_ro->Copy({})); @@ -424,8 +457,7 @@ TEST(FileSystemCatalogTest, TestACopyOfACatalogSystemTableAppliesItsOptions) { ASSERT_OK_AND_ASSIGN(int64_t dev_ro_rows, count_rows(dev_ro_copy)); EXPECT_EQ(dev_ro_rows, 2); - // The metadata tables copy the same way: the snapshots view serves the branch its copy names, - // and a branch holding no schema does not exist. + // Metadata views also route reads to the copied branch. ASSERT_OK_AND_ASSIGN(std::shared_ptr
snapshots, catalog.GetTable(Identifier("db1", "tbl1$snapshots"))); ASSERT_OK_AND_ASSIGN(int64_t main_snapshot_rows, count_rows(snapshots)); @@ -434,8 +466,10 @@ TEST(FileSystemCatalogTest, TestACopyOfACatalogSystemTableAppliesItsOptions) { snapshots->Copy({{Options::BRANCH, "dev"}})); ASSERT_OK_AND_ASSIGN(int64_t dev_snapshot_rows, count_rows(dev_snapshots)); EXPECT_EQ(dev_snapshot_rows, 1); - ASSERT_NOK_WITH_MSG(snapshots->Copy({{Options::BRANCH, "missing"}}), - "branch missing does not exist"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
missing, + snapshots->Copy({{Options::BRANCH, "missing"}})); + ASSERT_OK_AND_ASSIGN(int64_t missing_rows, count_rows(missing)); + EXPECT_EQ(missing_rows, 0); } TEST(FileSystemCatalogTest, TestGetTableServesTheSysDatabase) { @@ -469,24 +503,31 @@ TEST(FileSystemCatalogTest, TestGetTableServesTheSysDatabase) { catalog->GetTable(Identifier("sys", "catalog_options"))); ASSERT_TRUE(std::dynamic_pointer_cast(catalog_options) != nullptr); - // A global system table copies over the options it already carries and ignores the dynamic - // ones, as Java Paimon's `CatalogOptionsTable#copy` does, so its copy reads the same rows. + // Global system tables ignore dynamic options when copied. ASSERT_OK_AND_ASSIGN(std::shared_ptr
copied_catalog_options, catalog_options->Copy({{"ignored", "on-top"}})); EXPECT_NE(copied_catalog_options, catalog_options); + std::map copied_scan_builder_options; ScanContextBuilder copied_scan_builder(copied_catalog_options->GetRootPath()); - copied_scan_builder.SetOptions(options); + copied_scan_builder_options = options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr copied_scan_context, copied_scan_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_9, + copied_catalog_options->Copy(copied_scan_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr copied_scan, - copied_catalog_options->NewScan(std::move(copied_scan_context))); + configured_table_9->NewScan(std::move(copied_scan_context))); ASSERT_OK_AND_ASSIGN(std::shared_ptr copied_plan, copied_scan->CreatePlan()); + std::map copied_read_builder_options; ReadContextBuilder copied_read_builder(copied_catalog_options->GetRootPath()); - copied_read_builder.SetOptions(options); + copied_read_builder_options = options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr copied_read_context, copied_read_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_10, + copied_catalog_options->Copy(copied_read_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr copied_read, - copied_catalog_options->NewRead(std::move(copied_read_context))); + configured_table_10->NewRead(std::move(copied_read_context))); ASSERT_OK_AND_ASSIGN(std::unique_ptr copied_reader, copied_read->CreateReader(copied_plan->Splits())); ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch copied_result, copied_reader->NextBatch()); @@ -499,17 +540,25 @@ TEST(FileSystemCatalogTest, TestGetTableServesTheSysDatabase) { ASSERT_NOK_WITH_MSG(catalog->GetTable(Identifier("sys", "nope")), "not exist"); ASSERT_OK_AND_ASSIGN(auto tables, catalog->GetTable(Identifier("sys", "tables"))); + std::map scan_builder_options; ScanContextBuilder scan_builder("sys/tables"); - scan_builder.SetOptions(options); + scan_builder_options = options; + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto scan, tables->NewScan(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_11, + tables->Copy(scan_builder_options)); + ASSERT_OK_AND_ASSIGN(auto scan, configured_table_11->NewScan(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto plan, scan->CreatePlan()); ASSERT_EQ(plan->Splits().size(), 1); + std::map read_builder_options; ReadContextBuilder read_builder("sys/tables"); - read_builder.SetOptions(options); + read_builder_options = options; + ASSERT_OK_AND_ASSIGN(auto read_context, read_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto read, tables->NewRead(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_12, + tables->Copy(read_builder_options)); + ASSERT_OK_AND_ASSIGN(auto read, configured_table_12->NewRead(std::move(read_context))); ASSERT_OK_AND_ASSIGN(auto reader, read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(auto table_result, reader->NextBatch()); @@ -591,8 +640,7 @@ TEST(FileSystemCatalogTest, TestBranchTableLoadsBranchSchema) { ASSERT_TRUE(exists); ASSERT_OK_AND_ASSIGN(loaded_schema, catalog.LoadTableSchema(audit_log_identifier)); ASSERT_EQ((std::vector{"rowkind", "b1_c0"}), loaded_schema->FieldNames()); - // A view of a branch answers with the name the branch is addressed by, so reloading that - // name lands on the same branch rather than on the main one. + // The returned name must address the same branch when reloaded. ASSERT_OK_AND_ASSIGN(std::shared_ptr
branch_view, catalog.GetTable(audit_log_identifier)); ASSERT_EQ(branch_view->Name(), "tbl1$branch_b1$audit_log"); @@ -1697,13 +1745,15 @@ TEST(FileSystemCatalogTest, TestClaimingVersionManagementWithoutImplementingItIs ASSERT_NOK_WITH_MSG(CheckVersionManagementImplemented(claiming), "does not implement VersionManagedCatalog"); + std::map claiming_builder_options; CommitContextBuilder claiming_builder(dir->Str(), "commit_user_1"); + claiming_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr claiming_context, - claiming_builder.SetOptions(options) - .WithCatalog(claiming, Identifier("db", "tbl")) - .Finish()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(claiming_context)), - "does not implement VersionManagedCatalog"); + + claiming_builder.WithCatalog(claiming, Identifier("db", "tbl")).Finish()); + ASSERT_NOK_WITH_MSG( + FileStoreCommit::Create(std::move(claiming_context), claiming_builder_options), + "does not implement VersionManagedCatalog"); std::shared_ptr plain = std::make_shared(core_options.GetFileSystem(), dir->Str(), options); diff --git a/src/paimon/core/catalog/snapshot_commit.h b/src/paimon/core/catalog/snapshot_commit.h index 2062b6c2f..4df93f9b4 100644 --- a/src/paimon/core/catalog/snapshot_commit.h +++ b/src/paimon/core/catalog/snapshot_commit.h @@ -47,22 +47,15 @@ class SnapshotCommit { virtual Result> AcquireLock() { return std::unique_ptr(); } - /// The commit a table publishes its snapshots through, the C++ reading of the Java - /// `CatalogEnvironment#snapshotCommit`: a catalog that version-manages the table commits - /// through it, a caller that only wants the request built gets one with no catalog attached, - /// and everything else publishes by renaming the snapshot file into place. - /// - /// @param catalog The catalog the commit was handed, or null for one that has none. - /// @param catalog_identifier The identifier the catalog knows the table by, on the branch the - /// commit commits to. A version-managing catalog needs one and is refused without it. - /// @param options Table options merged with the writer's dynamic options. - /// @param table_id The catalog's UUID for the table, captured before the changes were - /// prepared; null when there is none. - /// @param commit_user The commit user of the snapshots published under a commit lock. - /// @param use_rest_catalog_commit Whether the caller asks for the request rather than for the - /// catalog to send it. - /// @param file_system The file system the snapshot file is renamed through. - /// @param snapshot_manager The manager that names the snapshot file renamed. + /// Selects catalog-backed or file-system snapshot publication. + /// @param catalog Optional catalog. + /// @param catalog_identifier Branch-qualified identifier required by version-managed catalogs. + /// @param options Effective table options. + /// @param table_id Table UUID captured before preparing changes, or nullopt. + /// @param commit_user Commit user for commit locking. + /// @param use_rest_catalog_commit Build the request without sending it through the catalog. + /// @param file_system File system used for snapshot rename. + /// @param snapshot_manager Manager providing the snapshot paths. static Result> Create( const std::shared_ptr& catalog, const std::optional& catalog_identifier, const CoreOptions& options, diff --git a/src/paimon/core/global_index/global_index_write_task.cpp b/src/paimon/core/global_index/global_index_write_task.cpp index f04d96bb0..3ba9c80bd 100644 --- a/src/paimon/core/global_index/global_index_write_task.cpp +++ b/src/paimon/core/global_index/global_index_write_task.cpp @@ -173,16 +173,18 @@ Result> CreateBatchReader( const std::string& table_path, const std::vector& read_field_names, const std::shared_ptr& indexed_split, const CoreOptions& core_options, const std::shared_ptr& pool) { + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - read_context_builder.SetOptions(core_options.ToMap()) - .WithFileSystem(core_options.GetFileSystem()) + read_context_builder_options = core_options.ToMap(); + read_context_builder.WithFileSystem(core_options.GetFileSystem()) .EnablePrefetch(true) .WithMemoryPool(pool) .SetReadFieldNames(read_field_names); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); return table_read->CreateReader(indexed_split); } diff --git a/src/paimon/core/index/pk/bucketed_primary_key_index_maintainer_test.cpp b/src/paimon/core/index/pk/bucketed_primary_key_index_maintainer_test.cpp index 85690b25f..75a0a8956 100644 --- a/src/paimon/core/index/pk/bucketed_primary_key_index_maintainer_test.cpp +++ b/src/paimon/core/index/pk/bucketed_primary_key_index_maintainer_test.cpp @@ -320,15 +320,17 @@ class BucketedPrimaryKeyIndexMaintainerTest : public ::testing::TestWithParam> CreateWriter(bool with_temp_directory = true, bool write_buffer_spillable = true) const { + std::map builder_options; WriteContextBuilder builder(table_path_, kCommitUser); std::map writer_options = options_; writer_options[Options::WRITE_BUFFER_SPILLABLE] = write_buffer_spillable ? "true" : "false"; - builder.SetOptions(writer_options).WithStreamingMode(true); + builder_options = writer_options; + builder.WithStreamingMode(true); if (with_temp_directory) { builder.WithTempDirectory(directory_->Str()); } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); - return FileStoreWrite::Create(std::move(context)); + return FileStoreWrite::Create(std::move(context), builder_options); } Result>> WriteAndPrepare( @@ -354,11 +356,13 @@ class BucketedPrimaryKeyIndexMaintainerTest : public ::testing::TestWithParam>& messages, int64_t commit_identifier) const { + std::map builder_options; CommitContextBuilder builder(table_path_, kCommitUser); - builder.SetOptions(options_); + builder_options = options_; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit, - FileStoreCommit::Create(std::move(context))); + FileStoreCommit::Create(std::move(context), builder_options)); return commit->Commit(messages, commit_identifier); } @@ -454,11 +458,13 @@ TEST_P(BucketedPrimaryKeyIndexMaintainerTest, BuildsRestoresAndReplacesCompacted std::shared_ptr value_predicate = PredicateBuilder::Equal( /*field_index=*/1, "value", FieldType::STRING, Literal(FieldType::STRING, indexed_value.data(), indexed_value.size())); + std::map scan_builder_options; ScanContextBuilder scan_builder(table_path_); - scan_builder.SetPredicate(value_predicate).AddOption(Options::GLOBAL_INDEX_ENABLED, "true"); + scan_builder_options[Options::GLOBAL_INDEX_ENABLED] = "true"; + scan_builder.SetPredicate(value_predicate); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + TableScan::Create(std::move(scan_context), scan_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, table_scan->CreatePlan()); int32_t indexed_split_count = 0; int32_t data_split_count = 0; @@ -472,16 +478,17 @@ TEST_P(BucketedPrimaryKeyIndexMaintainerTest, BuildsRestoresAndReplacesCompacted ASSERT_GT(indexed_split_count, 0); ASSERT_EQ(0, data_split_count); + std::map read_builder_options; ReadContextBuilder read_builder(table_path_); // The source build above keeps lazy decoding enabled. Disable it only for the final data-row // assertion, whose predicate reader is outside the maintenance path covered by this test. + read_builder_options["orc.read.enable-lazy-decoding"] = "false"; read_builder.SetReadFieldNames({"id", "value"}) .SetPredicate(value_predicate) - .AddOption("orc.read.enable-lazy-decoding", "false") .EnablePredicateFilter(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(std::move(read_context), read_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, @@ -686,11 +693,13 @@ TEST_P(BucketedPrimaryKeyIndexMaintainerTest, ContinuesCompactionWhenIndexBuildF std::shared_ptr value_predicate = PredicateBuilder::Equal( /*field_index=*/1, "value", FieldType::STRING, Literal(FieldType::STRING, indexed_value.data(), indexed_value.size())); + std::map scan_builder_options; ScanContextBuilder scan_builder(table_path_); - scan_builder.SetPredicate(value_predicate).AddOption(Options::GLOBAL_INDEX_ENABLED, "true"); + scan_builder_options[Options::GLOBAL_INDEX_ENABLED] = "true"; + scan_builder.SetPredicate(value_predicate); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + TableScan::Create(std::move(scan_context), scan_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, table_scan->CreatePlan()); int32_t indexed_split_count = 0; int32_t data_split_count = 0; @@ -704,14 +713,15 @@ TEST_P(BucketedPrimaryKeyIndexMaintainerTest, ContinuesCompactionWhenIndexBuildF ASSERT_EQ(0, indexed_split_count); ASSERT_GT(data_split_count, 0); + std::map read_builder_options; ReadContextBuilder read_builder(table_path_); + read_builder_options["orc.read.enable-lazy-decoding"] = "false"; read_builder.SetReadFieldNames({"id", "value"}) .SetPredicate(value_predicate) - .AddOption("orc.read.enable-lazy-decoding", "false") .EnablePredicateFilter(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(std::move(read_context), read_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, @@ -763,13 +773,13 @@ TEST_P(BucketedPrimaryKeyIndexMaintainerTest, DeletesPayloadAfterLastDefinitionI PathUtil::JoinPath(table_path_, "schema/schema-1"), schema_json, /*overwrite=*/false)); + std::map builder_options; WriteContextBuilder builder(table_path_, kCommitUser); - builder.SetOptions(evolved_options) - .WithStreamingMode(true) - .WithTempDirectory(directory_->Str()); + builder_options = evolved_options; + builder.WithStreamingMode(true).WithTempDirectory(directory_->Str()); ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr writer, - FileStoreWrite::Create(std::move(context))); + FileStoreWrite::Create(std::move(context), builder_options)); ASSERT_OK(writer->Compact(/*partition=*/{}, /*bucket=*/0, /*full_compaction=*/true)); ASSERT_OK_AND_ASSIGN(std::vector> cleanup_messages, writer->PrepareCommit(/*wait_compaction=*/true, diff --git a/src/paimon/core/index/pksorted/pk_sorted_data_file_reader.cpp b/src/paimon/core/index/pksorted/pk_sorted_data_file_reader.cpp index 9c3ff443e..b66bafcb8 100644 --- a/src/paimon/core/index/pksorted/pk_sorted_data_file_reader.cpp +++ b/src/paimon/core/index/pksorted/pk_sorted_data_file_reader.cpp @@ -48,18 +48,20 @@ Result> PkSortedDataFileReader::Create( const std::shared_ptr& executor, const std::shared_ptr& pool) { std::map read_options = options.ToMap(); read_options[Options::BRANCH] = branch; + std::map builder_options; ReadContextBuilder builder(root_path); + builder_options = read_options; + builder_options[Options::BRANCH] = branch; builder.SetReadFieldIds({field_id}) - .SetOptions(read_options) - .WithBranch(branch) .WithFileSystem(options.GetFileSystem()) .WithExecutor(executor) .WithMemoryPool(pool) .EnablePrefetch(false) .EnablePredicateFilter(false); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr read_context, builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr shared_internal_context, - InternalReadContext::Create(read_context, table_schema, read_options)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr shared_internal_context, + InternalReadContext::Create(read_context, table_schema, read_options, builder_options)); return std::unique_ptr( new PkSortedDataFileReader(path_factory, shared_internal_context, pool, executor)); } diff --git a/src/paimon/core/mergetree/compact/full_changelog_merge_tree_compact_rewriter.cpp b/src/paimon/core/mergetree/compact/full_changelog_merge_tree_compact_rewriter.cpp index 9fc0744ef..9de13a39a 100644 --- a/src/paimon/core/mergetree/compact/full_changelog_merge_tree_compact_rewriter.cpp +++ b/src/paimon/core/mergetree/compact/full_changelog_merge_tree_compact_rewriter.cpp @@ -65,18 +65,19 @@ FullChangelogMergeTreeCompactRewriter::Create( std::shared_ptr write_schema = SpecialFields::CompleteSequenceAndValueKindField(data_schema); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(path_factory_cache->RootPath()); - read_context_builder.SetOptions(options.ToMap()) - .WithFileSystem(options.GetFileSystem()) + read_context_builder_options = options.ToMap(); + read_context_builder.WithFileSystem(options.GetFileSystem()) .EnablePrefetch(true) .SetPrefetchMaxParallelNum(1) .SetPrefetchBatchCount(3) .WithMemoryPool(pool); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr internal_context, - InternalReadContext::Create(read_context, table_schema, options.ToMap())); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr internal_context, + InternalReadContext::Create(read_context, table_schema, options.ToMap(), + read_context_builder_options)); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr path_factory, path_factory_cache->GetOrCreatePathFactory(options.GetFileFormat()->Identifier())); diff --git a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.cpp b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.cpp index 390f95d84..160aeb250 100644 --- a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.cpp +++ b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.cpp @@ -73,18 +73,19 @@ LookupMergeTreeCompactRewriter::Create( auto write_schema = SpecialFields::CompleteSequenceAndValueKindField(data_schema); // TODO(xinyu.lxy): set executor + std::map read_context_builder_options; ReadContextBuilder read_context_builder(path_factory_cache->RootPath()); - read_context_builder.SetOptions(options.ToMap()) - .WithFileSystem(options.GetFileSystem()) + read_context_builder_options = options.ToMap(); + read_context_builder.WithFileSystem(options.GetFileSystem()) .EnablePrefetch(true) .SetPrefetchMaxParallelNum(1) .SetPrefetchBatchCount(3) .WithMemoryPool(pool); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr internal_context, - InternalReadContext::Create(read_context, table_schema, options.ToMap())); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr internal_context, + InternalReadContext::Create(read_context, table_schema, options.ToMap(), + read_context_builder_options)); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr path_factory, path_factory_cache->GetOrCreatePathFactory(options.GetFileFormat()->Identifier())); diff --git a/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory_test.cpp b/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory_test.cpp index 223abafed..118e2d931 100644 --- a/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory_test.cpp +++ b/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory_test.cpp @@ -115,9 +115,10 @@ class MergeTreeCompactManagerFactoryWriteTest : public ::testing::Test { table_options, /*ignore_if_exists=*/false)); + std::map context_builder_options; WriteContextBuilder context_builder(PathUtil::JoinPath(dir->Str(), "foo.db/bar"), "test"); if (!context_options.empty()) { - context_builder.SetOptions(context_options); + context_builder_options = context_options; } if (with_io_manager) { context_builder.WithTempDirectory(dir->Str()); @@ -125,7 +126,7 @@ class MergeTreeCompactManagerFactoryWriteTest : public ::testing::Test { PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, context_builder.Finish()); - return FileStoreWrite::Create(std::move(write_context)); + return FileStoreWrite::Create(std::move(write_context), context_builder_options); } Result> CreateSingleStringFileStoreWrite( diff --git a/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.cpp b/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.cpp index afe50ddfb..bc52d0505 100644 --- a/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.cpp +++ b/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.cpp @@ -73,18 +73,19 @@ Result> MergeTreeCompactRewriter::Crea auto write_schema = SpecialFields::CompleteSequenceAndValueKindField(data_schema); // TODO(xinyu.lxy): set executor + std::map read_context_builder_options; ReadContextBuilder read_context_builder(path_factory_cache->RootPath()); - read_context_builder.SetOptions(options.ToMap()) - .WithFileSystem(options.GetFileSystem()) + read_context_builder_options = options.ToMap(); + read_context_builder.WithFileSystem(options.GetFileSystem()) .EnablePrefetch(true) .SetPrefetchMaxParallelNum(1) .SetPrefetchBatchCount(3) .WithMemoryPool(pool); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr internal_context, - InternalReadContext::Create(read_context, table_schema, options.ToMap())); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr internal_context, + InternalReadContext::Create(read_context, table_schema, options.ToMap(), + read_context_builder_options)); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr path_factory, path_factory_cache->GetOrCreatePathFactory(options.GetFileFormat()->Identifier())); diff --git a/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter_test.cpp b/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter_test.cpp index 27424a1ae..ea7ca2a5f 100644 --- a/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter_test.cpp +++ b/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter_test.cpp @@ -60,7 +60,7 @@ class MergeTreeCompactRewriterTest : public testing::Test { ScanContextBuilder scan_context_builder(table_path); scan_context_builder.SetBucketFilter(bucket).SetPartitionFilter({partition}); PAIMON_ASSIGN_OR_RAISE(auto scan_context, scan_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context), {})); PAIMON_ASSIGN_OR_RAISE(auto result_plan, table_scan->CreatePlan()); auto splits = result_plan->Splits(); EXPECT_EQ(1, splits.size()); diff --git a/src/paimon/core/mergetree/lookup_levels.cpp b/src/paimon/core/mergetree/lookup_levels.cpp index 7d7aecedf..88b2e4a5c 100644 --- a/src/paimon/core/mergetree/lookup_levels.cpp +++ b/src/paimon/core/mergetree/lookup_levels.cpp @@ -57,18 +57,19 @@ Result>> LookupLevels::Create( auto partition_schema = DataField::ConvertDataFieldsToArrowSchema(partition_fields); // TODO(xinyu.lxy): set executor + std::map read_context_builder_options; ReadContextBuilder read_context_builder(path_factory->RootPath()); - read_context_builder.SetOptions(options.ToMap()) - .WithFileSystem(options.GetFileSystem()) + read_context_builder_options = options.ToMap(); + read_context_builder.WithFileSystem(options.GetFileSystem()) .EnablePrefetch(true) .SetPrefetchMaxParallelNum(1) .SetPrefetchBatchCount(3) .WithMemoryPool(pool); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr internal_read_context, - InternalReadContext::Create(read_context, table_schema, options.ToMap())); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr internal_read_context, + InternalReadContext::Create(read_context, table_schema, options.ToMap(), + read_context_builder_options)); auto split_read = std::make_unique(path_factory, internal_read_context, pool, CreateDefaultExecutor()); diff --git a/src/paimon/core/operation/append_only_file_store_scan_test.cpp b/src/paimon/core/operation/append_only_file_store_scan_test.cpp index b8dfbfced..82ac8800b 100644 --- a/src/paimon/core/operation/append_only_file_store_scan_test.cpp +++ b/src/paimon/core/operation/append_only_file_store_scan_test.cpp @@ -598,13 +598,16 @@ TEST(AppendOnlyFileStoreScanTest, TestReconstructPredicateWithNonCastedFields) { TEST(AppendOnlyFileStoreScanTest, TestReadPartitionEntries) { TimezoneGuard guard("Asia/Shanghai"); std::string table_path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09/"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::FILE_FORMAT, "orc") - .AddOption(Options::SCAN_SNAPSHOT_ID, "5"); + context_builder_options[Options::FILE_FORMAT] = "orc"; + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "5"; + ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); auto pool = GetDefaultPool(); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); auto typed_table_scan = dynamic_cast(table_scan.get()); ASSERT_TRUE(typed_table_scan); @@ -644,12 +647,15 @@ TEST(AppendOnlyFileStoreScanTest, TestReadPartitionEntries) { TEST(AppendOnlyFileStoreScanTest, TestScanDurationMetric) { TimezoneGuard guard("Asia/Shanghai"); std::string table_path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09/"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::FILE_FORMAT, "orc") - .AddOption(Options::SCAN_SNAPSHOT_ID, "5"); + context_builder_options[Options::FILE_FORMAT] = "orc"; + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "5"; + ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); auto typed_table_scan = dynamic_cast(table_scan.get()); ASSERT_TRUE(typed_table_scan); @@ -745,13 +751,14 @@ std::shared_ptr BuildScan(const std::string& table_path, const std::shared_ptr& predicate = nullptr, bool manifest_entry_lazy_decode_enabled = true, int32_t max_snapshots = 8) { + std::map context_builder_options; ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::FILE_FORMAT, "orc") - .AddOption(Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, - fmt::format("{}", max_snapshots)) - .AddOption(Options::SCAN_MANIFEST_ENTRY_LAZY_DECODE_ENABLED, - manifest_entry_lazy_decode_enabled ? "true" : "false") - .WithCache(cache); + context_builder_options[Options::FILE_FORMAT] = "orc"; + context_builder_options[Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS] = + fmt::format("{}", max_snapshots); + context_builder_options[Options::SCAN_MANIFEST_ENTRY_LAZY_DECODE_ENABLED] = + manifest_entry_lazy_decode_enabled ? "true" : "false"; + context_builder.WithCache(cache); if (bucket) { context_builder.SetBucketFilter(bucket.value()); } @@ -759,7 +766,8 @@ std::shared_ptr BuildScan(const std::string& table_path, context_builder.SetPredicate(predicate); } EXPECT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); - EXPECT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + EXPECT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); auto typed_table_scan = dynamic_cast(table_scan.get()); EXPECT_TRUE(typed_table_scan); return typed_table_scan->snapshot_reader_->scan_; diff --git a/src/paimon/core/operation/append_only_file_store_write.cpp b/src/paimon/core/operation/append_only_file_store_write.cpp index 15b89ba90..86f2d8c50 100644 --- a/src/paimon/core/operation/append_only_file_store_write.cpp +++ b/src/paimon/core/operation/append_only_file_store_write.cpp @@ -369,9 +369,10 @@ Result> AppendOnlyFileStoreWrite::CreateFilesReader const BinaryRow& partition, int32_t bucket, DeletionVector::Factory dv_factory, const std::vector>& files, const std::optional& veto_reason) const { + std::map context_builder_options; ReadContextBuilder context_builder(root_path_); - context_builder.SetOptions(options_.ToMap()) - .WithFileSystem(options_.GetFileSystem()) + context_builder_options = options_.ToMap(); + context_builder.WithFileSystem(options_.GetFileSystem()) .EnablePrefetch(true) .SetPrefetchMaxParallelNum(1) .SetPrefetchBatchCount(3) @@ -395,8 +396,9 @@ Result> AppendOnlyFileStoreWrite::CreateFilesReader } options[kParquetReadEnableDictionaryPassthrough] = "false"; } - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr internal_read_context, - InternalReadContext::Create(read_context, table_schema_, options)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr internal_read_context, + InternalReadContext::Create(read_context, table_schema_, options, context_builder_options)); auto read = std::make_unique(file_store_path_factory_, internal_read_context, pool_, compact_executor_); diff --git a/src/paimon/core/operation/append_only_file_store_write_test.cpp b/src/paimon/core/operation/append_only_file_store_write_test.cpp index bfeaefced..0d209ae4a 100644 --- a/src/paimon/core/operation/append_only_file_store_write_test.cpp +++ b/src/paimon/core/operation/append_only_file_store_write_test.cpp @@ -111,11 +111,13 @@ class AppendOnlyFileStoreWriteTest : public testing::Test { const std::string& table_path, const std::shared_ptr& schema, const std::map& options, const std::string& json, int64_t commit_identifier) const { + std::map builder_options; WriteContextBuilder builder(table_path, commit_user_); - builder.SetOptions(options); + builder_options = options; + EXPECT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); EXPECT_OK_AND_ASSIGN(auto file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(std::move(write_context), builder_options)); EXPECT_OK(file_store_write->Write(MakeBatch(schema, json))); EXPECT_OK_AND_ASSIGN(auto commit_msgs, file_store_write->PrepareCommit( /*wait_compaction=*/false, commit_identifier)); @@ -128,11 +130,13 @@ class AppendOnlyFileStoreWriteTest : public testing::Test { const std::map& options, const std::vector& write_schema, const std::string& json, int64_t commit_identifier) const { + std::map builder_options; WriteContextBuilder builder(table_path, commit_user_); - builder.SetOptions(options).WithWriteSchema(write_schema); + builder_options = options; + builder.WithWriteSchema(write_schema); EXPECT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); EXPECT_OK_AND_ASSIGN(auto file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(std::move(write_context), builder_options)); EXPECT_OK(file_store_write->Write(MakeBatch(schema, json))); EXPECT_OK_AND_ASSIGN(auto commit_msgs, file_store_write->PrepareCommit( /*wait_compaction=*/false, commit_identifier)); @@ -142,11 +146,13 @@ class AppendOnlyFileStoreWriteTest : public testing::Test { void Commit(const std::string& table_path, const std::map& options, const std::vector>& commit_msgs) const { + std::map builder_options; CommitContextBuilder builder(table_path, commit_user_); - builder.SetOptions(options); + builder_options = options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, builder.Finish()); ASSERT_OK_AND_ASSIGN(auto file_store_commit, - FileStoreCommit::Create(std::move(commit_context))); + FileStoreCommit::Create(std::move(commit_context), builder_options)); ASSERT_OK(file_store_commit->Commit(commit_msgs)); } @@ -228,7 +234,7 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestWriteWithInvalidBatch) { WriteContextBuilder builder(PathUtil::JoinPath(dir->Str(), "foo.db/bar"), commit_user_); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); ASSERT_OK_AND_ASSIGN(auto file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(std::move(write_context), {})); ASSERT_NOK_WITH_MSG(file_store_write->PrepareCommitWithProgress(/*commit_identifier=*/0), "PrepareCommitWithProgress is only supported by a real-time writer"); ASSERT_NOK_WITH_MSG(file_store_write->Write(nullptr), "batch is null pointer"); @@ -249,7 +255,7 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestWriteWithInvalidBatch) { commit_user_); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(std::move(write_context), {})); auto array = std::make_shared(); arrow::StringBuilder builder; for (size_t j = 0; j < 100; j++) { @@ -290,11 +296,14 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestRealtimeWriteTracksExternalOffsetRange) { ASSERT_OK_AND_ASSIGN(std::shared_ptr batch_realtime_context, RealtimeContext::Create()); + std::map batch_builder_options; WriteContextBuilder batch_builder(table_path, commit_user_); - batch_builder.SetOptions(options).WithRealtimeContext(batch_realtime_context); + batch_builder_options = options; + batch_builder.WithRealtimeContext(batch_realtime_context); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_context, batch_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_writer, - FileStoreWrite::Create(std::move(batch_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr batch_writer, + FileStoreWrite::Create(std::move(batch_context), batch_builder_options)); ASSERT_NOK_WITH_MSG(batch_writer->PrepareCommitWithProgress(/*commit_identifier=*/0), "PrepareCommitWithProgress requires streaming mode"); ASSERT_OK(batch_writer->Close()); @@ -302,10 +311,13 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestRealtimeWriteTracksExternalOffsetRange) ASSERT_OK_AND_ASSIGN(std::shared_ptr realtime_context, RealtimeContext::Create()); + std::map builder_options; WriteContextBuilder builder(table_path, commit_user_); - builder.SetOptions(options).WithStreamingMode(true).WithRealtimeContext(realtime_context); + builder_options = options; + builder.WithStreamingMode(true).WithRealtimeContext(realtime_context); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN(auto file_store_write, + FileStoreWrite::Create(std::move(write_context), builder_options)); ASSERT_OK(file_store_write->Write(MakeBatch(realtime_schema, R"([ [10, 1, "a"], @@ -378,14 +390,18 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestRealtimeWriteTracksExternalOffsetRange) } TEST_F(AppendOnlyFileStoreWriteTest, TestGetMaxSequenceNumberFromMultiPartition) { + std::map builder_options; WriteContextBuilder builder( paimon::test::GetDataDir() + "/orc/multi_partition_append_table.db/multi_partition_append_table/", commit_user_); - ASSERT_OK_AND_ASSIGN( - std::unique_ptr write_context, - builder.AddOption("file.format", "orc").AddOption("manifest.format", "orc").Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + builder_options["file.format"] = "orc"; + builder_options["manifest.format"] = "orc"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, + + builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto file_store_write, + FileStoreWrite::Create(std::move(write_context), builder_options)); auto write = dynamic_cast(file_store_write.get()); auto pool = GetDefaultPool(); { @@ -445,10 +461,13 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestSharedShreddingMapAdaptsAcrossRollingFi std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); + std::map builder_options; WriteContextBuilder builder(table_path, commit_user_); - builder.SetOptions(options); + builder_options = options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN(auto file_store_write, + FileStoreWrite::Create(std::move(write_context), builder_options)); ASSERT_OK(file_store_write->Write(MakeBatch(logical_schema, R"([ [1, [["a", 1], ["b", 2]]] ])"))); @@ -553,10 +572,13 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestSharedShreddingMultipleMapColumnsAdaptA CreateTable(dir->Str(), logical_schema, options); std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); + std::map builder_options; WriteContextBuilder builder(table_path, commit_user_); - builder.SetOptions(options); + builder_options = options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN(auto file_store_write, + FileStoreWrite::Create(std::move(write_context), builder_options)); ASSERT_OK(file_store_write->Write(MakeBatch(logical_schema, R"([ [1, [["a", 1], ["b", 2]], [["c", 3], ["d", 4], ["e", 5], ["f", 6]]] ])"))); diff --git a/src/paimon/core/operation/commit_context.cpp b/src/paimon/core/operation/commit_context.cpp index 3bbbab5d9..459a95a4d 100644 --- a/src/paimon/core/operation/commit_context.cpp +++ b/src/paimon/core/operation/commit_context.cpp @@ -41,37 +41,24 @@ CommitContext::CommitContext(const std::string& root_path, const std::string& co bool append_commit_check_conflict, const std::shared_ptr& memory_pool, const std::shared_ptr& executor, - const std::shared_ptr& specific_file_system, - const std::map& options) + const std::shared_ptr& specific_file_system) : CommitContext(root_path, commit_user, ignore_empty_commit, use_rest_catalog_commit, /*catalog=*/nullptr, /*identifier=*/std::nullopt, /*table_id=*/std::nullopt, append_commit_check_conflict, memory_pool, executor, specific_file_system, - options, - // Without an identifier the option is the only name a branch can have. - BranchManager::NormalizeBranch( - options.count(Options::BRANCH) ? options.at(Options::BRANCH) : ""), - // The option names the branch, and nothing else can when there is no - // identifier. - options.count(Options::BRANCH) - ? std::optional(BranchManager::NormalizeBranch( - options.at(Options::BRANCH))) - : std::nullopt, /*table_schema=*/std::nullopt, /*fs_scheme_to_identifier_map=*/{}) {} -CommitContext::CommitContext( - const std::string& root_path, const std::string& commit_user, bool ignore_empty_commit, - bool use_rest_catalog_commit, const std::shared_ptr& catalog, - const std::optional& identifier, const std::optional& table_id, - bool append_commit_check_conflict, const std::shared_ptr& memory_pool, - const std::shared_ptr& executor, - const std::shared_ptr& specific_file_system, - const std::map& options, const std::string& branch, - const std::optional& named_branch, - const std::optional& table_schema, - const std::map& fs_scheme_to_identifier_map) +CommitContext::CommitContext(const std::string& root_path, const std::string& commit_user, + bool ignore_empty_commit, bool use_rest_catalog_commit, + const std::shared_ptr& catalog, + const std::optional& identifier, + const std::optional& table_id, + bool append_commit_check_conflict, + const std::shared_ptr& memory_pool, + const std::shared_ptr& executor, + const std::shared_ptr& specific_file_system, + const std::optional& table_schema, + const std::map& fs_scheme_to_identifier_map) : root_path_(root_path), - branch_(branch), - named_branch_(named_branch), commit_user_(commit_user), ignore_empty_commit_(ignore_empty_commit), use_rest_catalog_commit_(use_rest_catalog_commit), @@ -83,7 +70,6 @@ CommitContext::CommitContext( executor_(executor), specific_file_system_(specific_file_system), fs_scheme_to_identifier_map_(fs_scheme_to_identifier_map), - options_(options), table_schema_(table_schema) {} CommitContext::~CommitContext() = default; @@ -104,7 +90,6 @@ class CommitContextBuilder::Impl { specific_file_system_.reset(); fs_scheme_to_identifier_map_.clear(); table_schema_ = std::nullopt; - options_.clear(); } private: @@ -121,7 +106,6 @@ class CommitContextBuilder::Impl { std::shared_ptr specific_file_system_; std::map fs_scheme_to_identifier_map_; std::optional table_schema_; - std::map options_; }; CommitContextBuilder::CommitContextBuilder(const std::string& root_path, @@ -133,18 +117,6 @@ CommitContextBuilder::CommitContextBuilder(const std::string& root_path, CommitContextBuilder::~CommitContextBuilder() = default; -CommitContextBuilder& CommitContextBuilder::AddOption(const std::string& key, - const std::string& value) { - impl_->options_[key] = value; - return *this; -} - -CommitContextBuilder& CommitContextBuilder::SetOptions( - const std::map& opts) { - impl_->options_ = opts; - return *this; -} - CommitContextBuilder& CommitContextBuilder::IgnoreEmptyCommit(bool ignore_empty_commit) { impl_->ignore_empty_commit_ = ignore_empty_commit; return *this; @@ -216,31 +188,6 @@ Result> CommitContextBuilder::Finish() { PAIMON_RETURN_NOT_OK( CatalogUtils::CheckNotSystemTable(impl_->identifier_.value(), "commit")); } - std::optional identifier_branch; - if (impl_->identifier_) { - PAIMON_ASSIGN_OR_RAISE(identifier_branch, impl_->identifier_->GetBranchName()); - } - PAIMON_ASSIGN_OR_RAISE(std::string branch, BranchManager::ResolveBranch( - impl_->identifier_, impl_->options_, - /*explicit_branch=*/std::nullopt, "commit")); - if (!BranchManager::IsMainBranch(branch)) { - // A commit which goes to a catalog, or is handed back as a request for one, names its - // branch by an object name, so the branch has to be one that reads back as itself. - // Refused here rather than at the commit, which writes metadata before publishing it. - if (impl_->catalog_ != nullptr || impl_->use_rest_catalog_commit_) { - PAIMON_RETURN_NOT_OK(BranchManager::CheckCatalogAddressableBranch(branch)); - } - if (impl_->catalog_ != nullptr) { - if (!identifier_branch) { - PAIMON_ASSIGN_OR_RAISE(std::string table_name, - impl_->identifier_.value().GetDataTableName()); - return Status::Invalid(fmt::format( - "a commit through a catalog addresses a branch by the table identifier, so " - "name branch '{}' there as '{}$branch_{}' rather than only in the '{}' option", - branch, table_name, branch, Options::BRANCH)); - } - } - } if (impl_->catalog_ != nullptr && impl_->use_rest_catalog_commit_) { return Status::Invalid( "a commit either goes to the catalog of WithCatalog() or is handed back as a request " @@ -248,9 +195,7 @@ Result> CommitContextBuilder::Finish() { } if (impl_->catalog_ != nullptr && impl_->specific_file_system_ == nullptr && impl_->fs_scheme_to_identifier_map_.empty()) { - // A catalog issuing per-table temporary credentials only hands them out through - // GetTableFileSystem, so the commit uses the credentials of this table. A scheme map - // routes by scheme instead, so the catalog is not asked when one is set. + // A scheme map takes precedence over catalog-issued per-table credentials. PAIMON_ASSIGN_OR_RAISE(impl_->specific_file_system_, impl_->catalog_->GetTableFileSystem(impl_->identifier_.value())); } @@ -258,18 +203,11 @@ Result> CommitContextBuilder::Finish() { if (impl_->root_path_.empty()) { return Status::Invalid("root path is empty"); } - // The branch the naming settings named, kept beside the resolved one so that a table can - // tell a context naming no branch - which it serves on its own - from one naming another. - std::optional named_branch; - if (identifier_branch || impl_->options_.find(Options::BRANCH) != impl_->options_.end()) { - named_branch = branch; - } auto ctx = std::make_unique( impl_->root_path_, impl_->commit_user_, impl_->ignore_empty_commit_, impl_->use_rest_catalog_commit_, impl_->catalog_, impl_->identifier_, impl_->table_id_, impl_->append_commit_check_conflict_, impl_->memory_pool_, impl_->executor_, - impl_->specific_file_system_, impl_->options_, branch, named_branch, - impl_->table_schema_, impl_->fs_scheme_to_identifier_map_); + impl_->specific_file_system_, impl_->table_schema_, impl_->fs_scheme_to_identifier_map_); impl_->Reset(); return ctx; } diff --git a/src/paimon/core/operation/commit_context_test.cpp b/src/paimon/core/operation/commit_context_test.cpp index bcbef8bcc..8970db696 100644 --- a/src/paimon/core/operation/commit_context_test.cpp +++ b/src/paimon/core/operation/commit_context_test.cpp @@ -38,12 +38,9 @@ TEST(CommitContextTest, TestLegacyConstructor) { auto pool = GetDefaultPool(); std::shared_ptr executor = CreateDefaultExecutor(); auto fs = std::make_shared(); - const std::map options = {{"key", "value"}}; - CommitContext context("table_root_path", "commit_user_1", false, true, true, pool, executor, fs, - options); + CommitContext context("table_root_path", "commit_user_1", false, true, true, pool, executor, + fs); ASSERT_EQ(context.GetRootPath(), "table_root_path"); - // Without an identifier the option is the only name a branch can have. - ASSERT_EQ(context.GetBranch(), "main"); ASSERT_EQ(context.GetCommitUser(), "commit_user_1"); ASSERT_FALSE(context.IgnoreEmptyCommit()); ASSERT_TRUE(context.UseRESTCatalogCommit()); @@ -51,7 +48,6 @@ TEST(CommitContextTest, TestLegacyConstructor) { ASSERT_EQ(context.GetMemoryPool(), pool); ASSERT_EQ(context.GetExecutor(), executor); ASSERT_EQ(context.GetSpecificFileSystem(), fs); - ASSERT_EQ(context.GetOptions(), options); ASSERT_EQ(context.GetCatalog(), nullptr); ASSERT_FALSE(context.GetIdentifier()); ASSERT_FALSE(context.GetTableId()); @@ -65,7 +61,6 @@ TEST(CommitContextTest, TestDefaultValue) { ASSERT_OK_AND_ASSIGN(auto expected_root_path, PathUtil::NormalizePath("table_root_path")); ASSERT_EQ(ctx->GetRootPath(), expected_root_path); - ASSERT_EQ(ctx->GetBranch(), "main"); ASSERT_EQ(ctx->GetCommitUser(), "commit_user_1"); ASSERT_TRUE(ctx->IgnoreEmptyCommit()); ASSERT_FALSE(ctx->UseRESTCatalogCommit()); @@ -76,7 +71,6 @@ TEST(CommitContextTest, TestDefaultValue) { ASSERT_TRUE(ctx->GetMemoryPool()); ASSERT_TRUE(ctx->GetExecutor()); ASSERT_FALSE(ctx->GetSpecificFileSystem()); - ASSERT_TRUE(ctx->GetOptions().empty()); ASSERT_TRUE(ctx->GetFileSystemSchemeToIdentifierMap().empty()); ASSERT_FALSE(ctx->GetSpecificTableSchema()); ASSERT_FALSE(ctx->GetTableId()); @@ -116,7 +110,6 @@ TEST(CommitContextTest, TestSetContent) { .WithExecutor(executor) .WithFileSystem(fs) .SetTableSchema("table-schema-json") - .AddOption("key", "value") .Finish()); ASSERT_OK_AND_ASSIGN(auto expected_root_path, PathUtil::NormalizePath("table_root_path")); @@ -133,7 +126,6 @@ TEST(CommitContextTest, TestSetContent) { ASSERT_EQ("table-schema-json", ctx->GetSpecificTableSchema().value()); std::map expected_options = {{"key", "value"}}; - ASSERT_EQ(ctx->GetOptions(), expected_options); } TEST(CommitContextTest, TestWithCatalog) { @@ -163,106 +155,4 @@ TEST(CommitContextTest, TestWithCatalog) { ASSERT_EQ(next_ctx->GetTableId(), std::nullopt); } -TEST(CommitContextTest, TestCatalogAddressesBranchByIdentifier) { - auto catalog = std::make_shared(); - - CommitContextBuilder branch_builder("table_root_path", "commit_user_1"); - ASSERT_OK_AND_ASSIGN( - auto branch_ctx, - branch_builder.WithCatalog(catalog, Identifier("db1", "t1$branch_dev")).Finish()); - ASSERT_EQ(branch_ctx->GetIdentifier().value(), Identifier("db1", "t1$branch_dev")); - // Finish() stores the resolved branch, so the factory reads it instead of resolving again. - ASSERT_EQ(branch_ctx->GetBranch(), "dev"); - CommitContextBuilder agreeing_builder("table_root_path", "commit_user_1"); - ASSERT_OK(agreeing_builder.WithCatalog(catalog, Identifier("db1", "t1$branch_dev")) - .AddOption(Options::BRANCH, "dev") - .Finish()); - CommitContextBuilder constructed_builder("table_root_path", "commit_user_1"); - ASSERT_OK_AND_ASSIGN( - auto constructed_ctx, - constructed_builder.WithCatalog(catalog, Identifier("db1", "t1", "dev")).Finish()); - ASSERT_EQ(constructed_ctx->GetIdentifier().value(), Identifier("db1", "t1$branch_dev")); - ASSERT_EQ(constructed_ctx->GetBranch(), "dev"); - CommitContextBuilder main_builder("table_root_path", "commit_user_1"); - ASSERT_OK(main_builder.WithCatalog(catalog, Identifier("db1", "t1$branch_main")) - .AddOption(Options::BRANCH, "main") - .Finish()); - - CommitContextBuilder mixed_builder("table_root_path", "commit_user_1"); - ASSERT_NOK_WITH_MSG(mixed_builder.WithCatalog(catalog, Identifier("db1", "t1$branch_dev")) - .AddOption(Options::BRANCH, "main") - .Finish(), - "but both 'dev' and 'main' were named"); - CommitContextBuilder mixed_option_builder("table_root_path", "commit_user_1"); - ASSERT_NOK_WITH_MSG( - mixed_option_builder.WithCatalog(catalog, Identifier("db1", "t1$branch_main")) - .AddOption(Options::BRANCH, "dev") - .Finish(), - "but both 'main' and 'dev' were named"); - - CommitContextBuilder option_only_builder("table_root_path", "commit_user_1"); - ASSERT_NOK_WITH_MSG(option_only_builder.WithCatalog(catalog, Identifier("db1", "t1")) - .AddOption(Options::BRANCH, "dev") - .Finish(), - "name branch 'dev' there as 't1$branch_dev'"); - - CommitContextBuilder upper_main_builder("table_root_path", "commit_user_1"); - ASSERT_NOK_WITH_MSG( - upper_main_builder.WithCatalog(catalog, Identifier("db1", "t1$branch_MAIN")).Finish(), - "a catalog names branch 'MAIN' as it names the main branch"); - - CommitContextBuilder splitter_builder("table_root_path", "commit_user_1"); - ASSERT_NOK_WITH_MSG(splitter_builder.WithCatalog(catalog, Identifier("db1", "t1")) - .AddOption(Options::BRANCH, "dev$options") - .Finish(), - "a branch a catalog addresses cannot contain '$'"); - - CommitContextBuilder system_table_builder("table_root_path", "commit_user_1"); - ASSERT_NOK_WITH_MSG( - system_table_builder.WithCatalog(catalog, Identifier("db1", "t1", "dev$options")).Finish(), - "Cannot 'commit' for system table"); -} - -TEST(CommitContextTest, TestBranch) { - CommitContextBuilder file_system_builder("table_root_path", "commit_user_1"); - ASSERT_OK_AND_ASSIGN(auto fs_branch_ctx, - file_system_builder.AddOption(Options::BRANCH, "dev").Finish()); - ASSERT_EQ(fs_branch_ctx->GetBranch(), "dev"); - - CommitContextBuilder upper_main_builder("table_root_path", "commit_user_1"); - ASSERT_OK_AND_ASSIGN(auto upper_main_ctx, - upper_main_builder.AddOption(Options::BRANCH, "MAIN").Finish()); - ASSERT_EQ(upper_main_ctx->GetOptions().at(Options::BRANCH), "MAIN"); - CommitContextBuilder splitter_builder("table_root_path", "commit_user_1"); - ASSERT_OK(splitter_builder.AddOption(Options::BRANCH, "dev$options").Finish()); - - CommitContextBuilder rest_builder("table_root_path", "commit_user_1"); - ASSERT_OK_AND_ASSIGN( - auto rest_ctx, - rest_builder.UseRESTCatalogCommit(true).AddOption(Options::BRANCH, "dev").Finish()); - ASSERT_TRUE(rest_ctx->UseRESTCatalogCommit()); - ASSERT_EQ(rest_ctx->GetBranch(), "dev"); - ASSERT_EQ(rest_ctx->GetOptions().at(Options::BRANCH), "dev"); - CommitContextBuilder rest_upper_main_builder("table_root_path", "commit_user_1"); - ASSERT_NOK_WITH_MSG(rest_upper_main_builder.UseRESTCatalogCommit(true) - .AddOption(Options::BRANCH, "MAIN") - .Finish(), - "a catalog names branch 'MAIN' as it names the main branch"); - - CommitContextBuilder escaping_builder("table_root_path", "commit_user_1"); - ASSERT_NOK_WITH_MSG(escaping_builder.AddOption(Options::BRANCH, "dev/../../outside").Finish(), - "branch name cannot contain path separators"); -} - -TEST(CommitContextTest, TestSetOptionsOverridesAddedOptions) { - CommitContextBuilder builder("table_root_path", "commit_user_1"); - builder.AddOption("old", "value"); - builder.SetOptions({{"key1", "value1"}, {"key2", "value2"}}); - - ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish()); - - std::map expected_options = {{"key1", "value1"}, {"key2", "value2"}}; - ASSERT_EQ(ctx->GetOptions(), expected_options); -} - } // namespace paimon::test diff --git a/src/paimon/core/operation/data_evolution_split_read_test.cpp b/src/paimon/core/operation/data_evolution_split_read_test.cpp index 4bd0e0c29..89f983c72 100644 --- a/src/paimon/core/operation/data_evolution_split_read_test.cpp +++ b/src/paimon/core/operation/data_evolution_split_read_test.cpp @@ -962,7 +962,7 @@ TEST_F(DataEvolutionSplitReadTest, TestSequentialVectorReadReusesMainReader) { builder.WithFileSystem(counting_fs).EnablePrefetch(false).SetReadFieldNames(projection); ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, builder.Finish()); ASSERT_OK_AND_ASSIGN(std::shared_ptr context, - InternalReadContext::Create(read_context, table_schema, options)); + InternalReadContext::Create(read_context, table_schema, options, {})); auto split_read = std::make_unique(path_factory, context, GetDefaultPool(), executor); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, split_read->CreateReader(split)); diff --git a/src/paimon/core/operation/file_store_commit.cpp b/src/paimon/core/operation/file_store_commit.cpp index af818e868..c5bf07d9d 100644 --- a/src/paimon/core/operation/file_store_commit.cpp +++ b/src/paimon/core/operation/file_store_commit.cpp @@ -27,12 +27,10 @@ namespace paimon { Result> FileStoreCommit::Create( - std::unique_ptr ctx) { + std::unique_ptr ctx, const std::map& options) { PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "commit")); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, FileStoreTableFactory::Create(*ctx)); - // The load folded the context's options into the table, so the operator below serves them - // from there and the context carries none of its own. - ctx->ClearOptions(); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, + FileStoreTableFactory::Create(*ctx, options)); return table->NewCommit(std::move(ctx)); } diff --git a/src/paimon/core/operation/file_store_commit_impl_test.cpp b/src/paimon/core/operation/file_store_commit_impl_test.cpp index 96c6de34d..1f8d94147 100644 --- a/src/paimon/core/operation/file_store_commit_impl_test.cpp +++ b/src/paimon/core/operation/file_store_commit_impl_test.cpp @@ -323,11 +323,14 @@ class FileStoreCommitImplTest : public testing::Test { } void CommitAsCompetitor(int32_t round, int64_t commit_identifier) { + std::map builder_options; CommitContextBuilder builder(table_path_, "competitor"); + builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr context, - builder.AddOption(Options::FILE_SYSTEM, "local").Finish()); + + builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr competitor, - FileStoreCommit::Create(std::move(context))); + FileStoreCommit::Create(std::move(context), builder_options)); ASSERT_OK(competitor->Commit(CommitMessagesOfRound(round), commit_identifier)); } @@ -343,19 +346,20 @@ class FileStoreCommitImplTest : public testing::Test { Result> CreateCatalogCommit( const CatalogCommitSpec& spec) const { + std::map builder_options; CommitContextBuilder builder(spec.root_path.empty() ? table_path_ : spec.root_path, spec.commit_user); - builder.AddOption(Options::FILE_SYSTEM, "local") - .WithCatalog(spec.catalog, spec.identifier.value()) + builder_options[Options::FILE_SYSTEM] = "local"; + builder.WithCatalog(spec.catalog, spec.identifier.value()) .AppendCommitCheckConflict(spec.append_commit_check_conflict); if (spec.table_id) { builder.WithTableId(spec.table_id.value()); } for (const auto& [key, value] : spec.options) { - builder.AddOption(key, value); + builder_options[key] = value; } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); - return FileStoreCommit::Create(std::move(context)); + return FileStoreCommit::Create(std::move(context), builder_options); } Result> CreateCatalogCommit( @@ -412,16 +416,18 @@ class FileStoreCommitImplTest : public testing::Test { void CheckExpireIsRefusedWithoutDeleting(const std::string& branch, const std::string& expected_msg) { + std::map builder_options; CommitContextBuilder builder(table_path_, "commit_user_1"); + builder_options[Options::FILE_SYSTEM] = "local"; + builder_options[Options::BRANCH] = branch; + builder_options[Options::SNAPSHOT_NUM_RETAINED_MIN] = "1"; + builder_options[Options::SNAPSHOT_NUM_RETAINED_MAX] = "1"; + builder_options[Options::SNAPSHOT_TIME_RETAINED] = "1ms"; ASSERT_OK_AND_ASSIGN(std::unique_ptr context, - builder.AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::BRANCH, branch) - .AddOption(Options::SNAPSHOT_NUM_RETAINED_MIN, "1") - .AddOption(Options::SNAPSHOT_NUM_RETAINED_MAX, "1") - .AddOption(Options::SNAPSHOT_TIME_RETAINED, "1ms") - .Finish()); + + builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(context))); + FileStoreCommit::Create(std::move(context), builder_options)); ASSERT_OK(PrepareFirstCommitMessageFiles()); ASSERT_OK(commit->Commit(CommitMessagesOfRound(1), /*commit_identifier=*/1)); ASSERT_OK(commit->DropPartition({{{"f1", "10"}}}, /*commit_identifier=*/2)); @@ -610,13 +616,16 @@ class FileStoreCommitImplTest : public testing::Test { }; TEST_F(FileStoreCommitImplTest, TestCommit) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); std::vector> msgs = GetCommitMessages(paimon::test::GetDataDir() + @@ -638,15 +647,17 @@ TEST_F(FileStoreCommitImplTest, TestCommit) { TEST_F(FileStoreCommitImplTest, TestRESTCatalogCommit) { TimezoneGuard guard("Asia/Shanghai"); + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .UseRESTCatalogCommit(true) - .WithTableId("table-uuid") - .Finish()); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN( + std::unique_ptr commit_context, - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.UseRESTCatalogCommit(true).WithTableId("table-uuid").Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); std::vector> msgs = CommitMessagesOfRound(1); ASSERT_NOK_WITH_MSG(commit->GetLastCommitTableRequest(), @@ -696,10 +707,14 @@ TEST_F(FileStoreCommitImplTest, TestRESTCatalogCommit) { TEST_F(FileStoreCommitImplTest, TestCommitCarriesBaseSnapshotUuid) { std::vector> msgs = CommitMessagesOfRound(1); + std::map base_builder_options; CommitContextBuilder base_builder(table_path_, "commit_user_1"); + base_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr base_context, - base_builder.AddOption(Options::FILE_SYSTEM, "local").Finish()); - ASSERT_OK_AND_ASSIGN(auto base_commit, FileStoreCommit::Create(std::move(base_context))); + + base_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto base_commit, + FileStoreCommit::Create(std::move(base_context), base_builder_options)); ASSERT_OK(base_commit->Commit(msgs, 1)); std::string snapshot_path = PathUtil::JoinPath(table_path_, "snapshot/snapshot-1"); @@ -707,13 +722,15 @@ TEST_F(FileStoreCommitImplTest, TestCommitCarriesBaseSnapshotUuid) { ASSERT_TRUE(base_snapshot.Uuid().has_value()); const std::string base_uuid = base_snapshot.Uuid().value(); + std::map rest_builder_options; CommitContextBuilder rest_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr rest_context, - rest_builder.AddOption(Options::FILE_SYSTEM, "local") - .UseRESTCatalogCommit(true) - .WithTableId("table-uuid") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto rest_commit, FileStoreCommit::Create(std::move(rest_context))); + rest_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN( + std::unique_ptr rest_context, + + rest_builder.UseRESTCatalogCommit(true).WithTableId("table-uuid").Finish()); + ASSERT_OK_AND_ASSIGN(auto rest_commit, + FileStoreCommit::Create(std::move(rest_context), rest_builder_options)); ASSERT_OK(rest_commit->Commit(msgs, 2)); ASSERT_OK_AND_ASSIGN(std::string request_str, rest_commit->GetLastCommitTableRequest()); @@ -773,25 +790,30 @@ TEST_F(FileStoreCommitImplTest, TestCatalogCommitTakesTheSnapshot) { } TEST_F(FileStoreCommitImplTest, TestCommitToBranch) { + std::map missing_builder_options; CommitContextBuilder missing_builder(table_path_, "commit_user_1"); + missing_builder_options[Options::FILE_SYSTEM] = "local"; + missing_builder_options[Options::BRANCH] = "dev"; ASSERT_OK_AND_ASSIGN(std::unique_ptr missing_context, - missing_builder.AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::BRANCH, "dev") - .Finish()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(missing_context)), - "not found latest schema in branch dev"); + + missing_builder.Finish()); + ASSERT_NOK_WITH_MSG( + FileStoreCommit::Create(std::move(missing_context), missing_builder_options), + "not found latest schema in branch dev"); SchemaManager branch_schema_manager(file_system_, table_path_, "dev"); ASSERT_OK(branch_schema_manager.CreateTable(arrow::schema(fields_), /*partition_keys=*/{"f1"}, /*primary_keys=*/{}, /*options=*/{})); + std::map builder_options; CommitContextBuilder builder(table_path_, "commit_user_1"); + builder_options[Options::FILE_SYSTEM] = "local"; + builder_options[Options::BRANCH] = "dev"; ASSERT_OK_AND_ASSIGN(std::unique_ptr context, - builder.AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::BRANCH, "dev") - .Finish()); + + builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(context))); + FileStoreCommit::Create(std::move(context), builder_options)); ASSERT_OK(commit->Commit(CommitMessagesOfRound(1), /*commit_identifier=*/1)); std::string branch_snapshots = PathUtil::JoinPath(table_path_, "branch/branch-dev/snapshot"); @@ -817,15 +839,15 @@ TEST_F(FileStoreCommitImplTest, TestRequestOnlyCommitToBranch) { ASSERT_OK(branch_schema_manager.CreateTable(arrow::schema(fields_), /*partition_keys=*/{"f1"}, /*primary_keys=*/{}, /*options=*/{})); + std::map builder_options; CommitContextBuilder builder(table_path_, "commit_user_1"); + builder_options[Options::FILE_SYSTEM] = "local"; + builder_options[Options::BRANCH] = "dev"; ASSERT_OK_AND_ASSIGN(std::unique_ptr context, - builder.AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::BRANCH, "dev") - .UseRESTCatalogCommit(true) - .WithTableId("table-uuid") - .Finish()); + + builder.UseRESTCatalogCommit(true).WithTableId("table-uuid").Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(context))); + FileStoreCommit::Create(std::move(context), builder_options)); ASSERT_OK(commit->Commit(CommitMessagesOfRound(1), /*commit_identifier=*/1)); ASSERT_OK_AND_ASSIGN(std::string request_str, commit->GetLastCommitTableRequest()); @@ -936,13 +958,15 @@ TEST_F(FileStoreCommitImplTest, TestCatalogCommitToBranchTakesTheBranchSnapshot) ASSERT_OK(branch_schema_manager.CreateTable(arrow::schema(fields_), /*partition_keys=*/{"f1"}, /*primary_keys=*/{}, /*options=*/{})); + std::map base_builder_options; CommitContextBuilder base_builder(table_path_, "commit_user_1"); + base_builder_options[Options::FILE_SYSTEM] = "local"; + base_builder_options[Options::BRANCH] = "dev"; ASSERT_OK_AND_ASSIGN(std::unique_ptr base_context, - base_builder.AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::BRANCH, "dev") - .Finish()); + + base_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr base_commit, - FileStoreCommit::Create(std::move(base_context))); + FileStoreCommit::Create(std::move(base_context), base_builder_options)); ASSERT_OK(base_commit->Commit(CommitMessagesOfRound(1), /*commit_identifier=*/1)); SnapshotManager branch_snapshot_manager(file_system_, table_path_, "dev"); ASSERT_OK_AND_ASSIGN(std::optional base_snapshot, @@ -1111,10 +1135,14 @@ TEST_F(FileStoreCommitImplTest, TestCatalogSchemaWinsOverTheOneUnderTheTablePath ASSERT_EQ(catalog->CommitCalls().size(), 1u); ASSERT_EQ(catalog->CommitCalls().front().snapshot.SchemaId(), 7); + std::map path_builder_options; CommitContextBuilder path_builder(table_path_, "commit_user_2"); + path_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr path_context, - path_builder.AddOption(Options::FILE_SYSTEM, "local").Finish()); - ASSERT_OK_AND_ASSIGN(auto path_commit, FileStoreCommit::Create(std::move(path_context))); + + path_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto path_commit, + FileStoreCommit::Create(std::move(path_context), path_builder_options)); std::vector> other_msgs = CommitMessagesOfRound(2); ASSERT_OK(path_commit->Commit(other_msgs, 2)); ASSERT_OK_AND_ASSIGN( @@ -1148,12 +1176,13 @@ TEST_F(FileStoreCommitImplTest, TestCatalogSchemaWhichIsNotADataTableSchemaIsRef } TEST_F(FileStoreCommitImplTest, TestTableIdWithNothingToSendItInIsRefused) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::FILE_SYSTEM, "local") - .WithTableId("table-uuid") - .Finish()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(commit_context)), + + context_builder.WithTableId("table-uuid").Finish()); + ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(commit_context), context_builder_options), "publishes the snapshot by writing it to the table directory"); auto catalog = CreateCatalog(); @@ -1325,14 +1354,15 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithoutAUsableLockIsNotCreated) { ASSERT_NOK_WITH_MSG(CreateCatalogCommit(catalog), "no commit lease for this table"); ASSERT_TRUE(catalog->CommitCalls().empty()); + std::map builder_options; CommitContextBuilder builder(table_path_, "commit_user_1"); + builder_options[Options::FILE_SYSTEM] = "local"; + builder_options[Options::REST_COMMIT_ACQUIRE_LOCK] = "true"; ASSERT_OK_AND_ASSIGN(std::unique_ptr context, - builder.AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::REST_COMMIT_ACQUIRE_LOCK, "true") - .UseRESTCatalogCommit(true) - .WithTableId("table-uuid") - .Finish()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(context)), "cannot hold a commit lease"); + + builder.UseRESTCatalogCommit(true).WithTableId("table-uuid").Finish()); + ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(context), builder_options), + "cannot hold a commit lease"); } TEST_F(FileStoreCommitImplTest, TestUnknownCatalogOutcomeKeepsTheManifests) { @@ -1631,7 +1661,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitReadsThroughTheCatalogsFileSystem) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.WithCatalog(catalog, Identifier("db", "tbl")).Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context), {})); ASSERT_OK(commit->Commit(CommitMessagesOfRound(1))); ASSERT_EQ(catalog->CommitCalls().size(), 1u); @@ -1669,7 +1699,7 @@ TEST_F(FileStoreCommitImplTest, TestAGivenFileSystemWinsOverTheCatalogs) { context_builder.WithCatalog(catalog, Identifier("db", "tbl")) .WithFileSystem(given_fs) .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context), {})); ASSERT_OK(commit->Commit(CommitMessagesOfRound(1))); ASSERT_EQ(catalog->CommitCalls().size(), 1u); @@ -1843,14 +1873,17 @@ TEST_F(FileStoreCommitImplTest, TestCatalogSnapshotLoaderFailurePropagates) { } TEST_F(FileStoreCommitImplTest, TestSnapshotSequenceMaxPropertyMergedOnCommit) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + context_builder_options[Options::WRITE_SEQUENCE_NUMBER_INIT_MODE] = "snapshot"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::WRITE_SEQUENCE_NUMBER_INIT_MODE, "snapshot") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -1890,16 +1923,18 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithConflictSnapshotAndRetryTenTimes) ASSERT_TRUE(TestUtil::CopyDirectory(test_data_path, table_path)); ASSERT_OK_AND_ASSIGN(std::shared_ptr fs, FileSystemFactory::Get("gmock_fs", table_path, {})); + std::map context_builder_options; CommitContextBuilder context_builder(table_path, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::COMMIT_MAX_RETRIES] = "10"; + context_builder_options[Options::COMMIT_MIN_RETRY_WAIT] = "1ms"; + context_builder_options[Options::COMMIT_MAX_RETRY_WAIT] = "1ms"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::COMMIT_MAX_RETRIES, "10") - .AddOption(Options::COMMIT_MIN_RETRY_WAIT, "1ms") - .AddOption(Options::COMMIT_MAX_RETRY_WAIT, "1ms") - .WithFileSystem(fs) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.WithFileSystem(fs).Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); std::string latest_hint = PathUtil::JoinPath(table_path, "snapshot/LATEST"); auto* mock_fs = dynamic_cast(fs.get()); @@ -1931,15 +1966,17 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithConflictSnapshotAndRetryOnce) { ASSERT_TRUE(TestUtil::CopyDirectory(test_data_path, table_path)); ASSERT_OK_AND_ASSIGN(std::shared_ptr fs, FileSystemFactory::Get("gmock_fs", table_path, {})); + std::map context_builder_options; CommitContextBuilder context_builder(table_path, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::COMMIT_MIN_RETRY_WAIT] = "1ms"; + context_builder_options[Options::COMMIT_MAX_RETRY_WAIT] = "1ms"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::COMMIT_MIN_RETRY_WAIT, "1ms") - .AddOption(Options::COMMIT_MAX_RETRY_WAIT, "1ms") - .WithFileSystem(fs) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.WithFileSystem(fs).Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); std::string latest_hint = PathUtil::JoinPath(table_path, "snapshot/LATEST"); auto* mock_fs = dynamic_cast(fs.get()); EXPECT_CALL(*mock_fs, ReadFile(testing::_, testing::_)) @@ -2003,17 +2040,19 @@ TEST_F(FileStoreCommitImplTest, TestRealtimeCommitRebasesAfterSnapshotConflict) ASSERT_OK_AND_ASSIGN(std::shared_ptr fs, FileSystemFactory::Get("gmock_fs", table_path_, {})); + std::map realtime_builder_options; CommitContextBuilder realtime_builder(table_path_, "realtime_commit_user"); + realtime_builder_options[Options::REALTIME_ENABLED] = "true"; + realtime_builder_options[Options::BUCKET] = "2"; + realtime_builder_options[Options::COMMIT_MAX_RETRIES] = "1"; + realtime_builder_options[Options::COMMIT_MIN_RETRY_WAIT] = "1ms"; + realtime_builder_options[Options::COMMIT_MAX_RETRY_WAIT] = "1ms"; ASSERT_OK_AND_ASSIGN(std::unique_ptr realtime_context, - realtime_builder.AddOption(Options::REALTIME_ENABLED, "true") - .AddOption(Options::BUCKET, "2") - .AddOption(Options::COMMIT_MAX_RETRIES, "1") - .AddOption(Options::COMMIT_MIN_RETRY_WAIT, "1ms") - .AddOption(Options::COMMIT_MAX_RETRY_WAIT, "1ms") - .WithFileSystem(fs) - .Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr realtime_commit, - FileStoreCommit::Create(std::move(realtime_context))); + + realtime_builder.WithFileSystem(fs).Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr realtime_commit, + FileStoreCommit::Create(std::move(realtime_context), realtime_builder_options)); auto realtime_commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(realtime_commit))); @@ -2034,7 +2073,7 @@ TEST_F(FileStoreCommitImplTest, TestRealtimeCommitRebasesAfterSnapshotConflict) ASSERT_OK_AND_ASSIGN(std::unique_ptr concurrent_context, concurrent_builder.WithFileSystem(file_system_).Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr concurrent_commit, - FileStoreCommit::Create(std::move(concurrent_context))); + FileStoreCommit::Create(std::move(concurrent_context), {})); auto* mock_fs = dynamic_cast(fs.get()); ASSERT_NE(nullptr, mock_fs); @@ -2078,13 +2117,15 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithAtomicWriteSnapshotTimeoutAndActua ASSERT_TRUE(TestUtil::CopyDirectory(test_data_path, table_path)); ASSERT_OK_AND_ASSIGN(std::shared_ptr fs, FileSystemFactory::Get("gmock_fs", table_path, {})); + std::map context_builder_options; CommitContextBuilder context_builder(table_path, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .WithFileSystem(fs) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.WithFileSystem(fs).Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); std::string new_snapshot_6 = PathUtil::JoinPath(table_path, "snapshot/snapshot-6"); auto* mock_fs = dynamic_cast(fs.get()); EXPECT_CALL(*mock_fs, AtomicStore(testing::StrEq(new_snapshot_6), testing::_)) @@ -2105,13 +2146,15 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithAtomicWriteSnapshotTimeoutAndActua bool exist, file_system_->Exists(PathUtil::JoinPath(table_path, "snapshot/snapshot-6"))); ASSERT_TRUE(exist); + std::map context_builder_2_options; CommitContextBuilder context_builder_2(table_path, "commit_user_1"); + context_builder_2_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context_2, - context_builder_2.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .WithFileSystem(fs) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit_2, FileStoreCommit::Create(std::move(commit_context_2))); + context_builder_2.WithFileSystem(fs).Finish()); + + ASSERT_OK_AND_ASSIGN(auto commit_2, FileStoreCommit::Create(std::move(commit_context_2), + context_builder_2_options)); ASSERT_OK_AND_ASSIGN(int32_t num_committed, commit_2->FilterAndCommit({{1, msgs}})); ASSERT_EQ(0, num_committed); std::string new_snapshot_7 = PathUtil::JoinPath(table_path, "snapshot/snapshot-7"); @@ -2129,14 +2172,17 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithAtomicWriteSnapshotTimeoutAndActua } TEST_F(FileStoreCommitImplTest, TestCommitWithSameMsgs) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "5kb"; + context_builder_options[Options::MANIFEST_MERGE_MIN_COUNT] = "2"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "5kb") - .AddOption(Options::MANIFEST_MERGE_MIN_COUNT, "2") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); { std::vector> msgs = @@ -2192,13 +2238,16 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithSameMsgs) { } TEST_F(FileStoreCommitImplTest, TestCommitMultipleTimes) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = dynamic_cast(commit.get()); { std::vector> msgs = @@ -2263,12 +2312,15 @@ TEST_F(FileStoreCommitImplTest, TestCommitMultipleTimes) { } TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatest) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + + context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = dynamic_cast(commit.get()); ASSERT_TRUE(commit_impl); @@ -2320,10 +2372,14 @@ TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatest) { } TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestThroughCatalog) { + std::map base_builder_options; CommitContextBuilder base_builder(table_path_, "commit_user_1"); + base_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr base_context, - base_builder.AddOption(Options::FILE_SYSTEM, "local").Finish()); - ASSERT_OK_AND_ASSIGN(auto base_commit, FileStoreCommit::Create(std::move(base_context))); + + base_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto base_commit, + FileStoreCommit::Create(std::move(base_context), base_builder_options)); for (int32_t round = 1; round <= 2; ++round) { ASSERT_OK(base_commit->Commit(CommitMessagesOfRound(round), round)); } @@ -2347,10 +2403,14 @@ TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestThroughCatalog) { } TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestHoldsALockScope) { + std::map base_builder_options; CommitContextBuilder base_builder(table_path_, "commit_user_1"); + base_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr base_context, - base_builder.AddOption(Options::FILE_SYSTEM, "local").Finish()); - ASSERT_OK_AND_ASSIGN(auto base_commit, FileStoreCommit::Create(std::move(base_context))); + + base_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto base_commit, + FileStoreCommit::Create(std::move(base_context), base_builder_options)); for (int32_t round = 1; round <= 2; ++round) { ASSERT_OK(base_commit->Commit(CommitMessagesOfRound(round), round)); } @@ -2379,10 +2439,14 @@ TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestHoldsALockScope) { } TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestNeedsTheTargetUnderTheTablePath) { + std::map base_builder_options; CommitContextBuilder base_builder(table_path_, "commit_user_1"); + base_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr base_context, - base_builder.AddOption(Options::FILE_SYSTEM, "local").Finish()); - ASSERT_OK_AND_ASSIGN(auto base_commit, FileStoreCommit::Create(std::move(base_context))); + + base_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto base_commit, + FileStoreCommit::Create(std::move(base_context), base_builder_options)); for (int32_t round = 1; round <= 2; ++round) { ASSERT_OK(base_commit->Commit(CommitMessagesOfRound(round), round)); } @@ -2396,23 +2460,29 @@ TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestNeedsTheTargetUnderTheTabl } TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestNoLatestSnapshotReturnsError) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + + context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); ASSERT_NOK_WITH_MSG(commit->RollbackToAsLatest(/*target_snapshot_id=*/1), "Latest snapshot is null, can not roll back."); } TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestTargetNotExistReturnsError) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + + context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); std::vector> msgs = GetCommitMessages(paimon::test::GetDataDir() + "/orc/append_09.db/append_09/commit_messages/commit_messages-01", @@ -2441,14 +2511,17 @@ TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestDeletionVectorOnlyChange) /*ignore_if_exists=*/false)); std::string dv_table_path = PathUtil::JoinPath(test_root_, "foo.db/dv_bar"); + std::map context_builder_options; CommitContextBuilder context_builder(dv_table_path, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + context_builder_options[Options::SNAPSHOT_NUM_RETAINED_MIN] = "1"; + context_builder_options[Options::SNAPSHOT_NUM_RETAINED_MAX] = "1"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::SNAPSHOT_NUM_RETAINED_MIN, "1") - .AddOption(Options::SNAPSHOT_NUM_RETAINED_MAX, "1") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + + context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = dynamic_cast(commit.get()); ASSERT_TRUE(commit_impl); @@ -2518,12 +2591,14 @@ TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestConcurrentConflictReturnsF // before writing. A stateful Exists() mock reproduces exactly that ordering. ASSERT_OK_AND_ASSIGN(std::shared_ptr fs, FileSystemFactory::Get("gmock_fs", table_path_, {})); + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .WithFileSystem(fs) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + + context_builder.WithFileSystem(fs).Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = dynamic_cast(commit.get()); ASSERT_TRUE(commit_impl); @@ -2560,12 +2635,15 @@ TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestConcurrentConflictReturnsF } TEST_F(FileStoreCommitImplTest, TestCommitAndOverwriteWithNoPartitionKey) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + + context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); { std::vector> msgs = GetCommitMessages(paimon::test::GetDataDir() + @@ -2605,12 +2683,15 @@ TEST_F(FileStoreCommitImplTest, TestCommitAndOverwriteWithNoPartitionKey) { } TEST_F(FileStoreCommitImplTest, TestCommitSuccessAfterIOException) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + + context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); std::vector> msgs = GetCommitMessages(paimon::test::GetDataDir() + @@ -2662,13 +2743,16 @@ TEST_F(FileStoreCommitImplTest, TestCommitSuccessAfterIOException) { } TEST_F(FileStoreCommitImplTest, TestCleanUpTmpManifests) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); { std::vector> msgs = @@ -2780,14 +2864,16 @@ TEST_F(FileStoreCommitImplTest, TestCleanUpTmpManifests) { } TEST_F(FileStoreCommitImplTest, TestCommitWithIgnoreEmptyCommit) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .IgnoreEmptyCommit(true) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.IgnoreEmptyCommit(true).Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); std::vector> msgs; ASSERT_OK(commit->Commit(msgs)); std::shared_ptr metrics = commit->GetCommitMetrics(); @@ -2798,14 +2884,16 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithIgnoreEmptyCommit) { } TEST_F(FileStoreCommitImplTest, TestTryOverwrite) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .IgnoreEmptyCommit(true) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.IgnoreEmptyCommit(true).Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); std::vector> msgs = GetCommitMessages(paimon::test::GetDataDir() + "/orc/append_09.db/append_09/commit_messages/" @@ -2828,14 +2916,16 @@ TEST_F(FileStoreCommitImplTest, TestTryOverwrite) { } TEST_F(FileStoreCommitImplTest, TestTryOverwriteFromNothing) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .IgnoreEmptyCommit(true) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.IgnoreEmptyCommit(true).Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = dynamic_cast(commit.get()); ASSERT_TRUE(commit_impl); std::vector changes; @@ -2862,14 +2952,16 @@ TEST_F(FileStoreCommitImplTest, TestTryOverwriteFromNothing) { } TEST_F(FileStoreCommitImplTest, TestTryOverwriteWithProperties) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .IgnoreEmptyCommit(true) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.IgnoreEmptyCommit(true).Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = dynamic_cast(commit.get()); ASSERT_TRUE(commit_impl); @@ -2889,14 +2981,16 @@ TEST_F(FileStoreCommitImplTest, TestTryOverwriteWithProperties) { } TEST_F(FileStoreCommitImplTest, TestTryOverwriteThenCommit) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .IgnoreEmptyCommit(true) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.IgnoreEmptyCommit(true).Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = dynamic_cast(commit.get()); ASSERT_TRUE(commit_impl); std::vector changes; @@ -3221,18 +3315,20 @@ TEST_F(FileStoreCommitImplTest, TestCatalogExpireWithoutPublishedSnapshots) { } TEST_F(FileStoreCommitImplTest, TestDropPartitionAndExpireSnapshot) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + context_builder_options[Options::SNAPSHOT_NUM_RETAINED_MIN] = "1"; + context_builder_options[Options::SNAPSHOT_NUM_RETAINED_MAX] = "1"; + context_builder_options[Options::SNAPSHOT_EXPIRE_LIMIT] = "30"; + context_builder_options[Options::SNAPSHOT_TIME_RETAINED] = "1ms"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::SNAPSHOT_NUM_RETAINED_MIN, "1") - .AddOption(Options::SNAPSHOT_NUM_RETAINED_MAX, "1") - .AddOption(Options::SNAPSHOT_EXPIRE_LIMIT, "30") - .AddOption(Options::SNAPSHOT_TIME_RETAINED, "1ms") - .IgnoreEmptyCommit(true) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.IgnoreEmptyCommit(true).Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); std::vector> msgs = GetCommitMessages(paimon::test::GetDataDir() + "/orc/append_09.db/append_09/commit_messages/" @@ -3293,18 +3389,20 @@ TEST_F(FileStoreCommitImplTest, TestExpireOnMainOfTableWithBranchesIsNotSupporte } TEST_F(FileStoreCommitImplTest, TestDropMultiPartitionAndExpireSnapshot) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + context_builder_options[Options::SNAPSHOT_NUM_RETAINED_MIN] = "1"; + context_builder_options[Options::SNAPSHOT_NUM_RETAINED_MAX] = "1"; + context_builder_options[Options::SNAPSHOT_EXPIRE_LIMIT] = "30"; + context_builder_options[Options::SNAPSHOT_TIME_RETAINED] = "1ms"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::SNAPSHOT_NUM_RETAINED_MIN, "1") - .AddOption(Options::SNAPSHOT_NUM_RETAINED_MAX, "1") - .AddOption(Options::SNAPSHOT_EXPIRE_LIMIT, "30") - .AddOption(Options::SNAPSHOT_TIME_RETAINED, "1ms") - .IgnoreEmptyCommit(true) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.IgnoreEmptyCommit(true).Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); std::vector> msgs = GetCommitMessages(paimon::test::GetDataDir() + "/orc/append_09.db/append_09/commit_messages/" @@ -3340,14 +3438,16 @@ TEST_F(FileStoreCommitImplTest, TestDropMultiPartitionAndExpireSnapshot) { } TEST_F(FileStoreCommitImplTest, TestTruncateTable) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .IgnoreEmptyCommit(true) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.IgnoreEmptyCommit(true).Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); std::vector> msgs = GetCommitMessages(paimon::test::GetDataDir() + "/orc/append_09.db/append_09/commit_messages/" @@ -3373,10 +3473,14 @@ TEST_F(FileStoreCommitImplTest, TestTruncateTable) { } TEST_F(FileStoreCommitImplTest, TestAbortDeletesDataAndIndexFiles) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::FILE_SYSTEM, "local").Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + + context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -3425,10 +3529,14 @@ TEST_F(FileStoreCommitImplTest, TestAbortDeletesDataAndIndexFiles) { } TEST_F(FileStoreCommitImplTest, AbortIgnoresMissingFilesAndFailsForNonImplMessage) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::FILE_SYSTEM, "local").Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + + context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -3449,10 +3557,14 @@ TEST_F(FileStoreCommitImplTest, AbortIgnoresDeleteFailures) { // A delete that fails with an IO error (e.g. permission denied) must be swallowed by // best-effort cleanup, mirroring Java deleteQuietly: Abort still returns OK and does not // propagate the failure. + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::FILE_SYSTEM, "local").Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + + context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = dynamic_cast(commit.get()); ASSERT_TRUE(commit_impl); @@ -3488,13 +3600,15 @@ TEST_F(FileStoreCommitImplTest, TestTruncateEmptyTable) { // Truncating a table that has never been committed succeeds and produces an OVERWRITE snapshot // with no added and no deleted files: there is nothing to overwrite, but the overwrite is still // materialized. + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .IgnoreEmptyCommit(true) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + + context_builder.IgnoreEmptyCommit(true).Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); ASSERT_OK(commit->TruncateTable(/*commit_identifier=*/1)); @@ -3513,14 +3627,16 @@ TEST_F(FileStoreCommitImplTest, TestTruncateEmptyTable) { } TEST_F(FileStoreCommitImplTest, TestCreateManifestCommittable) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .IgnoreEmptyCommit(true) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.IgnoreEmptyCommit(true).Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); std::vector> msgs; @@ -3533,14 +3649,17 @@ TEST_F(FileStoreCommitImplTest, TestCreateManifestCommittable) { } TEST_F(FileStoreCommitImplTest, TestCollectChanges) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + context_builder_options[Options::BUCKET] = "10"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::BUCKET, "10") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); std::vector> msgs = GetCommitMessages(paimon::test::GetDataDir() + "/orc/append_09.db/append_09/commit_messages/" @@ -3576,13 +3695,16 @@ TEST_F(FileStoreCommitImplTest, TestCollectChanges) { } TEST_F(FileStoreCommitImplTest, TestFilterCommitted) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -3607,13 +3729,16 @@ TEST_F(FileStoreCommitImplTest, TestFilterCommitted) { } TEST_F(FileStoreCommitImplTest, TestFilterCommittedWithMultipleCommittables) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -3647,13 +3772,16 @@ TEST_F(FileStoreCommitImplTest, TestFilterCommittedWithMultipleCommittables) { } TEST_F(FileStoreCommitImplTest, TestFilterCommittedRejectsDuplicateIdentifiers) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -3678,13 +3806,16 @@ TEST_F(FileStoreCommitImplTest, TestFilterCommittedRejectsDuplicateIdentifiers) } TEST_F(FileStoreCommitImplTest, FilterAndCommit) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); std::vector data_files = { @@ -3725,13 +3856,16 @@ TEST_F(FileStoreCommitImplTest, FilterAndCommit) { } TEST_F(FileStoreCommitImplTest, FilterAndCommitWithNotExistFile) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -3747,13 +3881,16 @@ TEST_F(FileStoreCommitImplTest, FilterAndCommitWithNotExistFile) { } TEST_F(FileStoreCommitImplTest, FilterAndCommitWithCompactedChangelogFakePath) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -3781,13 +3918,16 @@ TEST_F(FileStoreCommitImplTest, FilterAndCommitWithCompactedChangelogFakePath) { } TEST_F(FileStoreCommitImplTest, FilterAndCommitSkipCompactBeforeFileCheck) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -3814,13 +3954,16 @@ TEST_F(FileStoreCommitImplTest, FilterAndCommitSkipCompactBeforeFileCheck) { } TEST_F(FileStoreCommitImplTest, TestOverwriteNonSpecifyPartition) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -3853,13 +3996,16 @@ TEST_F(FileStoreCommitImplTest, TestOverwriteNonSpecifyPartition) { } TEST_F(FileStoreCommitImplTest, TestCommitWithIndexFiles) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); const BinaryRow partition = CreateIntRow(10); @@ -3884,15 +4030,18 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithIndexFiles) { } TEST_F(FileStoreCommitImplTest, TestCommitWithGlobalIndexFilesChecksConflicts) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + context_builder_options[Options::ROW_TRACKING_ENABLED] = "true"; + context_builder_options[Options::DATA_EVOLUTION_ENABLED] = "true"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::ROW_TRACKING_ENABLED, "true") - .AddOption(Options::DATA_EVOLUTION_ENABLED, "true") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); const BinaryRow partition = CreateIntRow(10); @@ -3916,14 +4065,16 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithGlobalIndexFilesChecksConflicts) { } TEST_F(FileStoreCommitImplTest, TestCommitWithCompactIndexFiles) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .IgnoreEmptyCommit(true) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.IgnoreEmptyCommit(true).Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); const BinaryRow partition = CreateIntRow(10); @@ -3949,13 +4100,16 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithCompactIndexFiles) { } TEST_F(FileStoreCommitImplTest, TestCommitWithDeletedIndexFiles) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); const BinaryRow partition = CreateIntRow(10); @@ -3987,14 +4141,16 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithDeletedIndexFiles) { } TEST_F(FileStoreCommitImplTest, TestCommitWithCompactDeletedIndexFiles) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .IgnoreEmptyCommit(true) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.IgnoreEmptyCommit(true).Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); const BinaryRow partition = CreateIntRow(10); @@ -4027,13 +4183,16 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithCompactDeletedIndexFiles) { } TEST_F(FileStoreCommitImplTest, TestOverwriteWithCompactIndexFiles) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); const BinaryRow partition = CreateIntRow(10); @@ -4061,13 +4220,16 @@ TEST_F(FileStoreCommitImplTest, TestOverwriteWithCompactIndexFiles) { } TEST_F(FileStoreCommitImplTest, TestFilterAndOverwrite) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); std::vector data_files = { @@ -4116,13 +4278,16 @@ TEST_F(FileStoreCommitImplTest, TestFilterAndOverwrite) { } TEST_F(FileStoreCommitImplTest, TestFilterAndOverwriteWithCompactIndexFiles) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); const BinaryRow partition = CreateIntRow(10); @@ -4155,13 +4320,16 @@ TEST_F(FileStoreCommitImplTest, TestFilterAndOverwriteWithCompactIndexFiles) { } TEST_F(FileStoreCommitImplTest, TestOverwriteWithSpecifyPartition) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -4189,13 +4357,16 @@ TEST_F(FileStoreCommitImplTest, TestOverwriteWithSpecifyPartition) { } TEST_F(FileStoreCommitImplTest, TestOverwriteWithSameFile) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -4219,14 +4390,17 @@ TEST_F(FileStoreCommitImplTest, TestOverwriteWithSameFile) { } TEST_F(FileStoreCommitImplTest, TestAppendDiscardDuplicateFiles) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + context_builder_options[Options::COMMIT_DISCARD_DUPLICATE_FILES] = "true"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::COMMIT_DISCARD_DUPLICATE_FILES, "true") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -4251,12 +4425,15 @@ TEST_F(FileStoreCommitImplTest, TestAppendDiscardDuplicateFiles) { } TEST_F(FileStoreCommitImplTest, TestCommitWithIOException) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + + context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); std::vector> msgs = GetCommitMessages(paimon::test::GetDataDir() + @@ -4283,12 +4460,15 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithIOException) { std::string tmp_table_path = tmp_dir->Str(); ScopeGuard guard([&io_hook]() { io_hook->Clear(); }); io_hook->Reset(i, IOHook::Mode::RETURN_ERROR); + std::map context_builder2_options; CommitContextBuilder context_builder2(tmp_table_path, "commit_user_1"); + context_builder2_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder2_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context2, - context_builder2.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - auto commit2 = FileStoreCommit::Create(std::move(commit_context2)); + + context_builder2.Finish()); + auto commit2 = + FileStoreCommit::Create(std::move(commit_context2), context_builder2_options); CHECK_HOOK_STATUS(commit2.status(), i); CHECK_HOOK_STATUS(commit2.value()->Commit(msgs), i); commit_run_complete = true; @@ -4314,7 +4494,7 @@ TEST_F(FileStoreCommitImplTest, TestObjectStoreAllowedWithRESTCatalogCommit) { // REST commit with local path should work without the object store flag CommitContextBuilder builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(auto ctx, builder.UseRESTCatalogCommit(true).Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(ctx))); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(ctx), {})); std::vector> msgs = CommitMessagesOfRound(1); ASSERT_OK(commit->Commit(msgs)); @@ -4325,7 +4505,7 @@ TEST_F(FileStoreCommitImplTest, TestObjectStoreAllowedWithRESTCatalogCommit) { CommitContextBuilder catalog_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(auto catalog_ctx, catalog_builder.WithCatalog(catalog, Identifier("db", "tbl")).Finish()); - ASSERT_OK_AND_ASSIGN(auto catalog_commit, FileStoreCommit::Create(std::move(catalog_ctx))); + ASSERT_OK_AND_ASSIGN(auto catalog_commit, FileStoreCommit::Create(std::move(catalog_ctx), {})); ASSERT_OK(catalog_commit->Commit(msgs)); ASSERT_EQ(catalog->CommitCalls().size(), 1u); @@ -4338,11 +4518,11 @@ TEST_F(FileStoreCommitImplTest, TestObjectStoreAllowedWithRESTCatalogCommit) { ASSERT_OK_AND_ASSIGN( auto object_store_ctx, object_store_builder.WithCatalog(object_store_catalog, Identifier("db", "tbl")).Finish()); - ASSERT_OK(FileStoreCommit::Create(std::move(object_store_ctx))); + ASSERT_OK(FileStoreCommit::Create(std::move(object_store_ctx), {})); CommitContextBuilder plain_builder(object_store_path, "commit_user_1"); ASSERT_OK_AND_ASSIGN(auto plain_ctx, plain_builder.WithFileSystem(file_system_).Finish()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(plain_ctx)), + ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(plain_ctx), {}), "invalid scheme oss for local file system"); } @@ -4364,10 +4544,13 @@ TEST_F(FileStoreCommitImplTest, TestFixedBucketPKTableCommitAllowed) { std::string pk_table_path = PathUtil::JoinPath(pk_root, "db.db/pk_tbl"); + std::map builder_options; CommitContextBuilder builder(pk_table_path, "test_user"); - builder.AddOption(Options::FILE_SYSTEM, "local").UseRESTCatalogCommit(true); + builder_options[Options::FILE_SYSTEM] = "local"; + builder.UseRESTCatalogCommit(true); ASSERT_OK_AND_ASSIGN(auto commit_context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto committer, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN(auto committer, + FileStoreCommit::Create(std::move(commit_context), builder_options)); ASSERT_TRUE(committer != nullptr); } @@ -4397,14 +4580,17 @@ TEST_F(FileStoreCommitImplTest, ValidateCommitOptionsAllowsManifestDeleteFileDro } TEST_F(FileStoreCommitImplTest, TestGetAllFilesKeepsValueStats) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::MANIFEST_DELETE_FILE_DROP_STATS] = "true"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::MANIFEST_DELETE_FILE_DROP_STATS, "true") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_context))); + + context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr commit, + FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = dynamic_cast(commit.get()); ASSERT_TRUE(commit_impl); @@ -4426,14 +4612,17 @@ TEST_F(FileStoreCommitImplTest, TestGetAllFilesKeepsValueStats) { } TEST_F(FileStoreCommitImplTest, TestOverwriteDropsDeleteFileStats) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::MANIFEST_DELETE_FILE_DROP_STATS] = "true"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::MANIFEST_DELETE_FILE_DROP_STATS, "true") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_context))); + + context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr commit, + FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = dynamic_cast(commit.get()); ASSERT_TRUE(commit_impl); @@ -4501,21 +4690,28 @@ TEST_F(FileStoreCommitImplTest, TestOverwriteDropsDeleteFileStats) { } TEST_F(FileStoreCommitImplTest, DropPartitionWithEmptyPartitionsFails) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::FILE_SYSTEM, "local").Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + + context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); ASSERT_NOK_WITH_MSG(commit->DropPartition({}, /*commit_identifier=*/1), "partitions list cannot be empty"); } TEST_F(FileStoreCommitImplTest, FilterAndCommitMultipleIdentifiersAndEmptyInput) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + + context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -4549,10 +4745,14 @@ TEST_F(FileStoreCommitImplTest, FilterAndCommitMultipleIdentifiersAndEmptyInput) } TEST_F(FileStoreCommitImplTest, CheckFilesExistenceFailsForNonImplCommitMessage) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::FILE_SYSTEM, "local").Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + + context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -4563,10 +4763,14 @@ TEST_F(FileStoreCommitImplTest, CheckFilesExistenceFailsForNonImplCommitMessage) } TEST_F(FileStoreCommitImplTest, CheckFilesExistenceCollectsIndexFilePaths) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::FILE_SYSTEM, "local").Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + + context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); const BinaryRow partition = CreateIntRow(10); @@ -4590,13 +4794,16 @@ TEST_F(FileStoreCommitImplTest, CheckFilesExistenceCollectsIndexFilePaths) { } TEST_F(FileStoreCommitImplTest, OverwriteStaticPartitionValidatesFileOwnership) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + context_builder_options[Options::DYNAMIC_PARTITION_OVERWRITE] = "false"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::DYNAMIC_PARTITION_OVERWRITE, "false") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + + context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -4618,12 +4825,15 @@ TEST_F(FileStoreCommitImplTest, OverwriteStaticPartitionValidatesFileOwnership) } TEST_F(FileStoreCommitImplTest, OverwriteWithChangelogFilesLogsWarning) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + + context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); const BinaryRow partition = CreateIntRow(10); @@ -4663,12 +4873,15 @@ TEST_F(FileStoreCommitImplTest, OverwriteUpgradesNonOverlappingPrimaryKeyFiles) /*ignore_if_exists=*/false)); std::string pk_table_path = PathUtil::JoinPath(pk_root, "db.db/pk_tbl"); + std::map builder_options; CommitContextBuilder builder(pk_table_path, "test_user"); + builder_options[Options::FILE_SYSTEM] = "local"; + builder_options[Options::OVERWRITE_UPGRADE] = "true"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - builder.AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::OVERWRITE_UPGRADE, "true") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + + builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, + FileStoreCommit::Create(std::move(commit_context), builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -4713,13 +4926,15 @@ TEST_F(FileStoreCommitImplTest, OverwriteUpgradesNonOverlappingPrimaryKeyFiles) } TEST_F(FileStoreCommitImplTest, CommitWithAppendCommitCheckConflict) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .AppendCommitCheckConflict(true) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + + context_builder.AppendCommitCheckConflict(true).Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); std::vector> msgs = GetCommitMessages(paimon::test::GetDataDir() + @@ -4736,12 +4951,15 @@ TEST_F(FileStoreCommitImplTest, SnapshotSequenceMaxFallsBackToManifestScan) { // First snapshot is committed in the default (scan) mode, so it does NOT carry the // max-sequence-number property. { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + + context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context), + context_builder_options)); std::vector> msgs1 = GetCommitMessages(paimon::test::GetDataDir() + "/orc/append_09.db/append_09/commit_messages/commit_messages-01", @@ -4751,13 +4969,16 @@ TEST_F(FileStoreCommitImplTest, SnapshotSequenceMaxFallsBackToManifestScan) { // Second commit uses snapshot-init mode; since the previous snapshot lacks the property, // the max sequence number is recomputed by scanning the base manifests. + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + context_builder_options[Options::WRITE_SEQUENCE_NUMBER_INIT_MODE] = "snapshot"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::WRITE_SEQUENCE_NUMBER_INIT_MODE, "snapshot") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + + context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -4780,12 +5001,15 @@ TEST_F(FileStoreCommitImplTest, SnapshotSequenceMaxFallsBackToManifestScan) { } TEST_F(FileStoreCommitImplTest, FilterAndOverwriteWithSpecifiedPartition) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + + context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -4806,12 +5030,15 @@ TEST_F(FileStoreCommitImplTest, FilterAndOverwriteWithSpecifiedPartition) { } TEST_F(FileStoreCommitImplTest, TryUpgradeReturnsInputWhenOverwriteUpgradeDisabled) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::FILE_SYSTEM] = "local"; + context_builder_options[Options::OVERWRITE_UPGRADE] = "false"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::OVERWRITE_UPGRADE, "false") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + + context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -4839,12 +5066,15 @@ TEST_F(FileStoreCommitImplTest, TryUpgradeReturnsInputWhenEntryLevelAboveZero) { /*ignore_if_exists=*/false)); std::string pk_table_path = PathUtil::JoinPath(pk_root, "db.db/pk_tbl"); + std::map builder_options; CommitContextBuilder builder(pk_table_path, "test_user"); + builder_options[Options::FILE_SYSTEM] = "local"; + builder_options[Options::OVERWRITE_UPGRADE] = "true"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - builder.AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::OVERWRITE_UPGRADE, "true") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + + builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, + FileStoreCommit::Create(std::move(commit_context), builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -4859,12 +5089,15 @@ TEST_F(FileStoreCommitImplTest, TryUpgradeReturnsInputWhenEntryLevelAboveZero) { } TEST_F(FileStoreCommitImplTest, CheckSameBucketFromSnapshotReturnsOkForEmptyDelta) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + + context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -4882,10 +5115,14 @@ TEST_F(FileStoreCommitImplTest, CheckSameBucketFromSnapshotReturnsOkForEmptyDelt } TEST_F(FileStoreCommitImplTest, MaxSequenceNumberReturnsNulloptForEmptyManifests) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::FILE_SYSTEM, "local").Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + + context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -4895,10 +5132,14 @@ TEST_F(FileStoreCommitImplTest, MaxSequenceNumberReturnsNulloptForEmptyManifests } TEST_F(FileStoreCommitImplTest, RowIdCheckConflictSetsCheckSnapshotAndReturnsSelf) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::FILE_SYSTEM, "local").Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + + context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); diff --git a/src/paimon/core/operation/file_store_commit_test.cpp b/src/paimon/core/operation/file_store_commit_test.cpp index 753f2bba7..aa61dfb46 100644 --- a/src/paimon/core/operation/file_store_commit_test.cpp +++ b/src/paimon/core/operation/file_store_commit_test.cpp @@ -81,12 +81,15 @@ TEST(FileStoreCommitTest, TestCreate) { /*ignore_if_exists=*/false)); std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); + std::map context_builder_options; CommitContextBuilder context_builder(table_path, "commit_user"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + + context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = dynamic_cast(commit.get()); ASSERT_TRUE(commit_impl); } @@ -100,9 +103,8 @@ TEST(FileStoreCommitTest, TestCreateWithCatalogRequiresIdentifier) { dir->Str(), "commit_user", /*ignore_empty_commit=*/true, /*use_rest_catalog_commit=*/false, catalog, /*identifier=*/std::nullopt, /*table_id=*/std::nullopt, /*append_commit_check_conflict=*/false, GetDefaultPool(), - CreateDefaultExecutor(), /*specific_file_system=*/nullptr, options, - BranchManager::DEFAULT_MAIN_BRANCH, /*named_branch=*/std::nullopt); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(commit_context)), + CreateDefaultExecutor(), /*specific_file_system=*/nullptr); + ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(commit_context), {}), "a catalog commit requires a table identifier"); } @@ -127,7 +129,7 @@ TEST(FileStoreCommitTest, TestCatalogCommitUsesPerTableFileSystem) { CommitContextBuilder builder(table_path, "commit_user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, builder.WithCatalog(catalog, identifier).Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(ctx))); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(ctx), {})); // The file system was resolved per table, not from the catalog-wide one. const std::vector& requests = catalog->TableFileSystemRequests(); @@ -162,12 +164,15 @@ TEST(FileStoreCommitTest, TestAppendDvIndexShouldUseOverwriteCommitKind) { /*ignore_if_exists=*/false)); std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); + std::map context_builder_options; CommitContextBuilder context_builder(table_path, "commit_user"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + + context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); LinkedHashMap dv_ranges; dv_ranges.insert_or_assign("data-file-1", diff --git a/src/paimon/core/operation/file_store_write.cpp b/src/paimon/core/operation/file_store_write.cpp index 36ea1dea1..2cd3bc97e 100644 --- a/src/paimon/core/operation/file_store_write.cpp +++ b/src/paimon/core/operation/file_store_write.cpp @@ -25,12 +25,11 @@ namespace paimon { -Result> FileStoreWrite::Create(std::unique_ptr ctx) { +Result> FileStoreWrite::Create( + std::unique_ptr ctx, const std::map& options) { PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "write")); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, FileStoreTableFactory::Create(*ctx)); - // The load folded the context's options into the table, so the operator below serves them - // from there and the context carries none of its own. - ctx->ClearOptions(); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, + FileStoreTableFactory::Create(*ctx, options)); return table->NewWrite(std::move(ctx)); } diff --git a/src/paimon/core/operation/file_store_write_test.cpp b/src/paimon/core/operation/file_store_write_test.cpp index ffee33161..ad1c664ae 100644 --- a/src/paimon/core/operation/file_store_write_test.cpp +++ b/src/paimon/core/operation/file_store_write_test.cpp @@ -88,14 +88,16 @@ TEST(FileStoreWriteTest, TestRealtimeCatalogCommitRefreshAndRecovery) { }; auto create_writer = [&](const std::shared_ptr& realtime_context) -> Result> { + std::map builder_options; WriteContextBuilder builder(table_path, "writer"); + builder_options[Options::FILE_SYSTEM] = "unregistered-file-system"; PAIMON_ASSIGN_OR_RAISE(std::unique_ptr ctx, + builder.WithCatalog(catalog, identifier) .WithStreamingMode(true) .WithRealtimeContext(realtime_context) - .AddOption(Options::FILE_SYSTEM, "unregistered-file-system") .Finish()); - return FileStoreWrite::Create(std::move(ctx)); + return FileStoreWrite::Create(std::move(ctx), builder_options); }; ASSERT_OK_AND_ASSIGN(auto realtime_context, RealtimeContext::Create()); ASSERT_OK_AND_ASSIGN(auto writer, create_writer(realtime_context)); @@ -106,7 +108,7 @@ TEST(FileStoreWriteTest, TestRealtimeCatalogCommitRefreshAndRecovery) { CommitContextBuilder commit_builder(table_path, "writer"); ASSERT_OK_AND_ASSIGN(auto commit_ctx, commit_builder.WithCatalog(catalog, identifier).Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_ctx))); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_ctx), {})); ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, commit->CommitWithProgress(progress, 1, std::nullopt)); ASSERT_EQ(snapshot_id, 1); @@ -149,11 +151,11 @@ TEST(FileStoreWriteTest, TestCreateWithInvalidInput) { WriteContextBuilder builder(dir->Str(), "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx1, builder.WithMemoryPool(nullptr).Finish()); - ASSERT_NOK(FileStoreWrite::Create(std::move(ctx1))); + ASSERT_NOK(FileStoreWrite::Create(std::move(ctx1), {})); ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx2, builder.WithExecutor(nullptr).Finish()); - ASSERT_NOK(FileStoreWrite::Create(std::move(ctx2))); - ASSERT_NOK(FileStoreWrite::Create(/*context=*/nullptr)); + ASSERT_NOK(FileStoreWrite::Create(std::move(ctx2), {})); + ASSERT_NOK(FileStoreWrite::Create(/*context=*/nullptr, {})); } TEST(FileStoreWriteTest, TestCreateAppendTable) { @@ -180,7 +182,7 @@ TEST(FileStoreWriteTest, TestCreateAppendTable) { } ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(std::move(write_context), {})); ASSERT_OK(file_store_write->Close()); } } @@ -202,7 +204,7 @@ TEST(FileStoreWriteTest, TestCatalogWriteAsksTheCatalogOnlyForTheMainBranchSchem std::unique_ptr main_context, main_builder.WithCatalog(catalog, Identifier("db", "table$branch_main")).Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr main_write, - FileStoreWrite::Create(std::move(main_context))); + FileStoreWrite::Create(std::move(main_context), {})); ASSERT_OK(main_write->Close()); ASSERT_EQ(catalog->LoadTableSchemaIdentifiers().size(), 1u); ASSERT_EQ(catalog->LoadTableSchemaIdentifiers().front(), Identifier("db", "table")); @@ -216,7 +218,7 @@ TEST(FileStoreWriteTest, TestCatalogWriteAsksTheCatalogOnlyForTheMainBranchSchem std::unique_ptr branch_context, branch_builder.WithCatalog(catalog, Identifier("db", "table$branch_dev")).Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr branch_write, - FileStoreWrite::Create(std::move(branch_context))); + FileStoreWrite::Create(std::move(branch_context), {})); ASSERT_OK(branch_write->Close()); ASSERT_EQ(catalog->LoadTableSchemaIdentifiers().size(), 1u); } @@ -243,7 +245,7 @@ TEST(FileStoreWriteTest, TestCatalogWriteUsesPerTableFileSystem) { ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, builder.WithCatalog(catalog, identifier).Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr writer, - FileStoreWrite::Create(std::move(ctx))); + FileStoreWrite::Create(std::move(ctx), {})); ASSERT_OK(writer->Close()); // The file system was resolved per table, not from the catalog-wide one. @@ -267,11 +269,14 @@ TEST(FileStoreWriteTest, TestCreateAppendTableWithInvalidBucket) { ASSERT_OK(catalog->CreateTable(Identifier("foo", "bar"), &schema, /*partition_keys=*/{"f0", "f3"}, /*primary_keys=*/{}, options, /*ignore_if_exists=*/false)); + std::map context_builder_options; WriteContextBuilder context_builder(PathUtil::JoinPath(dir->Str(), "foo.db/bar"), "commit_user_1"); + context_builder_options[Options::BUCKET] = "-2"; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.AddOption(Options::BUCKET, "-2").Finish()); - ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_context)), + + context_builder.Finish()); + ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_context), context_builder_options), "not support bucket -2 in append table"); } @@ -293,13 +298,14 @@ TEST(FileStoreWriteTest, TestCreateAppendTableWithInvalidWriteType) { ASSERT_OK(catalog->CreateTable(Identifier("foo", "bar"), &schema, /*partition_keys=*/{}, /*primary_keys=*/{}, options, /*ignore_if_exists=*/false)); + std::map context_builder_options; WriteContextBuilder context_builder(PathUtil::JoinPath(dir->Str(), "foo.db/bar"), "commit_user_1"); + context_builder_options[Options::BUCKET] = "-1"; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.AddOption(Options::BUCKET, "-1") - .WithWriteSchema({"field_non_exist"}) - .Finish()); - ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_context)), + + context_builder.WithWriteSchema({"field_non_exist"}).Finish()); + ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_context), context_builder_options), "write field field_non_exist does not exist in table schema"); } @@ -319,12 +325,15 @@ TEST(FileStoreWriteTest, TestCreatePrimaryKeyTable) { ASSERT_OK(catalog->CreateTable(Identifier("foo", "bar"), &schema, /*partition_keys=*/{"f0"}, /*primary_keys=*/{"f0", "f1", "f4"}, options, /*ignore_if_exists=*/false)); + std::map context_builder_options; WriteContextBuilder context_builder(PathUtil::JoinPath(dir->Str(), "foo.db/bar"), "commit_user_1"); + context_builder_options[Options::BUCKET] = "2"; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.AddOption(Options::BUCKET, "2").Finish()); + + context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(std::move(write_context), context_builder_options)); auto fs = std::make_shared(); SchemaManager schema_manager(fs, PathUtil::JoinPath(dir->Str(), "foo.db/bar")); @@ -356,7 +365,7 @@ TEST(FileStoreWriteTest, TestCreatePrimaryKeyTableWithInvalidBucket) { WriteContextBuilder context_builder(PathUtil::JoinPath(dir->Str(), "foo.db/bar"), "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_context)), + ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_context), {}), "not support bucket -1 in key value table"); } diff --git a/src/paimon/core/operation/internal_read_context.cpp b/src/paimon/core/operation/internal_read_context.cpp index e17cfbd4e..1beb19d76 100644 --- a/src/paimon/core/operation/internal_read_context.cpp +++ b/src/paimon/core/operation/internal_read_context.cpp @@ -95,7 +95,7 @@ std::shared_ptr InternalReadContext::CreateDependencyCatalogCont for (const auto& [key, value] : catalog_options) { dependency_options[key] = value; } - const auto& read_options = read_context_->GetOptions(); + const auto& read_options = dynamic_options_; for (const auto& [key, value] : read_options) { dependency_options[key] = value; } @@ -279,16 +279,26 @@ std::optional InternalReadContext::TryResolveSpecialFieldByName( Result> InternalReadContext::CreateForTable( const std::shared_ptr& context, const std::shared_ptr& table_schema, - CoreOptions core_options, const std::string& root_path) { + CoreOptions core_options, const std::string& root_path, + const std::map& dynamic_options) { core_options.WithCache(context->GetCache()); - return CreateFromCoreOptions(context, table_schema, std::move(core_options), root_path); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr result, + CreateFromCoreOptions(context, table_schema, std::move(core_options), root_path)); + result->dynamic_options_ = dynamic_options; + return result; } Result> InternalReadContext::Create( const std::shared_ptr& context, const std::shared_ptr& table_schema, - const std::map& options) { - return CreateFromOptions(context, table_schema, options, /*file_system=*/nullptr, - /*path=*/std::nullopt); + const std::map& options, + const std::map& dynamic_options) { + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr result, + CreateFromOptions(context, table_schema, options, /*file_system=*/nullptr, + /*path=*/std::nullopt)); + result->dynamic_options_ = dynamic_options; + return result; } Result> InternalReadContext::CreateFromOptions( @@ -334,7 +344,11 @@ Result> InternalReadContext::CreateForFallb } else { options.erase(Options::BRANCH); } - return CreateFromOptions(context, table_schema, options, file_system, root_path); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr result, + CreateFromOptions(context, table_schema, options, file_system, root_path)); + result->dynamic_options_ = table_dynamic_options; + return result; } Result> InternalReadContext::CreateFromCoreOptions( diff --git a/src/paimon/core/operation/internal_read_context.h b/src/paimon/core/operation/internal_read_context.h index a710ca15e..8e05db14f 100644 --- a/src/paimon/core/operation/internal_read_context.h +++ b/src/paimon/core/operation/internal_read_context.h @@ -58,38 +58,24 @@ class InternalReadContext { std::string field_name; }; - /// Builds the context of the read a table serves: the options are the table's own, already - /// parsed, and the cache the context carries is armed on them here. - /// @param root_path The root path the table was opened with. The read reads from it in place of - /// the path the context names. + /// Builds a read context using the table's resolved options and the operation's cache. + /// @param root_path Table root path, overriding the context's path. static Result> CreateForTable( const std::shared_ptr& read_context, const std::shared_ptr& table_schema, CoreOptions core_options, - const std::string& root_path); + const std::string& root_path, const std::map& dynamic_options); - /// Builds the context of a read whose options are composed on the spot rather than the table's - /// own: the reads a write or a lookup serve inside themselves. Such a read serves the file - /// system the context resolved and reads from the path the context names. A fallback branch - /// reads through `CreateForFallbackBranch` instead, whose schema belongs to another branch. + /// Builds an internal write/lookup read from effective options and the context's file system. static Result> Create( const std::shared_ptr& read_context, const std::shared_ptr& table_schema, - const std::map& options); - - /// The read context of a fallback branch, whose schema is not the table's own: it is read - /// through a manager of that branch, and the read serves the options that schema was loaded - /// with, the branch it was read from among them. A branch the context carries is not honoured - /// here, and neither are options the context adds on its own: the fallback reader serves the - /// table the way the main reader does, and a load merges the context's options into the - /// dynamic options handed in here. The C++ reading of the Java - /// `FileStoreTableFactory#createOtherBranchTable`, which loads the other branch's table the - /// same way for the fallback reader to serve. - /// @param file_system The file system the table was opened with, in place of one resolved from - /// the context. - /// @param root_path The root path the table was opened with, in place of the one the context - /// names. - /// @param table_dynamic_options The dynamic options the load merged into the table - and the - /// ones its copies accumulated. + const std::map& options, + const std::map& dynamic_options); + + /// Loads the fallback branch schema and applies the table's accumulated dynamic options. + /// @param file_system File system used to open the table. + /// @param root_path Table root path, overriding the context's path. + /// @param table_dynamic_options Overrides accumulated when loading or copying the table. static Result> CreateForFallbackBranch( const std::shared_ptr& context, const std::string& branch, const std::shared_ptr& file_system, const std::string& root_path, @@ -200,17 +186,14 @@ class InternalReadContext { const CoreOptions& options, std::optional path, const std::shared_ptr& arrow_pool); - /// The tail of a read whose options are a map: they resolve to `CoreOptions` against - /// `file_system`, or against the context's own when it is null, and then go where - /// `CreateFromCoreOptions` takes them. + /// Resolves option maps using file_system, or the context's file system when null. static Result> CreateFromOptions( const std::shared_ptr& read_context, const std::shared_ptr& table_schema, const std::map& options, const std::shared_ptr& file_system, std::optional path); - /// The tail both `Create` entries share: the options arrive parsed and cached, and the - /// validations and the read schema below are the same whichever way they were resolved. + /// Validates the resolved options and builds the read schema. static Result> CreateFromCoreOptions( const std::shared_ptr& read_context, const std::shared_ptr& table_schema, CoreOptions core_options, @@ -233,9 +216,9 @@ class InternalReadContext { std::shared_ptr read_schema_; std::shared_ptr arrow_pool_; CoreOptions options_; + std::map dynamic_options_; std::map index_score_bindings_; - /// The root path a table reads through, or empty for the read context's own, which a caller - /// without a table leaves it as. + /// Optional table root path; unset for reads using the context's path. std::optional path_; }; diff --git a/src/paimon/core/operation/internal_read_context_test.cpp b/src/paimon/core/operation/internal_read_context_test.cpp index 6e4957f4e..c4b6746f0 100644 --- a/src/paimon/core/operation/internal_read_context_test.cpp +++ b/src/paimon/core/operation/internal_read_context_test.cpp @@ -57,7 +57,7 @@ TEST(InternalReadContext, TestReadWithUnspecifiedSchema) { ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); ASSERT_OK_AND_ASSIGN(auto internal_context, InternalReadContext::Create(std::move(read_context), table_schema, - table_schema->Options())); + table_schema->Options(), {})); std::vector read_fields = {DataField(0, arrow::field("f0", arrow::utf8())), DataField(1, arrow::field("f1", arrow::int32())), DataField(2, arrow::field("f2", arrow::int32())), @@ -75,7 +75,7 @@ TEST(InternalReadContext, TestReadWithSpecifiedSchema) { ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); ASSERT_OK_AND_ASSIGN(auto internal_context, InternalReadContext::Create(std::move(read_context), table_schema, - table_schema->Options())); + table_schema->Options(), {})); std::vector read_fields = {DataField(3, arrow::field("f3", arrow::float64())), DataField(0, arrow::field("f0", arrow::utf8()))}; auto expected_schema = DataField::ConvertDataFieldsToArrowSchema(read_fields); @@ -99,10 +99,12 @@ TEST(InternalReadContext, TestDependencyReadMergesTableCatalogAndReadOptions) { {CatalogOptions::URI, "http://read"}, {CatalogOptions::TOKEN, "read-token"}, {Options::BLOB_VIEW_RESOLVE_ENABLED, "false"}}; + std::map builder_options; ReadContextBuilder builder(path); - ASSERT_OK_AND_ASSIGN( - std::shared_ptr read_context, - builder.WithCatalog(catalog, Identifier("db1", "t1")).SetOptions(read_options).Finish()); + builder_options = read_options; + ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, + + builder.WithCatalog(catalog, Identifier("db1", "t1")).Finish()); std::map options = table_schema->Options(); options[Options::READ_BATCH_SIZE] = "64"; options[CatalogOptions::DATA_TOKEN_ENABLED] = "false"; @@ -110,8 +112,9 @@ TEST(InternalReadContext, TestDependencyReadMergesTableCatalogAndReadOptions) { for (const auto& [key, value] : read_options) { options[key] = value; } - ASSERT_OK_AND_ASSIGN(auto internal_context, - InternalReadContext::Create(read_context, table_schema, options)); + ASSERT_OK_AND_ASSIGN( + auto internal_context, + InternalReadContext::Create(read_context, table_schema, options, builder_options)); auto dependency_context = internal_context->CreateDependencyCatalogContext(); auto expected_options = options; @@ -119,7 +122,6 @@ TEST(InternalReadContext, TestDependencyReadMergesTableCatalogAndReadOptions) { ASSERT_EQ(dependency_context->root_path, catalog->GetRootPath()); ASSERT_EQ(dependency_context->options, expected_options); ASSERT_EQ(dependency_context->options.at("header.X-Paimon-Read-Via"), "table-view"); - ASSERT_EQ(read_context->GetOptions(), read_options); ASSERT_EQ(catalog->GetOptions(), catalog_options); ASSERT_EQ(internal_context->GetCoreOptions().ToMap(), options); } @@ -148,20 +150,22 @@ TEST(InternalReadContext, TestDependencyReadKeepsTheOutermostView) { if (test_case.read_header) { read_options["header.X-Paimon-Read-Via"] = test_case.read_header; } + std::map builder_options; ReadContextBuilder builder(path); - ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, - builder.WithCatalog(catalog, Identifier("db1", "nested-view")) - .SetOptions(read_options) - .Finish()); + builder_options = read_options; + ASSERT_OK_AND_ASSIGN( + std::shared_ptr read_context, + + builder.WithCatalog(catalog, Identifier("db1", "nested-view")).Finish()); auto options = table_schema->Options(); options["header.X-Paimon-Read-Via"] = test_case.read_header ? test_case.read_header : "table-view"; - ASSERT_OK_AND_ASSIGN(auto internal_context, - InternalReadContext::Create(read_context, table_schema, options)); + ASSERT_OK_AND_ASSIGN( + auto internal_context, + InternalReadContext::Create(read_context, table_schema, options, builder_options)); auto dependency_context = internal_context->CreateDependencyCatalogContext(); ASSERT_EQ(dependency_context->options.at("header.X-Paimon-Read-Via"), test_case.expected_header); - ASSERT_EQ(read_context->GetOptions(), read_options); ASSERT_EQ(catalog->GetOptions(), catalog_options); ASSERT_EQ(internal_context->GetCoreOptions().ToMap(), options); } @@ -205,13 +209,15 @@ TEST(InternalReadContext, TestDependencyReadHeaderUsesFinalMetastore) { read_options[CatalogOptions::METASTORE] = test_case.read_metastore; options[CatalogOptions::METASTORE] = test_case.read_metastore; } + std::map builder_options; ReadContextBuilder builder(path); + builder_options = read_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, - builder.WithCatalog(catalog, Identifier("db1", "t1")) - .SetOptions(read_options) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto internal_context, - InternalReadContext::Create(read_context, table_schema, options)); + + builder.WithCatalog(catalog, Identifier("db1", "t1")).Finish()); + ASSERT_OK_AND_ASSIGN( + auto internal_context, + InternalReadContext::Create(read_context, table_schema, options, builder_options)); auto dependency_context = internal_context->CreateDependencyCatalogContext(); if (test_case.expected_metastore) { ASSERT_EQ(dependency_context->options.at(CatalogOptions::METASTORE), @@ -224,7 +230,6 @@ TEST(InternalReadContext, TestDependencyReadHeaderUsesFinalMetastore) { } else { ASSERT_EQ(dependency_context->options.count("header.X-Paimon-Read-Via"), 0u); } - ASSERT_EQ(read_context->GetOptions(), read_options); ASSERT_EQ(catalog->GetOptions(), catalog_options); ASSERT_EQ(internal_context->GetCoreOptions().ToMap(), options); } @@ -234,14 +239,18 @@ TEST(InternalReadContext, TestDependencyReadHeaderNeedsAnIdentifier) { std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; SchemaManager schema_manager(std::make_shared(), path); ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); + std::map builder_options; ReadContextBuilder builder(path); + builder_options[CatalogOptions::METASTORE] = "REST"; ASSERT_OK_AND_ASSIGN(auto read_context, - builder.AddOption(CatalogOptions::METASTORE, "REST").Finish()); + + builder.Finish()); ASSERT_FALSE(read_context->GetIdentifier()); auto options = table_schema->Options(); options[CatalogOptions::METASTORE] = "REST"; - ASSERT_OK_AND_ASSIGN(auto internal_context, InternalReadContext::Create(std::move(read_context), - table_schema, options)); + ASSERT_OK_AND_ASSIGN( + auto internal_context, + InternalReadContext::Create(std::move(read_context), table_schema, options, {})); auto dependency_context = internal_context->CreateDependencyCatalogContext(); ASSERT_EQ(dependency_context->options.count("header.X-Paimon-Read-Via"), 0u); } @@ -258,7 +267,7 @@ TEST(InternalReadContext, TestDependencyReadHeaderReportsTheBranchOfTheStartingT builder.WithCatalog(catalog, Identifier("db1", "t1", "dev")).Finish()); ASSERT_OK_AND_ASSIGN(auto internal_context, InternalReadContext::Create(std::move(read_context), table_schema, - table_schema->Options())); + table_schema->Options(), {})); auto dependency_context = internal_context->CreateDependencyCatalogContext(); ASSERT_EQ(dependency_context->options.at("header.X-Paimon-Read-Via"), "%7B%22database%22%3A%22db1%22%2C%22object%22%3A%22t1%24branch_dev%22%7D"); @@ -279,7 +288,7 @@ TEST(InternalReadContext, TestFileSystemSchemeMapReachesTheDependencyRead) { ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); ASSERT_OK_AND_ASSIGN(auto internal_context, InternalReadContext::Create(std::move(read_context), table_schema, - table_schema->Options())); + table_schema->Options(), {})); ASSERT_EQ(internal_context->GetFileSystemSchemeToIdentifierMap(), scheme_map); ASSERT_EQ(internal_context->CreateDependencyCatalogContext()->fs_scheme_to_identifier_map, scheme_map); @@ -294,7 +303,7 @@ TEST(InternalReadContext, TestReadWithSpecifiedFieldId) { ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); ASSERT_OK_AND_ASSIGN(auto internal_context, InternalReadContext::Create(std::move(read_context), table_schema, - table_schema->Options())); + table_schema->Options(), {})); std::vector read_fields = {DataField(3, arrow::field("f3", arrow::float64())), DataField(0, arrow::field("f0", arrow::utf8()))}; auto expected_schema = DataField::ConvertDataFieldsToArrowSchema(read_fields); @@ -313,7 +322,7 @@ TEST(InternalReadContext, TestReadWithSpecifiedFieldIdAndSchema) { ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); ASSERT_OK_AND_ASSIGN(auto internal_context, InternalReadContext::Create(std::move(read_context), table_schema, - table_schema->Options())); + table_schema->Options(), {})); std::vector read_fields = {DataField(3, arrow::field("f3", arrow::float64())), DataField(0, arrow::field("f0", arrow::utf8()))}; auto expected_schema = DataField::ConvertDataFieldsToArrowSchema(read_fields); @@ -335,7 +344,7 @@ TEST(InternalReadContext, TestReadWithRowTrackingAndScoreFields) { new_options[Options::DATA_EVOLUTION_ENABLED] = "true"; ASSERT_OK_AND_ASSIGN( auto internal_context, - InternalReadContext::Create(std::move(read_context), table_schema, new_options)); + InternalReadContext::Create(std::move(read_context), table_schema, new_options, {})); std::vector read_fields = { DataField(3, arrow::field("f3", arrow::float64())), DataField(0, arrow::field("f0", arrow::utf8())), SpecialFields::RowId(), @@ -352,7 +361,7 @@ TEST(InternalReadContext, TestReadWithRowTrackingAndScoreFields) { SchemaManager schema_manager(std::make_shared(), read_context->GetPath()); ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); ASSERT_NOK_WITH_MSG(InternalReadContext::Create(std::move(read_context), table_schema, - table_schema->Options()), + table_schema->Options(), {}), "Get field _ROW_ID failed: not exist in table schema"); } { @@ -364,7 +373,7 @@ TEST(InternalReadContext, TestReadWithRowTrackingAndScoreFields) { SchemaManager schema_manager(std::make_shared(), read_context->GetPath()); ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); ASSERT_NOK_WITH_MSG(InternalReadContext::Create(std::move(read_context), table_schema, - table_schema->Options()), + table_schema->Options(), {}), "Get field _INDEX_SCORE failed: not exist in table schema"); } } @@ -383,7 +392,7 @@ TEST(InternalReadContext, TestVectorSearchIndexScoreProjection) { ASSERT_OK_AND_ASSIGN(auto read_context, builder.Finish()); ASSERT_OK_AND_ASSIGN(auto internal_context, InternalReadContext::Create(std::move(read_context), table_schema, - table_schema->Options())); + table_schema->Options(), {})); std::shared_ptr expected = DataField::ConvertDataFieldsToArrowSchema( {DataField(0, arrow::field("f0", arrow::utf8())), SpecialFields::IndexScore()}); EXPECT_TRUE(internal_context->GetReadSchema()->Equals(expected)); @@ -394,7 +403,7 @@ TEST(InternalReadContext, TestVectorSearchIndexScoreProjection) { ASSERT_OK_AND_ASSIGN(auto read_context, builder.Finish()); ASSERT_OK_AND_ASSIGN(auto internal_context, InternalReadContext::Create(std::move(read_context), table_schema, - table_schema->Options())); + table_schema->Options(), {})); EXPECT_EQ(arrow::Type::FLOAT, internal_context->GetReadSchema()->field(1)->type()->id()); } { @@ -407,7 +416,7 @@ TEST(InternalReadContext, TestVectorSearchIndexScoreProjection) { ASSERT_OK_AND_ASSIGN(auto read_context, builder.Finish()); ASSERT_OK_AND_ASSIGN(auto internal_context, InternalReadContext::Create(std::move(read_context), table_schema, - table_schema->Options())); + table_schema->Options(), {})); EXPECT_EQ(std::vector({"_INDEX_SCORE", "f0"}), internal_context->GetReadSchema()->field_names()); } @@ -426,12 +435,12 @@ TEST(InternalReadContext, TestPrimaryKeyVectorSearchRequiresDeletionVectors) { ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr internal, - InternalReadContext::Create(context, table_schema, table_schema->Options())); + InternalReadContext::Create(context, table_schema, table_schema->Options(), {})); EXPECT_TRUE(internal->HasVectorOrFullTextSearch()); std::map options = table_schema->Options(); options[Options::DELETION_VECTORS_ENABLED] = "false"; - ASSERT_NOK_WITH_MSG(InternalReadContext::Create(context, table_schema, options), + ASSERT_NOK_WITH_MSG(InternalReadContext::Create(context, table_schema, options, {}), "primary-key tables requires deletion vectors"); } @@ -444,8 +453,9 @@ TEST(InternalReadContext, TestFileFullTextSearchScoreIsNotSupported) { ReadContextBuilder builder(path); builder.SetReadFieldNames({"f0", "_INDEX_SCORE"}).SetFullTextSearch(search); ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); - ASSERT_NOK_WITH_MSG(InternalReadContext::Create(context, table_schema, table_schema->Options()), - "does not support score output"); + ASSERT_NOK_WITH_MSG( + InternalReadContext::Create(context, table_schema, table_schema->Options(), {}), + "does not support score output"); } TEST(InternalReadContext, TestFullTextMatchValidatesProjection) { @@ -477,7 +487,7 @@ TEST(InternalReadContext, TestFullTextMatchValidatesProjection) { if (test_case.expected_error.empty()) { ASSERT_OK_AND_ASSIGN( std::shared_ptr internal, - InternalReadContext::Create(context, table_schema, table_schema->Options())); + InternalReadContext::Create(context, table_schema, table_schema->Options(), {})); ASSERT_TRUE(internal->GetIndexScoreBindings().empty()); ASSERT_FALSE(internal->NeedsScores( {InternalReadContext::IndexScoreSource::Type::FULL_TEXT_MATCH, match->field_name})); @@ -486,7 +496,7 @@ TEST(InternalReadContext, TestFullTextMatchValidatesProjection) { "CreateWithSchema does not support"); } else { ASSERT_NOK_WITH_MSG( - InternalReadContext::Create(context, table_schema, table_schema->Options()), + InternalReadContext::Create(context, table_schema, table_schema->Options(), {}), test_case.expected_error); } } @@ -521,7 +531,7 @@ TEST(InternalReadContext, TestPerFieldTopKScoreBindings) { ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); ASSERT_OK_AND_ASSIGN( std::shared_ptr internal, - InternalReadContext::Create(context, table_schema, table_schema->Options())); + InternalReadContext::Create(context, table_schema, table_schema->Options(), {})); ASSERT_EQ(internal->GetReadSchema()->field_names(), projection); ASSERT_TRUE(internal->NeedsScores(source)); ASSERT_FALSE(internal->NeedsScores( @@ -557,7 +567,7 @@ class InternalReadContextScoreTest : public testing::Test { Result> Create(ReadContextBuilder* builder) { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr context, builder->Finish()); - return InternalReadContext::Create(context, table_schema_, table_schema_->Options()); + return InternalReadContext::Create(context, table_schema_, table_schema_->Options(), {}); } std::shared_ptr table_schema_; @@ -783,7 +793,7 @@ TEST(InternalReadContext, TestReadWithValueKindField) { ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); ASSERT_OK_AND_ASSIGN(auto internal_context, InternalReadContext::Create(std::move(read_context), table_schema, - table_schema->Options())); + table_schema->Options(), {})); std::vector read_fields = {DataField(3, arrow::field("f3", arrow::float64())), SpecialFields::ValueKind(), DataField(0, arrow::field("f0", arrow::utf8()))}; @@ -809,7 +819,7 @@ TEST(InternalReadContext, TestReadWithFieldIdsAndSpecialFields) { new_options[Options::DATA_EVOLUTION_ENABLED] = "true"; ASSERT_OK_AND_ASSIGN( auto internal_context, - InternalReadContext::Create(std::move(read_context), table_schema, new_options)); + InternalReadContext::Create(std::move(read_context), table_schema, new_options, {})); std::vector read_fields = { DataField(3, arrow::field("f3", arrow::float64())), DataField(0, arrow::field("f0", arrow::utf8())), SpecialFields::RowId(), @@ -838,7 +848,7 @@ TEST(InternalReadContext, TestReadWithProjectedSchemaAndSpecialFields) { ASSERT_OK_AND_ASSIGN(auto unique_read_context, context_builder.Finish()); std::shared_ptr read_context = std::move(unique_read_context); ASSERT_NOK_WITH_MSG( - InternalReadContext::Create(read_context, table_schema, table_schema->Options()), + InternalReadContext::Create(read_context, table_schema, table_schema->Options(), {}), "not exist in table schema"); } @@ -857,7 +867,7 @@ TEST(InternalReadContext, TestReadWithProjectedSchemaAndSpecialFields) { std::shared_ptr read_context = std::move(unique_read_context); ASSERT_OK_AND_ASSIGN( auto internal_context, - InternalReadContext::Create(read_context, table_schema, enabled_options)); + InternalReadContext::Create(read_context, table_schema, enabled_options, {})); auto expected_schema = DataField::ConvertDataFieldsToArrowSchema(projected_fields); ASSERT_TRUE(internal_context->GetReadSchema()->Equals(expected_schema)); } @@ -881,7 +891,7 @@ TEST(InternalReadContext, TestReadWithProjectedSchemaWithoutFieldIds) { ASSERT_OK_AND_ASSIGN( auto internal_context, - InternalReadContext::Create(read_context, table_schema, table_schema->Options())); + InternalReadContext::Create(read_context, table_schema, table_schema->Options(), {})); std::vector expected_fields = { DataField(3, arrow::field("f3", arrow::float64())), @@ -913,7 +923,7 @@ TEST(InternalReadContext, TestProjectedSchemaMetadataWhitelist) { ASSERT_OK_AND_ASSIGN( auto internal_context, - InternalReadContext::Create(read_context, table_schema, table_schema->Options())); + InternalReadContext::Create(read_context, table_schema, table_schema->Options(), {})); auto aligned_field = internal_context->GetReadSchema()->GetFieldByName("f0"); ASSERT_TRUE(aligned_field); @@ -954,7 +964,7 @@ TEST(InternalReadContext, TestMapSharedShreddingAccessRequiresSharedShreddingLay std::shared_ptr read_context = std::move(unique_read_context); ASSERT_NOK_WITH_MSG( - InternalReadContext::Create(read_context, table_schema, table_schema->Options()), + InternalReadContext::Create(read_context, table_schema, table_schema->Options(), {}), "Selected-key MAP pushdown only supports top-level shared-shredding MAP field: tags"); } diff --git a/src/paimon/core/operation/key_value_file_store_write_test.cpp b/src/paimon/core/operation/key_value_file_store_write_test.cpp index c0e94e236..94ea4f9c2 100644 --- a/src/paimon/core/operation/key_value_file_store_write_test.cpp +++ b/src/paimon/core/operation/key_value_file_store_write_test.cpp @@ -142,7 +142,7 @@ class KeyValueFileStoreWriteTest : public ::testing::Test { PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, context_builder.Finish()); - return FileStoreWrite::Create(std::move(write_context)); + return FileStoreWrite::Create(std::move(write_context), {}); } Status WriteSingleStringRow(FileStoreWrite* file_store_write, int32_t bucket, @@ -196,11 +196,13 @@ class KeyValueFileStoreWriteTest : public ::testing::Test { const std::string& table_path, const std::shared_ptr& schema, const std::map& options, const std::string& json, int64_t commit_identifier) const { + std::map builder_options; WriteContextBuilder builder(table_path, "test"); - builder.SetOptions(options); + builder_options = options; + EXPECT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); EXPECT_OK_AND_ASSIGN(auto file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(std::move(write_context), builder_options)); EXPECT_OK(file_store_write->Write(MakeBatch(schema, json))); EXPECT_OK_AND_ASSIGN(auto commit_msgs, file_store_write->PrepareCommit( /*wait_compaction=*/false, commit_identifier)); @@ -219,11 +221,13 @@ class KeyValueFileStoreWriteTest : public ::testing::Test { void Commit(const std::string& table_path, const std::map& options, const std::vector>& commit_msgs) const { + std::map builder_options; CommitContextBuilder builder(table_path, "test"); - builder.SetOptions(options); + builder_options = options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, builder.Finish()); ASSERT_OK_AND_ASSIGN(auto file_store_commit, - FileStoreCommit::Create(std::move(commit_context))); + FileStoreCommit::Create(std::move(commit_context), builder_options)); ASSERT_OK(file_store_commit->Commit(commit_msgs)); } @@ -340,7 +344,7 @@ TEST_F(KeyValueFileStoreWriteTest, TestWriteWithInvalidBatch) { WriteContextBuilder builder(PathUtil::JoinPath(dir->Str(), "foo.db/bar"), commit_user); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); ASSERT_OK_AND_ASSIGN(auto file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(std::move(write_context), {})); ASSERT_NOK_WITH_MSG(file_store_write->Write(nullptr), "batch is null pointer"); } { @@ -359,7 +363,7 @@ TEST_F(KeyValueFileStoreWriteTest, TestWriteWithInvalidBatch) { commit_user); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(std::move(write_context), {})); auto array = std::make_shared(); arrow::StringBuilder builder; for (size_t j = 0; j < 100; j++) { @@ -391,7 +395,7 @@ TEST_F(KeyValueFileStoreWriteTest, TestWriteWithInvalidBatch) { commit_user); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(std::move(write_context), {})); auto array = std::make_shared(); arrow::StringBuilder builder; for (size_t j = 0; j < 100; j++) { @@ -442,14 +446,15 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimeWrite) { ASSERT_OK_AND_ASSIGN(std::shared_ptr realtime_context, RealtimeContext::Create()); + std::map builder_options; WriteContextBuilder builder(table_path, "test"); - builder.SetOptions(options) - .WithStreamingMode(true) + builder_options = options; + builder.WithStreamingMode(true) .WithRealtimeContext(realtime_context) .WithTempDirectory(dir->Str()); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr writer, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(std::move(write_context), builder_options)); std::unique_ptr batch = MakeBatch(realtime_schema, R"([ @@ -505,15 +510,17 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimeWrite) { SCOPED_TRACE(expected_error); ASSERT_OK_AND_ASSIGN(std::shared_ptr invalid_context, RealtimeContext::Create()); + std::map invalid_builder_options; WriteContextBuilder invalid_builder(table_path, "invalid"); - invalid_builder.SetOptions(options) - .WithStreamingMode(true) + invalid_builder_options = options; + invalid_builder.WithStreamingMode(true) .WithRealtimeContext(invalid_context) .WithTempDirectory(dir->Str()); ASSERT_OK_AND_ASSIGN(std::unique_ptr invalid_write_context, invalid_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr invalid_writer, - FileStoreWrite::Create(std::move(invalid_write_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr invalid_writer, + FileStoreWrite::Create(std::move(invalid_write_context), invalid_builder_options)); ASSERT_OK(invalid_writer->Write(MakeBatch(realtime_schema, R"([[30, 1, "existing"]])"))); ASSERT_OK_AND_ASSIGN(std::vector prepared, invalid_writer->PrepareCommitWithProgress(/*commit_identifier=*/0)); @@ -544,14 +551,13 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimePool) { std::shared_ptr pool = std::make_shared(); ASSERT_OK_AND_ASSIGN(std::shared_ptr realtime_context, RealtimeContext::Create()); + std::map builder_options; WriteContextBuilder builder(table_path, "test"); - builder.SetOptions(options) - .WithStreamingMode(true) - .WithRealtimeContext(realtime_context) - .WithMemoryPool(pool); + builder_options = options; + builder.WithStreamingMode(true).WithRealtimeContext(realtime_context).WithMemoryPool(pool); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr writer, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(std::move(write_context), builder_options)); const int64_t allocations_before_write = pool->allocation_count; ASSERT_OK(writer->Write(MakeBatch(realtime_schema, R"([[0, 1, "one"]])"))); @@ -567,15 +573,17 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimePool) { std::shared_ptr rejecting_pool = std::make_shared(); ASSERT_OK_AND_ASSIGN(std::shared_ptr rejecting_context, RealtimeContext::Create()); + std::map rejecting_builder_options; WriteContextBuilder rejecting_builder(table_path, "rejecting"); - rejecting_builder.SetOptions(options) - .WithStreamingMode(true) + rejecting_builder_options = options; + rejecting_builder.WithStreamingMode(true) .WithRealtimeContext(rejecting_context) .WithMemoryPool(rejecting_pool); ASSERT_OK_AND_ASSIGN(std::unique_ptr rejecting_write_context, rejecting_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr rejecting_writer, - FileStoreWrite::Create(std::move(rejecting_write_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr rejecting_writer, + FileStoreWrite::Create(std::move(rejecting_write_context), rejecting_builder_options)); ASSERT_OK(rejecting_writer->Write(MakeBatch(realtime_schema, "[]"))); const int64_t rejecting_allocations_before_write = rejecting_pool->allocation_count; rejecting_pool->reject_allocations = true; @@ -605,13 +613,15 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimeLimits) { ASSERT_OK_AND_ASSIGN(std::shared_ptr initial_context, RealtimeContext::Create()); + std::map initial_builder_options; WriteContextBuilder initial_builder(table_path, "initial"); - initial_builder.SetOptions(options).WithStreamingMode(true).WithRealtimeContext( - initial_context); + initial_builder_options = options; + initial_builder.WithStreamingMode(true).WithRealtimeContext(initial_context); ASSERT_OK_AND_ASSIGN(std::unique_ptr initial_write_context, initial_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr initial_writer, - FileStoreWrite::Create(std::move(initial_write_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr initial_writer, + FileStoreWrite::Create(std::move(initial_write_context), initial_builder_options)); ASSERT_OK(initial_writer->Write(MakeBatch(realtime_schema, R"([[0, 0, "initial"]])"))); ASSERT_OK_AND_ASSIGN(std::vector initial_progress, initial_writer->PrepareCommitWithProgress(0)); @@ -623,22 +633,27 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimeLimits) { initial_message->GetNewFilesIncrement().NewFiles()[0]->AssignSequenceNumber(max - 2, max - 2); initial_progress[0].offset_range = OffsetRange(0, max - 1); + std::map commit_builder_options; CommitContextBuilder commit_builder(table_path, "initial"); - commit_builder.SetOptions(options); + commit_builder_options = options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr committer, - FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr committer, + FileStoreCommit::Create(std::move(commit_context), commit_builder_options)); ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, committer->CommitWithProgress(initial_progress, 0, std::nullopt)); ASSERT_OK(initial_writer->Close()); ASSERT_OK_AND_ASSIGN(std::shared_ptr realtime_context, RealtimeContext::Create()); + std::map builder_options; WriteContextBuilder builder(table_path, "boundary"); - builder.SetOptions(options).WithStreamingMode(true).WithRealtimeContext(realtime_context); + builder_options = options; + builder.WithStreamingMode(true).WithRealtimeContext(realtime_context); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr writer, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(std::move(write_context), builder_options)); ASSERT_OK( writer->Write(MakeBatch(realtime_schema, fmt::format(R"([[{}, 1, "legal"]])", max - 1)))); ASSERT_OK_AND_ASSIGN(auto store_rows, ReadRealtimePrimaryKeyStoreRows(realtime_context)); @@ -751,7 +766,8 @@ TEST_F(KeyValueFileStoreWriteTest, TestSpillSimple) { context_builder.WithTempDirectory(dir->Str()); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN(auto file_store_write, + FileStoreWrite::Create(std::move(write_context), {})); auto key_value_file_store_write = dynamic_cast(file_store_write.get()); auto get_writer = [&](int32_t bucket) -> std::shared_ptr { auto partition_iter = key_value_file_store_write->writers_.find(BinaryRow::EmptyRow()); @@ -806,11 +822,14 @@ TEST_F(KeyValueFileStoreWriteTest, TestWriterRestoreKeepsValueStats) { /*ignore_if_exists=*/false)); std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); + std::map first_context_builder_options; WriteContextBuilder first_context_builder(table_path, "first-writer"); - first_context_builder.SetOptions(options); + first_context_builder_options = options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr first_context, first_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto first_write, FileStoreWrite::Create(std::move(first_context))); + ASSERT_OK_AND_ASSIGN(auto first_write, FileStoreWrite::Create(std::move(first_context), + first_context_builder_options)); ASSERT_OK(WriteSingleStringRow(first_write.get(), /*bucket=*/0, "alice")); ASSERT_OK_AND_ASSIGN(auto commit_messages, first_write->PrepareCommit(/*wait_compaction=*/false, 1)); @@ -823,11 +842,15 @@ TEST_F(KeyValueFileStoreWriteTest, TestWriterRestoreKeepsValueStats) { SimpleStats::EmptyStats()); Commit(table_path, options, commit_messages); + std::map restored_context_builder_options; WriteContextBuilder restored_context_builder(table_path, "restored-writer"); - restored_context_builder.SetOptions(options); + restored_context_builder_options = options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr restored_context, restored_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto restored_write, FileStoreWrite::Create(std::move(restored_context))); + ASSERT_OK_AND_ASSIGN( + auto restored_write, + FileStoreWrite::Create(std::move(restored_context), restored_context_builder_options)); auto key_value_write = dynamic_cast(restored_write.get()); ASSERT_NE(nullptr, key_value_write); ASSERT_OK_AND_ASSIGN(std::shared_ptr restore_files, @@ -860,7 +883,8 @@ TEST_F(KeyValueFileStoreWriteTest, TestSpillDiskQuotaExhaustedFallsBackToFlushDa context_builder.WithTempDirectory(dir->Str()); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN(auto file_store_write, + FileStoreWrite::Create(std::move(write_context), {})); // Disk quota is 1 byte, so spill will exhaust quota immediately and fall back to // FlushWriteBuffer (writing data files directly instead of spill temp files). @@ -908,7 +932,8 @@ TEST_F(KeyValueFileStoreWriteTest, TestMultiRoundSpillWithSameKeyDeduplication) context_builder.WithTempDirectory(dir->Str()).WithStreamingMode(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN(auto file_store_write, + FileStoreWrite::Create(std::move(write_context), {})); // Round 1: alice, bob, alice (duplicate key) → after dedup: alice + bob = 2 rows ASSERT_OK(WriteSingleStringRow(file_store_write.get(), /*bucket=*/0, "alice")); diff --git a/src/paimon/core/operation/merge_file_split_read_test.cpp b/src/paimon/core/operation/merge_file_split_read_test.cpp index 2096c2aa9..7fc553f99 100644 --- a/src/paimon/core/operation/merge_file_split_read_test.cpp +++ b/src/paimon/core/operation/merge_file_split_read_test.cpp @@ -91,23 +91,28 @@ class MergeFileSplitReadTest : public ::testing::Test, } std::shared_ptr CreateInternalReadContext( - const std::shared_ptr& read_context, int32_t schema_id = 0) { + const std::shared_ptr& read_context, int32_t schema_id, + const std::map& dynamic_options) { SchemaManager schema_manager(fs_, read_context->GetPath()); EXPECT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(schema_id)); + auto options = ReadOptions(); + for (const auto& [key, value] : dynamic_options) { + options[key] = value; + } EXPECT_OK_AND_ASSIGN(auto context, InternalReadContext::Create(read_context, table_schema, - read_context->GetOptions())); + options, dynamic_options)); return context; } + std::map ReadOptions() const { + auto [use_min_heap, enable_io_prefetch, enable_multi_thread_row_to_batch] = GetParam(); + return {{Options::SORT_ENGINE, use_min_heap ? "min-heap" : "loser-tree"}, + {"test.enable-adaptive-prefetch-strategy", "false"}}; + } + void AddOptions(ReadContextBuilder* context_builder) const { auto [use_min_heap, enable_io_prefetch, enable_multi_thread_row_to_batch] = GetParam(); - if (use_min_heap) { - context_builder->AddOption(Options::SORT_ENGINE, "min-heap"); - } else { - context_builder->AddOption(Options::SORT_ENGINE, "loser-tree"); - } if (enable_io_prefetch) { - context_builder->AddOption("test.enable-adaptive-prefetch-strategy", "false"); context_builder->EnablePrefetch(true); context_builder->SetPrefetchBatchCount(/*batch_count=*/3); } else { @@ -609,6 +614,7 @@ TEST_F(MergeFileSplitReadTest, TestGenerateKeyPredicates2) { TEST_P(MergeFileSplitReadTest, TestSimple) { std::string path = paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor"; + std::map context_builder_options; ReadContextBuilder context_builder(path); std::vector raw_read_fields = {DataField(1, arrow::field("k1", arrow::int32())), @@ -620,13 +626,14 @@ TEST_P(MergeFileSplitReadTest, TestSimple) { ASSERT_TRUE(read_schema); context_builder.SetReadFieldNames({"k1", "p1", "s1", "v0", "v1"}); - context_builder.SetOptions({{Options::SEQUENCE_FIELD, "s0,s1"}, - {Options::MERGE_ENGINE, "deduplicate"}, - {Options::IGNORE_DELETE, "true"}}); + context_builder_options = {{Options::SEQUENCE_FIELD, "s0,s1"}, + {Options::MERGE_ENGINE, "deduplicate"}, + {Options::IGNORE_DELETE, "true"}}; + AddOptions(&context_builder); ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); - auto internal_context = CreateInternalReadContext(read_context); + auto internal_context = CreateInternalReadContext(read_context, 0, context_builder_options); ASSERT_OK_AND_ASSIGN( std::unique_ptr split_read, @@ -675,6 +682,7 @@ TEST_P(MergeFileSplitReadTest, TestSimple) { TEST_P(MergeFileSplitReadTest, TestRealtimeReadConcatenatesOrderedDiskSections) { std::string path = paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor"; + std::map context_builder_options; ReadContextBuilder context_builder(path); std::vector raw_read_fields = {DataField(0, arrow::field("k0", arrow::int32())), DataField(1, arrow::field("k1", arrow::int32())), @@ -685,11 +693,13 @@ TEST_P(MergeFileSplitReadTest, TestRealtimeReadConcatenatesOrderedDiskSections) ASSERT_TRUE(read_schema); context_builder.SetReadFieldNames({"k0", "k1", "s1", "v0"}); - context_builder.SetOptions( - {{Options::SEQUENCE_FIELD, "s0,s1"}, {Options::MERGE_ENGINE, "deduplicate"}}); + context_builder_options = {{Options::SEQUENCE_FIELD, "s0,s1"}, + {Options::MERGE_ENGINE, "deduplicate"}}; + AddOptions(&context_builder); ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); - std::shared_ptr internal_context = CreateInternalReadContext(read_context); + std::shared_ptr internal_context = + CreateInternalReadContext(read_context, 0, context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr split_read, CreateMergeFileSplitRead(internal_context)); @@ -742,6 +752,7 @@ TEST_P(MergeFileSplitReadTest, TestRealtimeReadConcatenatesOrderedDiskSections) TEST_P(MergeFileSplitReadTest, TestLookUp) { std::string path = paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor"; + std::map context_builder_options; ReadContextBuilder context_builder(path); std::vector raw_read_fields = {DataField(1, arrow::field("k1", arrow::int32())), @@ -753,14 +764,15 @@ TEST_P(MergeFileSplitReadTest, TestLookUp) { ASSERT_TRUE(read_schema); context_builder.SetReadFieldNames({"k1", "p1", "s1", "v0", "v1"}); - context_builder.SetOptions({{Options::SEQUENCE_FIELD, "s0,s1"}, - {Options::MERGE_ENGINE, "deduplicate"}, - {Options::IGNORE_DELETE, "true"}, - {Options::FORCE_LOOKUP, "true"}}); + context_builder_options = {{Options::SEQUENCE_FIELD, "s0,s1"}, + {Options::MERGE_ENGINE, "deduplicate"}, + {Options::IGNORE_DELETE, "true"}, + {Options::FORCE_LOOKUP, "true"}}; + AddOptions(&context_builder); ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); - auto internal_context = CreateInternalReadContext(read_context); + auto internal_context = CreateInternalReadContext(read_context, 0, context_builder_options); ASSERT_OK_AND_ASSIGN(auto batch_reader, CreateReader(internal_context, PrepareDataSplit())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result_array, @@ -790,15 +802,17 @@ TEST_P(MergeFileSplitReadTest, TestLookUp) { TEST_P(MergeFileSplitReadTest, TestReadWithLimits) { std::string path = paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor"; + std::map context_builder_options; ReadContextBuilder context_builder(path); - context_builder.SetOptions({{Options::SEQUENCE_FIELD, "s0,s1"}, - {Options::MERGE_ENGINE, "deduplicate"}, - {Options::IGNORE_DELETE, "true"}, - {Options::READ_BATCH_SIZE, "1"}}); + context_builder_options = {{Options::SEQUENCE_FIELD, "s0,s1"}, + {Options::MERGE_ENGINE, "deduplicate"}, + {Options::IGNORE_DELETE, "true"}, + {Options::READ_BATCH_SIZE, "1"}}; + AddOptions(&context_builder); ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); - auto internal_context = CreateInternalReadContext(read_context); + auto internal_context = CreateInternalReadContext(read_context, 0, context_builder_options); ASSERT_OK_AND_ASSIGN(auto batch_reader, CreateReader(internal_context, PrepareDataSplit())); @@ -816,6 +830,7 @@ TEST_P(MergeFileSplitReadTest, TestReadWithLimits) { TEST_P(MergeFileSplitReadTest, TestDeduplicateMergeEngineWithDeleteMsg) { std::string path = paimon::test::GetDataDir() + "/parquet/pk_table_partial_update.db/pk_table_partial_update"; + std::map context_builder_options; ReadContextBuilder context_builder(path); std::vector raw_read_fields = {DataField(0, arrow::field("k0", arrow::int32())), @@ -827,11 +842,12 @@ TEST_P(MergeFileSplitReadTest, TestDeduplicateMergeEngineWithDeleteMsg) { ASSERT_TRUE(read_schema); context_builder.SetReadFieldNames({"k0", "k1", "v0", "v1", "v2"}); - context_builder.SetOptions({{Options::MERGE_ENGINE, "deduplicate"}}); + context_builder_options = {{Options::MERGE_ENGINE, "deduplicate"}}; + AddOptions(&context_builder); ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); - auto internal_context = CreateInternalReadContext(read_context); + auto internal_context = CreateInternalReadContext(read_context, 0, context_builder_options); ASSERT_OK_AND_ASSIGN(auto batch_reader, CreateReader(internal_context, PrepareDataSplit2())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result_array, @@ -856,6 +872,7 @@ TEST_P(MergeFileSplitReadTest, TestDeduplicateMergeEngineWithDeleteMsg) { TEST_P(MergeFileSplitReadTest, TestReadWithPredicate) { std::string path = paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor"; + std::map context_builder_options; ReadContextBuilder context_builder(path); std::vector raw_read_fields = {DataField(1, arrow::field("k1", arrow::int32())), @@ -868,9 +885,10 @@ TEST_P(MergeFileSplitReadTest, TestReadWithPredicate) { ASSERT_TRUE(read_schema); context_builder.SetReadFieldNames({"k1", "p1", "s1", "s0", "v0", "v1"}); - context_builder.SetOptions({{Options::SEQUENCE_FIELD, "s0,s1"}, - {Options::MERGE_ENGINE, "deduplicate"}, - {Options::IGNORE_DELETE, "true"}}); + context_builder_options = {{Options::SEQUENCE_FIELD, "s0,s1"}, + {Options::MERGE_ENGINE, "deduplicate"}, + {Options::IGNORE_DELETE, "true"}}; + context_builder.EnableLateMaterializing(false); AddOptions(&context_builder); @@ -890,7 +908,7 @@ TEST_P(MergeFileSplitReadTest, TestReadWithPredicate) { context_builder.SetPredicate(predicate_result); ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); - auto internal_context = CreateInternalReadContext(read_context); + auto internal_context = CreateInternalReadContext(read_context, 0, context_builder_options); ASSERT_OK_AND_ASSIGN(auto batch_reader, CreateReader(internal_context, PrepareDataSplit())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result_array, @@ -919,6 +937,7 @@ TEST_P(MergeFileSplitReadTest, TestReadWithPredicate) { TEST_P(MergeFileSplitReadTest, TestReadWithPredicateAndLateMaterializing) { std::string path = paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor"; + std::map context_builder_options; ReadContextBuilder context_builder(path); std::vector raw_read_fields = {DataField(1, arrow::field("k1", arrow::int32())), @@ -931,9 +950,10 @@ TEST_P(MergeFileSplitReadTest, TestReadWithPredicateAndLateMaterializing) { ASSERT_TRUE(read_schema); context_builder.SetReadFieldNames({"k1", "p1", "s1", "s0", "v0", "v1"}); - context_builder.SetOptions({{Options::SEQUENCE_FIELD, "s0,s1"}, - {Options::MERGE_ENGINE, "deduplicate"}, - {Options::IGNORE_DELETE, "true"}}); + context_builder_options = {{Options::SEQUENCE_FIELD, "s0,s1"}, + {Options::MERGE_ENGINE, "deduplicate"}, + {Options::IGNORE_DELETE, "true"}}; + AddOptions(&context_builder); context_builder.EnableLateMaterializing(true); // key predicate, always pushed down into the data files @@ -948,7 +968,7 @@ TEST_P(MergeFileSplitReadTest, TestReadWithPredicateAndLateMaterializing) { context_builder.EnablePredicateFilter(true).EnableLateMaterializing(true); ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); - auto internal_context = CreateInternalReadContext(read_context); + auto internal_context = CreateInternalReadContext(read_context, 0, context_builder_options); ASSERT_OK_AND_ASSIGN(auto batch_reader, CreateReader(internal_context, PrepareDataSplit())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result_array, @@ -976,6 +996,7 @@ TEST_P(MergeFileSplitReadTest, TestReadWithPredicateAndLateMaterializing) { TEST_P(MergeFileSplitReadTest, TestReadWithAlterTable) { std::string path = paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor"; + std::map context_builder_options; ReadContextBuilder context_builder(path); std::vector raw_read_fields = {DataField(0, arrow::field("k1", arrow::int32())), @@ -991,14 +1012,16 @@ TEST_P(MergeFileSplitReadTest, TestReadWithAlterTable) { ASSERT_TRUE(read_schema); context_builder.SetReadFieldNames({"k1", "k0", "p0", "p1", "s1", "s0", "v0", "v1", "v2"}); - context_builder.SetOptions({{Options::SEQUENCE_FIELD, "s0,s1"}, - {Options::MERGE_ENGINE, "deduplicate"}, - {Options::IGNORE_DELETE, "true"}}); + context_builder_options = {{Options::SEQUENCE_FIELD, "s0,s1"}, + {Options::MERGE_ENGINE, "deduplicate"}, + {Options::IGNORE_DELETE, "true"}}; + AddOptions(&context_builder); ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); - auto internal_context = CreateInternalReadContext(read_context, /*schema_id=*/7); + auto internal_context = + CreateInternalReadContext(read_context, /*schema_id=*/7, context_builder_options); ASSERT_OK_AND_ASSIGN(auto batch_reader, CreateReader(internal_context, PrepareDataSplit())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result_array, @@ -1028,6 +1051,7 @@ TEST_P(MergeFileSplitReadTest, TestReadWithAlterTable) { TEST_P(MergeFileSplitReadTest, TestReadWithAlterTableWithReverseSequence) { std::string path = paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor"; + std::map context_builder_options; ReadContextBuilder context_builder(path); std::vector raw_read_fields = {DataField(8, arrow::field("v2", arrow::int32())), @@ -1040,14 +1064,16 @@ TEST_P(MergeFileSplitReadTest, TestReadWithAlterTableWithReverseSequence) { ASSERT_TRUE(read_schema); context_builder.SetReadFieldNames({"v2", "p1", "k0", "p0", "s0", "v0"}); - context_builder.SetOptions({{Options::SEQUENCE_FIELD, "s0,s1"}, - {Options::MERGE_ENGINE, "deduplicate"}, - {Options::IGNORE_DELETE, "true"}}); + context_builder_options = {{Options::SEQUENCE_FIELD, "s0,s1"}, + {Options::MERGE_ENGINE, "deduplicate"}, + {Options::IGNORE_DELETE, "true"}}; + AddOptions(&context_builder); ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); - auto internal_context = CreateInternalReadContext(read_context, /*schema_id=*/7); + auto internal_context = + CreateInternalReadContext(read_context, /*schema_id=*/7, context_builder_options); ASSERT_OK_AND_ASSIGN(auto batch_reader, CreateReader(internal_context, PrepareDataSplit())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result_array, @@ -1077,6 +1103,7 @@ TEST_P(MergeFileSplitReadTest, TestReadWithAlterTableWithReverseSequence) { TEST_P(MergeFileSplitReadTest, TestAggregateMergeEngine) { std::string path = paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor"; + std::map context_builder_options; ReadContextBuilder context_builder(path); std::vector raw_read_fields = {DataField(1, arrow::field("k1", arrow::int32())), @@ -1088,14 +1115,15 @@ TEST_P(MergeFileSplitReadTest, TestAggregateMergeEngine) { ASSERT_TRUE(read_schema); context_builder.SetReadFieldNames({"k1", "p1", "s1", "v0", "v1"}); - context_builder.SetOptions({{Options::SEQUENCE_FIELD, "s0,s1"}, - {Options::MERGE_ENGINE, "aggregation"}, - {"fields.v1.aggregate-function", "bool_and"}, - {"fields.v0.aggregate-function", "sum"}, - {Options::IGNORE_DELETE, "true"}}); + context_builder_options = {{Options::SEQUENCE_FIELD, "s0,s1"}, + {Options::MERGE_ENGINE, "aggregation"}, + {"fields.v1.aggregate-function", "bool_and"}, + {"fields.v0.aggregate-function", "sum"}, + {Options::IGNORE_DELETE, "true"}}; + AddOptions(&context_builder); ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); - auto internal_context = CreateInternalReadContext(read_context); + auto internal_context = CreateInternalReadContext(read_context, 0, context_builder_options); ASSERT_OK_AND_ASSIGN(auto batch_reader, CreateReader(internal_context, PrepareDataSplit())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result_array, @@ -1125,6 +1153,7 @@ TEST_P(MergeFileSplitReadTest, TestAggregateMergeEngine) { TEST_P(MergeFileSplitReadTest, TestPartialUpdateMergeEngine) { std::string path = paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor"; + std::map context_builder_options; ReadContextBuilder context_builder(path); std::vector raw_read_fields = {DataField(1, arrow::field("k1", arrow::int32())), @@ -1135,14 +1164,15 @@ TEST_P(MergeFileSplitReadTest, TestPartialUpdateMergeEngine) { ASSERT_TRUE(read_schema); context_builder.SetReadFieldNames({"k1", "p1", "s1", "v0"}); - context_builder.SetOptions({{Options::SEQUENCE_FIELD, "s0,s1"}, - {Options::MERGE_ENGINE, "partial-update"}, - {"fields.v1.sequence-group", "v0"}, - {"fields.v0.aggregate-function", "first_value"}, - {Options::IGNORE_DELETE, "true"}}); + context_builder_options = {{Options::SEQUENCE_FIELD, "s0,s1"}, + {Options::MERGE_ENGINE, "partial-update"}, + {"fields.v1.sequence-group", "v0"}, + {"fields.v0.aggregate-function", "first_value"}, + {Options::IGNORE_DELETE, "true"}}; + AddOptions(&context_builder); ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); - auto internal_context = CreateInternalReadContext(read_context); + auto internal_context = CreateInternalReadContext(read_context, 0, context_builder_options); ASSERT_OK_AND_ASSIGN(auto batch_reader, CreateReader(internal_context, PrepareDataSplit())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result_array, @@ -1172,6 +1202,7 @@ TEST_P(MergeFileSplitReadTest, TestPartialUpdateMergeEngine) { TEST_P(MergeFileSplitReadTest, TestPartialUpdateMergeEngineWithIgnoreDelete) { std::string path = paimon::test::GetDataDir() + "/parquet/pk_table_partial_update.db/pk_table_partial_update"; + std::map context_builder_options; ReadContextBuilder context_builder(path); std::vector raw_read_fields = {DataField(0, arrow::field("k0", arrow::int32())), @@ -1183,11 +1214,12 @@ TEST_P(MergeFileSplitReadTest, TestPartialUpdateMergeEngineWithIgnoreDelete) { ASSERT_TRUE(read_schema); context_builder.SetReadFieldNames({"k0", "k1", "v0", "v1", "v2"}); - context_builder.SetOptions( - {{Options::MERGE_ENGINE, "partial-update"}, {Options::IGNORE_DELETE, "true"}}); + context_builder_options = {{Options::MERGE_ENGINE, "partial-update"}, + {Options::IGNORE_DELETE, "true"}}; + AddOptions(&context_builder); ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); - auto internal_context = CreateInternalReadContext(read_context); + auto internal_context = CreateInternalReadContext(read_context, 0, context_builder_options); ASSERT_OK_AND_ASSIGN(auto batch_reader, CreateReader(internal_context, PrepareDataSplit2())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result_array, @@ -1212,6 +1244,7 @@ TEST_P(MergeFileSplitReadTest, TestPartialUpdateMergeEngineWithIgnoreDelete) { TEST_P(MergeFileSplitReadTest, TestPartialUpdateMergeEngineWithRemoveRecordOnDelete) { std::string path = paimon::test::GetDataDir() + "/parquet/pk_table_partial_update.db/pk_table_partial_update"; + std::map context_builder_options; ReadContextBuilder context_builder(path); std::vector raw_read_fields = {DataField(0, arrow::field("k0", arrow::int32())), @@ -1223,11 +1256,12 @@ TEST_P(MergeFileSplitReadTest, TestPartialUpdateMergeEngineWithRemoveRecordOnDel ASSERT_TRUE(read_schema); context_builder.SetReadFieldNames({"k0", "k1", "v0", "v1", "v2"}); - context_builder.SetOptions({{Options::MERGE_ENGINE, "partial-update"}, - {Options::PARTIAL_UPDATE_REMOVE_RECORD_ON_DELETE, "true"}}); + context_builder_options = {{Options::MERGE_ENGINE, "partial-update"}, + {Options::PARTIAL_UPDATE_REMOVE_RECORD_ON_DELETE, "true"}}; + AddOptions(&context_builder); ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); - auto internal_context = CreateInternalReadContext(read_context); + auto internal_context = CreateInternalReadContext(read_context, 0, context_builder_options); ASSERT_OK_AND_ASSIGN(auto batch_reader, CreateReader(internal_context, PrepareDataSplit2())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result_array, @@ -1252,6 +1286,7 @@ TEST_P(MergeFileSplitReadTest, TestPartialUpdateMergeEngineWithRemoveRecordOnDel TEST_P(MergeFileSplitReadTest, TestEmptyPlan) { std::string path = paimon::test::GetDataDir() + "/parquet/pk_table_partial_update.db/pk_table_partial_update"; + std::map context_builder_options; ReadContextBuilder context_builder(path); std::vector raw_read_fields = {DataField(0, arrow::field("k0", arrow::int32())), @@ -1263,11 +1298,12 @@ TEST_P(MergeFileSplitReadTest, TestEmptyPlan) { ASSERT_TRUE(read_schema); context_builder.SetReadFieldNames({"k0", "k1", "v0", "v1", "v2"}); - context_builder.SetOptions({{Options::MERGE_ENGINE, "partial-update"}, - {Options::PARTIAL_UPDATE_REMOVE_RECORD_ON_DELETE, "true"}}); + context_builder_options = {{Options::MERGE_ENGINE, "partial-update"}, + {Options::PARTIAL_UPDATE_REMOVE_RECORD_ON_DELETE, "true"}}; + AddOptions(&context_builder); ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); - auto internal_context = CreateInternalReadContext(read_context); + auto internal_context = CreateInternalReadContext(read_context, 0, context_builder_options); std::vector> empty_data_split; ASSERT_OK_AND_ASSIGN(auto batch_reader, CreateReader(internal_context, empty_data_split)); ASSERT_OK_AND_ASSIGN(std::shared_ptr read_result, @@ -1285,18 +1321,20 @@ TEST_P(MergeFileSplitReadTest, TestIOException) { DataField(6, arrow::field("v0", arrow::float64())), DataField(7, arrow::field("v1", arrow::boolean()))}; + std::map context_builder_options; ReadContextBuilder context_builder(path); auto read_schema = DataField::ConvertDataFieldsToArrowSchema(raw_read_fields); ASSERT_TRUE(read_schema); context_builder.SetReadFieldNames({"k1", "p1", "s1", "v0", "v1"}); - context_builder.SetOptions({{Options::SEQUENCE_FIELD, "s0,s1"}, - {Options::MERGE_ENGINE, "deduplicate"}, - {Options::IGNORE_DELETE, "true"}}); + context_builder_options = {{Options::SEQUENCE_FIELD, "s0,s1"}, + {Options::MERGE_ENGINE, "deduplicate"}, + {Options::IGNORE_DELETE, "true"}}; + AddOptions(&context_builder); ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); - auto internal_context = CreateInternalReadContext(read_context); + auto internal_context = CreateInternalReadContext(read_context, 0, context_builder_options); auto fields_with_row_kind = read_schema->fields(); fields_with_row_kind.insert(fields_with_row_kind.begin(), @@ -1336,6 +1374,7 @@ TEST_P(MergeFileSplitReadTest, TestIOException) { TEST_P(MergeFileSplitReadTest, Test09VersionWithoutInlineFieldId) { std::string path = paimon::test::GetDataDir() + "/orc/pk_09.db/pk_09"; + std::map context_builder_options; ReadContextBuilder context_builder(path); std::vector raw_read_fields = {DataField(3, arrow::field("f3", arrow::float64())), DataField(2, arrow::field("f2", arrow::int32())), @@ -1344,9 +1383,10 @@ TEST_P(MergeFileSplitReadTest, Test09VersionWithoutInlineFieldId) { auto read_schema = DataField::ConvertDataFieldsToArrowSchema(raw_read_fields); ASSERT_TRUE(read_schema); context_builder.SetReadFieldNames({"f3", "f2", "f0", "f1"}); - context_builder.SetOptions({{Options::FILE_FORMAT, "orc"}, - {Options::MERGE_ENGINE, "deduplicate"}, - {"orc.read.enable-metrics", "true"}}); + context_builder_options = {{Options::FILE_FORMAT, "orc"}, + {Options::MERGE_ENGINE, "deduplicate"}, + {"orc.read.enable-metrics", "true"}}; + AddOptions(&context_builder); ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); auto meta1 = std::make_shared( @@ -1391,7 +1431,7 @@ TEST_P(MergeFileSplitReadTest, Test09VersionWithoutInlineFieldId) { paimon::test::GetDataDir() + "/orc/pk_09.db/pk_09/f1=10/bucket-1/", {meta1, meta2}); ASSERT_OK_AND_ASSIGN(auto data_split, builder.WithSnapshot(5).IsStreaming(false).RawConvertible(false).Build()); - auto internal_context = CreateInternalReadContext(read_context); + auto internal_context = CreateInternalReadContext(read_context, 0, context_builder_options); ASSERT_OK_AND_ASSIGN(auto batch_reader, CreateReader(internal_context, {data_split})); ASSERT_OK_AND_ASSIGN(std::shared_ptr result_array, ReadResultCollector::CollectResult(batch_reader.get())); diff --git a/src/paimon/core/operation/raw_file_split_read_test.cpp b/src/paimon/core/operation/raw_file_split_read_test.cpp index 77f92109c..b2b8f6f92 100644 --- a/src/paimon/core/operation/raw_file_split_read_test.cpp +++ b/src/paimon/core/operation/raw_file_split_read_test.cpp @@ -145,14 +145,18 @@ class RawFileSplitReadTest : public ::testing::Test { Result> ReadSearchWithDeletionVector( const std::shared_ptr& read_context, const std::string& index_type, - const RoaringBitmap32& deleted_rows, std::shared_ptr table_schema) const { + const RoaringBitmap32& deleted_rows, std::shared_ptr table_schema, + const std::map& dynamic_options) const { SchemaManager schema_manager(std::make_shared(), read_context->GetPath()); if (!table_schema) { PAIMON_ASSIGN_OR_RAISE(table_schema, schema_manager.ReadSchema(0)); } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr effective_schema, + table_schema->MergeOptions(dynamic_options)); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr internal_context, - InternalReadContext::Create(read_context, table_schema, table_schema->Options())); + InternalReadContext::Create(read_context, table_schema, effective_schema->Options(), + dynamic_options)); auto split = std::dynamic_pointer_cast(PrepareDataSplits().front()); if (!split) { return Status::Invalid("Raw search test split must be a DataSplitImpl"); @@ -197,10 +201,12 @@ class RawFileSplitReadTest : public ::testing::Test { std::string path = paimon::test::GetDataDir() + "/orc/multi_partition_append_table.db/" "multi_partition_append_table"; + std::map context_builder_options; ReadContextBuilder context_builder(path); context_builder.SetReadFieldNames(read_schema->field_names()); if (dv_factory) { - context_builder.AddOption(Options::FILE_INDEX_READ_ENABLED, "true"); + context_builder_options[Options::FILE_INDEX_READ_ENABLED] = "true"; + context_builder.SetPredicate(PredicateBuilder::IsNotNull(0, "f0", FieldType::STRING)); } ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); @@ -209,7 +215,8 @@ class RawFileSplitReadTest : public ::testing::Test { ASSERT_OK_AND_ASSIGN( auto internal_context, - InternalReadContext::Create(read_context, table_schema, table_schema->Options())); + InternalReadContext::Create(read_context, table_schema, table_schema->Options(), + context_builder_options)); auto data_splits = PrepareDataSplits(); const auto& core_options = internal_context->GetCoreOptions(); auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); @@ -468,7 +475,7 @@ TEST_F(RawFileSplitReadTest, TestVectorSearchScoreProjection) { ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); ASSERT_OK_AND_ASSIGN( auto internal_context, - InternalReadContext::Create(read_context, table_schema, table_schema->Options())); + InternalReadContext::Create(read_context, table_schema, table_schema->Options(), {})); auto data_splits = PrepareDataSplits(); auto data_split = std::dynamic_pointer_cast(data_splits.front()); @@ -514,9 +521,9 @@ TEST_F(RawFileSplitReadTest, TestVectorSearchScoreProjection) { without_score_builder.SetReadFieldNames({"f0"}).SetVectorSearch(vector_search); ASSERT_OK_AND_ASSIGN(std::shared_ptr without_score_context, without_score_builder.Finish()); - ASSERT_OK_AND_ASSIGN( - auto without_score_internal_context, - InternalReadContext::Create(without_score_context, table_schema, table_schema->Options())); + ASSERT_OK_AND_ASSIGN(auto without_score_internal_context, + InternalReadContext::Create(without_score_context, table_schema, + table_schema->Options(), {})); auto without_score_split_read = std::make_unique( path_factory, std::move(without_score_internal_context), pool_, executor); ASSERT_OK_AND_ASSIGN(std::unique_ptr without_score_reader, @@ -545,7 +552,7 @@ TEST_F(RawFileSplitReadTest, TestVectorSearchFiltersDeletionVectorBeforeTopK) { ASSERT_OK_AND_ASSIGN( std::shared_ptr result, ReadSearchWithDeletionVector(read_context, MockSearchFileIndex::VECTOR_IDENTIFIER, - RoaringBitmap32::From({0}), /*table_schema=*/nullptr)); + RoaringBitmap32::From({0}), /*table_schema=*/nullptr, {})); std::shared_ptr expected; ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON( arrow::struct_({arrow::field("_VALUE_KIND", arrow::int8()), @@ -570,7 +577,7 @@ TEST_F(RawFileSplitReadTest, TestFullTextSearchFiltersDeletionVectorBeforeLimit) ASSERT_OK_AND_ASSIGN( std::shared_ptr result, ReadSearchWithDeletionVector(read_context, MockSearchFileIndex::FULL_TEXT_IDENTIFIER, - RoaringBitmap32::From({0}), /*table_schema=*/nullptr)); + RoaringBitmap32::From({0}), /*table_schema=*/nullptr, {})); std::shared_ptr expected; ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON( arrow::struct_({arrow::field("_VALUE_KIND", arrow::int8()), @@ -609,8 +616,10 @@ TEST_F(RawFileSplitReadTest, TestVectorSearchRenamedUnprojectedFieldInOldFile) { auto search = std::make_shared("renamed_f3", 2, std::vector{1.0f}, nullptr, nullptr, VectorSearch::DistanceType::EUCLIDEAN, std::map{}); + std::map builder_options; ReadContextBuilder builder(path); - builder.SetOptions(options).SetReadFieldNames({"f0", "_INDEX_SCORE"}).SetVectorSearch(search); + builder_options = options; + builder.SetReadFieldNames({"f0", "_INDEX_SCORE"}).SetVectorSearch(search); ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); auto expected_type = arrow::struct_({arrow::field("_VALUE_KIND", arrow::int8()), @@ -621,8 +630,9 @@ TEST_F(RawFileSplitReadTest, TestVectorSearchRenamedUnprojectedFieldInOldFile) { {RoaringBitmap32::From({0}), R"([[0,"Tony",0.75]])"}}; for (const auto& [deleted, expected_json] : cases) { SCOPED_TRACE(deleted.ToString()); - ASSERT_OK_AND_ASSIGN(auto actual, ReadSearchWithDeletionVector(context, identifier, deleted, - current_schema)); + ASSERT_OK_AND_ASSIGN( + auto actual, ReadSearchWithDeletionVector(context, identifier, deleted, current_schema, + builder_options)); auto expected_result = arrow::ipc::internal::json::ArrayFromJSON(expected_type, expected_json); ASSERT_TRUE(expected_result.ok()) << expected_result.status().ToString(); @@ -638,13 +648,14 @@ TEST_F(RawFileSplitReadTest, TestVectorSearchRenamedUnprojectedFieldInOldFile) { // The old file's f3 is field ID 3; the current table's f3 is a different field with ID 4. auto missing_search = std::make_shared(*search); missing_search->field_name = "f3"; + std::map missing_field_builder_options; ReadContextBuilder missing_field_builder(path); - missing_field_builder.SetOptions(options).SetReadFieldNames({"f0"}).SetVectorSearch( - missing_search); + missing_field_builder_options = options; + missing_field_builder.SetReadFieldNames({"f0"}).SetVectorSearch(missing_search); ASSERT_OK_AND_ASSIGN(std::shared_ptr missing_context, missing_field_builder.Finish()); ASSERT_NOK_WITH_MSG(ReadSearchWithDeletionVector(missing_context, identifier, RoaringBitmap32(), - current_schema), + current_schema, missing_field_builder_options), "has no matching field ID"); } @@ -676,8 +687,10 @@ TEST_F(RawFileSplitReadTest, TestFullTextSearchRenamedUnprojectedFieldInOldFile) original_schema->PartitionKeys(), {}, options)); auto search = std::make_shared("renamed_f3", R"({"match":{"query":"match"}})", 2); + std::map builder_options; ReadContextBuilder builder(path); - builder.SetOptions(options).SetReadFieldNames({"f0"}).SetFullTextSearch(search); + builder_options = options; + builder.SetReadFieldNames({"f0"}).SetFullTextSearch(search); ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); auto expected_type = arrow::struct_( @@ -687,8 +700,9 @@ TEST_F(RawFileSplitReadTest, TestFullTextSearchRenamedUnprojectedFieldInOldFile) {RoaringBitmap32::From({0}), R"([[0,"Tony"]])"}}; for (const auto& [deleted, expected_json] : cases) { SCOPED_TRACE(deleted.ToString()); - ASSERT_OK_AND_ASSIGN(auto actual, ReadSearchWithDeletionVector(context, identifier, deleted, - current_schema)); + ASSERT_OK_AND_ASSIGN( + auto actual, ReadSearchWithDeletionVector(context, identifier, deleted, current_schema, + builder_options)); auto expected_result = arrow::ipc::internal::json::ArrayFromJSON(expected_type, expected_json); ASSERT_TRUE(expected_result.ok()) << expected_result.status().ToString(); @@ -704,13 +718,14 @@ TEST_F(RawFileSplitReadTest, TestFullTextSearchRenamedUnprojectedFieldInOldFile) // The old file's f3 is field ID 3; the current table's f3 is a different field with ID 4. auto missing_search = std::make_shared(*search); missing_search->field_name = "f3"; + std::map missing_field_builder_options; ReadContextBuilder missing_field_builder(path); - missing_field_builder.SetOptions(options).SetReadFieldNames({"f0"}).SetFullTextSearch( - missing_search); + missing_field_builder_options = options; + missing_field_builder.SetReadFieldNames({"f0"}).SetFullTextSearch(missing_search); ASSERT_OK_AND_ASSIGN(std::shared_ptr missing_context, missing_field_builder.Finish()); ASSERT_NOK_WITH_MSG(ReadSearchWithDeletionVector(missing_context, identifier, RoaringBitmap32(), - current_schema), + current_schema, missing_field_builder_options), "has no matching field ID"); } @@ -725,7 +740,7 @@ TEST_F(RawFileSplitReadTest, TestEmptyPlan) { ASSERT_OK_AND_ASSIGN( auto internal_context, - InternalReadContext::Create(read_context, table_schema, table_schema->Options())); + InternalReadContext::Create(read_context, table_schema, table_schema->Options(), {})); const auto& core_options = internal_context->GetCoreOptions(); auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); ASSERT_OK_AND_ASSIGN(std::vector external_paths, @@ -775,7 +790,7 @@ TEST_F(RawFileSplitReadTest, TestMatch) { ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); ASSERT_OK_AND_ASSIGN( std::shared_ptr internal_context, - InternalReadContext::Create(read_context, table_schema, table_schema->Options())); + InternalReadContext::Create(read_context, table_schema, table_schema->Options(), {})); ASSERT_OK_AND_ASSIGN(std::shared_ptr executor, CreateDefaultExecutor(/*thread_count=*/2)); auto split_read = std::make_unique( diff --git a/src/paimon/core/operation/read_context.cpp b/src/paimon/core/operation/read_context.cpp index 40ed358de..3a24acbbd 100644 --- a/src/paimon/core/operation/read_context.cpp +++ b/src/paimon/core/operation/read_context.cpp @@ -37,9 +37,9 @@ namespace paimon { class Predicate; ReadContext::ReadContext( - const std::string& path, const std::string& branch, - const std::vector& read_field_names, const std::vector& read_field_ids, - const std::shared_ptr& predicate, const std::shared_ptr& vector_search, + const std::string& path, const std::vector& read_field_names, + const std::vector& read_field_ids, const std::shared_ptr& predicate, + const std::shared_ptr& vector_search, const std::shared_ptr& full_text_search, bool enable_predicate_filter, bool enable_prefetch, bool enable_late_materializing, uint32_t prefetch_batch_count, uint32_t prefetch_max_parallel_num, bool enable_multi_thread_row_to_batch, @@ -47,12 +47,10 @@ ReadContext::ReadContext( const std::shared_ptr& memory_pool, const std::shared_ptr& executor, const std::shared_ptr& specific_file_system, const std::map& fs_scheme_to_identifier_map, - const std::shared_ptr& realtime_context, - const std::map& options, bool read_ahead_cache_enabled, + const std::shared_ptr& realtime_context, bool read_ahead_cache_enabled, const CacheConfig& cache_config, const std::shared_ptr& cache, WarmupLevel warmup_level, const std::shared_ptr& catalog, const std::optional& identifier) : path_(path), - branch_(branch), read_field_names_(read_field_names), read_field_ids_(read_field_ids), predicate_(predicate), @@ -71,7 +69,6 @@ ReadContext::ReadContext( specific_file_system_(specific_file_system), fs_scheme_to_identifier_map_(fs_scheme_to_identifier_map), realtime_context_(realtime_context), - options_(options), read_ahead_cache_enabled_(read_ahead_cache_enabled), cache_config_(cache_config), cache_(cache), @@ -101,12 +98,10 @@ class ReadContextBuilder::Impl { public: friend class ReadContextBuilder; void Reset() { - branch_.reset(); read_field_names_.clear(); read_field_ids_.clear(); read_schema_.reset(); fs_scheme_to_identifier_map_.clear(); - options_.clear(); predicate_.reset(); vector_search_.reset(); full_text_search_.reset(); @@ -132,14 +127,10 @@ class ReadContextBuilder::Impl { private: std::string path_; - /// Unset until `WithBranch()` names one, so that a branch named only by the `branch` - /// option is not overruled by the default. - std::optional branch_; std::vector read_field_names_; std::vector read_field_ids_; std::unique_ptr read_schema_; std::map fs_scheme_to_identifier_map_; - std::map options_; std::shared_ptr predicate_; std::shared_ptr vector_search_; std::shared_ptr full_text_search_; @@ -173,17 +164,6 @@ ReadContextBuilder::~ReadContextBuilder() = default; ReadContextBuilder::ReadContextBuilder(ReadContextBuilder&&) noexcept = default; ReadContextBuilder& ReadContextBuilder::operator=(ReadContextBuilder&&) noexcept = default; -ReadContextBuilder& ReadContextBuilder::AddOption(const std::string& key, - const std::string& value) { - impl_->options_[key] = value; - return *this; -} - -ReadContextBuilder& ReadContextBuilder::SetOptions(const std::map& opts) { - impl_->options_ = opts; - return *this; -} - ReadContextBuilder& ReadContextBuilder::SetReadFieldNames( const std::vector& read_field_names) { impl_->read_field_names_ = read_field_names; @@ -277,11 +257,6 @@ ReadContextBuilder& ReadContextBuilder::SetTableSchema(const std::string& table_ return *this; } -ReadContextBuilder& ReadContextBuilder::WithBranch(const std::string& branch) { - impl_->branch_ = branch; - return *this; -} - ReadContextBuilder& ReadContextBuilder::WithFileSystemSchemeToIdentifierMap( const std::map& fs_scheme_to_identifier_map) { impl_->fs_scheme_to_identifier_map_ = fs_scheme_to_identifier_map; @@ -356,9 +331,6 @@ Result> ReadContextBuilder::Finish() { PAIMON_ASSIGN_OR_RAISE(impl_->specific_file_system_, impl_->catalog_->GetTableFileSystem(impl_->identifier_.value())); } - PAIMON_ASSIGN_OR_RAISE( - std::string branch, - BranchManager::ResolveBranch(impl_->identifier_, impl_->options_, impl_->branch_, "read")); if (impl_->enable_prefetch_ && impl_->prefetch_max_parallel_num_ == 0) { return Status::Invalid("prefetch max parallel num should be greater than 0"); } @@ -385,15 +357,14 @@ Result> ReadContextBuilder::Finish() { } auto ctx = std::make_unique( - impl_->path_, branch, impl_->read_field_names_, impl_->read_field_ids_, impl_->predicate_, + impl_->path_, impl_->read_field_names_, impl_->read_field_ids_, impl_->predicate_, impl_->vector_search_, impl_->full_text_search_, impl_->enable_predicate_filter_, impl_->enable_prefetch_, impl_->enable_late_materializing_, impl_->prefetch_batch_count_, impl_->prefetch_max_parallel_num_, impl_->enable_multi_thread_row_to_batch_, impl_->row_to_batch_thread_number_, impl_->table_schema_, impl_->memory_pool_, impl_->executor_, impl_->specific_file_system_, impl_->fs_scheme_to_identifier_map_, - impl_->realtime_context_, impl_->options_, impl_->read_ahead_cache_enabled_, - impl_->cache_config_, impl_->cache_, impl_->warmup_level_, impl_->catalog_, - impl_->identifier_); + impl_->realtime_context_, impl_->read_ahead_cache_enabled_, impl_->cache_config_, + impl_->cache_, impl_->warmup_level_, impl_->catalog_, impl_->identifier_); if (impl_->read_schema_ && impl_->read_schema_->release) { ctx->SetReadSchema(std::move(impl_->read_schema_)); } diff --git a/src/paimon/core/operation/read_context_test.cpp b/src/paimon/core/operation/read_context_test.cpp index fd1b1ebc5..a2845ef36 100644 --- a/src/paimon/core/operation/read_context_test.cpp +++ b/src/paimon/core/operation/read_context_test.cpp @@ -85,33 +85,6 @@ TEST(ReadContextTest, TestWithCatalogNullRejected) { "cannot read through a null catalog"); } -TEST(ReadContextTest, TestWithCatalogKeepsReadOptionsSeparate) { - auto catalog = std::make_shared(); - catalog->SetTableFileSystem(std::make_shared()); - catalog->SetOptions({{CatalogOptions::METASTORE, "rest"}, - {CatalogOptions::URI, "http://catalog"}, - {Options::BLOB_VIEW_RESOLVE_ENABLED, "false"}}); - - ReadContextBuilder builder("table_root_path"); - ASSERT_OK_AND_ASSIGN(auto ctx, builder.WithCatalog(catalog, Identifier("db1", "t1")).Finish()); - // Finishing a read neither imports catalog defaults nor generates a dependency header. - ASSERT_TRUE(ctx->GetOptions().empty()); - - catalog->SetOptions({{CatalogOptions::METASTORE, "rest"}, - {CatalogOptions::URI, "http://catalog"}, - {"header.X-Paimon-Read-Via", "catalog-view"}}); - ASSERT_OK_AND_ASSIGN(auto nested_ctx, - builder.WithCatalog(catalog, Identifier("db1", "t1")).Finish()); - ASSERT_TRUE(nested_ctx->GetOptions().empty()); - const std::map read_options = { - {Options::BLOB_VIEW_RESOLVE_ENABLED, "true"}, - {"header.X-Paimon-Read-Via", "explicit-view"}}; - ASSERT_OK_AND_ASSIGN( - auto explicit_ctx, - builder.WithCatalog(catalog, Identifier("db1", "t1")).SetOptions(read_options).Finish()); - ASSERT_EQ(explicit_ctx->GetOptions(), read_options); -} - TEST(ReadContextTest, TestDefaultValue) { ReadContextBuilder builder("table_root_path"); ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish()); @@ -121,7 +94,6 @@ TEST(ReadContextTest, TestDefaultValue) { ASSERT_TRUE(ctx->GetExecutor()); ASSERT_TRUE(ctx->GetReadFieldNames().empty()); ASSERT_TRUE(ctx->GetReadFieldIds().empty()); - ASSERT_TRUE(ctx->GetOptions().empty()); ASSERT_FALSE(ctx->GetPredicate()); ASSERT_FALSE(ctx->HasVectorOrFullTextSearch()); ASSERT_FALSE(ctx->EnablePredicateFilter()); @@ -132,7 +104,6 @@ TEST(ReadContextTest, TestDefaultValue) { ASSERT_EQ(3, ctx->GetPrefetchMaxParallelNum()); ASSERT_FALSE(ctx->EnableMultiThreadRowToBatch()); ASSERT_EQ(1, ctx->GetRowToBatchThreadNumber()); - ASSERT_EQ("main", ctx->GetBranch()); ASSERT_TRUE(ctx->GetFileSystemSchemeToIdentifierMap().empty()); ASSERT_FALSE(ctx->GetSpecificFileSystem()); } @@ -146,7 +117,6 @@ TEST(ReadContextTest, TestSetContent) { cache_config.SetHoleSizeLimit(128); cache_config.SetPreBufferLimit(2048); - builder.AddOption("key", "value"); builder.SetReadFieldNames({"f1", "f2"}); builder.SetReadFieldIds({0, 1}); auto predicate = @@ -163,7 +133,7 @@ TEST(ReadContextTest, TestSetContent) { builder.WithMemoryPool(memory_pool); builder.WithExecutor(executor); builder.SetTableSchema("table-schema-json"); - builder.WithBranch("rt"); + builder.WithCacheConfig(cache_config); auto fs = std::make_shared(); builder.WithFileSystem(fs); @@ -190,13 +160,11 @@ TEST(ReadContextTest, TestSetContent) { ASSERT_EQ(executor, ctx->GetExecutor()); ASSERT_TRUE(ctx->GetSpecificTableSchema().has_value()); ASSERT_EQ("table-schema-json", ctx->GetSpecificTableSchema().value()); - ASSERT_EQ("rt", ctx->GetBranch()); ASSERT_EQ(512U, ctx->GetCacheConfig().GetRangeSizeLimit()); ASSERT_EQ(128U, ctx->GetCacheConfig().GetHoleSizeLimit()); ASSERT_EQ(2048U, ctx->GetCacheConfig().GetPreBufferLimit()); ASSERT_TRUE(ctx->GetFileSystemSchemeToIdentifierMap().empty()); std::map expected_options = {{"key", "value"}}; - ASSERT_EQ(expected_options, ctx->GetOptions()); ASSERT_EQ(ctx->GetSpecificFileSystem(), fs); ASSERT_TRUE(ctx->GetCache()); } @@ -219,73 +187,6 @@ TEST(ReadContextTest, TestSetWarmupLevel) { ASSERT_EQ(WarmupLevel::RAW, second_ctx->GetWarmupLevel()); } -TEST(ReadContextTest, TestSetOptionsOverridesAddedOptions) { - ReadContextBuilder builder("table_root_path"); - builder.AddOption("old", "value"); - builder.SetOptions({{"key1", "value1"}, {"key2", "value2"}}); - - ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish()); - - std::map expected_options = {{"key1", "value1"}, {"key2", "value2"}}; - ASSERT_EQ(expected_options, ctx->GetOptions()); -} - -TEST(ReadContextTest, TestBranch) { - ReadContextBuilder option_builder("table_root_path"); - ASSERT_OK_AND_ASSIGN(auto option_ctx, option_builder.AddOption(Options::BRANCH, "rt").Finish()); - ASSERT_EQ("rt", option_ctx->GetBranch()); - - ReadContextBuilder agreeing_builder("table_root_path"); - ASSERT_OK_AND_ASSIGN( - auto agreeing_ctx, - agreeing_builder.WithBranch("rt").AddOption(Options::BRANCH, "rt").Finish()); - ASSERT_EQ("rt", agreeing_ctx->GetBranch()); - - ReadContextBuilder mixed_builder("table_root_path"); - ASSERT_NOK_WITH_MSG(mixed_builder.WithBranch("rt").AddOption(Options::BRANCH, "dev").Finish(), - "but both 'dev' and 'rt' were named"); - - // An empty branch selects the main branch and stays accepted. - ReadContextBuilder main_builder("table_root_path"); - main_builder.WithBranch(""); - ASSERT_OK_AND_ASSIGN(auto ctx, main_builder.Finish()); - ASSERT_EQ(BranchManager::DEFAULT_MAIN_BRANCH, ctx->GetBranch()); - ReadContextBuilder empty_builder("table_root_path"); - ASSERT_NOK_WITH_MSG(empty_builder.WithBranch("").AddOption(Options::BRANCH, "dev").Finish(), - "but both 'dev' and 'main' were named"); - - // The branch names a directory under the table path, so a value that is not a single path - // component is rejected when the context is built. - ReadContextBuilder escaping_builder("table_root_path"); - escaping_builder.WithBranch("rt/../../../../../outside"); - ASSERT_NOK_WITH_MSG(escaping_builder.Finish(), "branch name cannot contain path separators"); -} - -TEST(ReadContextTest, TestBranchAgreesWithIdentifier) { - // A read through a catalog names its table with an Identifier that can itself carry a branch, - // so that branch and the one WithBranch() names must agree, exactly as the write and commit - // builders already require: one read cannot be aimed at two branches at once. - auto catalog = std::make_shared(); - catalog->SetTableFileSystem(std::make_shared()); - - // The identifier's branch is honored when WithBranch() stays silent. - ReadContextBuilder identifier_builder("table_root_path"); - identifier_builder.WithCatalog(catalog, Identifier("db1", "t1", "dev")); - ASSERT_OK_AND_ASSIGN(auto identifier_ctx, identifier_builder.Finish()); - ASSERT_EQ("dev", identifier_ctx->GetBranch()); - - // An agreeing WithBranch() is accepted. - ReadContextBuilder agreeing_builder("table_root_path"); - agreeing_builder.WithCatalog(catalog, Identifier("db1", "t1", "dev")).WithBranch("dev"); - ASSERT_OK_AND_ASSIGN(auto agreeing_ctx, agreeing_builder.Finish()); - ASSERT_EQ("dev", agreeing_ctx->GetBranch()); - - // A disagreeing WithBranch() is refused, and both branches are named. - ReadContextBuilder mixed_builder("table_root_path"); - mixed_builder.WithCatalog(catalog, Identifier("db1", "t1", "dev")).WithBranch("rt"); - ASSERT_NOK_WITH_MSG(mixed_builder.Finish(), "but both 'dev' and 'rt' were named"); -} - TEST(ReadContextTest, TestFileSystemAndSchemeMapConflict) { ReadContextBuilder builder("table_root_path"); auto fs = std::make_shared(); diff --git a/src/paimon/core/operation/scan_context.cpp b/src/paimon/core/operation/scan_context.cpp index 5d1c7610c..e8a95582f 100644 --- a/src/paimon/core/operation/scan_context.cpp +++ b/src/paimon/core/operation/scan_context.cpp @@ -30,22 +30,17 @@ namespace paimon { class Predicate; -ScanContext::ScanContext(const std::string& path, bool is_streaming_mode, - std::optional limit, - const std::shared_ptr& scan_filter, - const std::shared_ptr& global_index_result, - const std::shared_ptr& realtime_context, - const std::shared_ptr& memory_pool, - const std::shared_ptr& executor, - const std::shared_ptr& specific_file_system, - const std::optional& table_schema, - const std::map& options, - const std::shared_ptr& cache, - const std::shared_ptr& catalog, - const std::optional& identifier, const std::string& branch, - const std::map& fs_scheme_to_identifier_map) +ScanContext::ScanContext( + const std::string& path, bool is_streaming_mode, std::optional limit, + const std::shared_ptr& scan_filter, + const std::shared_ptr& global_index_result, + const std::shared_ptr& realtime_context, + const std::shared_ptr& memory_pool, const std::shared_ptr& executor, + const std::shared_ptr& specific_file_system, + const std::optional& table_schema, const std::shared_ptr& cache, + const std::shared_ptr& catalog, const std::optional& identifier, + const std::map& fs_scheme_to_identifier_map) : path_(path), - branch_(branch), is_streaming_mode_(is_streaming_mode), limit_(limit), scan_filters_(scan_filter), @@ -56,7 +51,6 @@ ScanContext::ScanContext(const std::string& path, bool is_streaming_mode, specific_file_system_(specific_file_system), fs_scheme_to_identifier_map_(fs_scheme_to_identifier_map), table_schema_(table_schema), - options_(options), cache_(cache), catalog_(catalog), identifier_(identifier) {} @@ -80,7 +74,6 @@ class ScanContextBuilder::Impl { specific_file_system_.reset(); fs_scheme_to_identifier_map_.clear(); table_schema_ = std::nullopt; - options_.clear(); cache_.reset(); catalog_.reset(); identifier_.reset(); @@ -101,7 +94,6 @@ class ScanContextBuilder::Impl { std::shared_ptr specific_file_system_; std::map fs_scheme_to_identifier_map_; std::optional table_schema_; - std::map options_; std::shared_ptr cache_; std::shared_ptr catalog_; std::optional identifier_; @@ -151,18 +143,6 @@ ScanContextBuilder& ScanContextBuilder::WithRealtimeContext( return *this; } -ScanContextBuilder& ScanContextBuilder::SetOptions( - const std::map& options) { - impl_->options_ = options; - return *this; -} - -ScanContextBuilder& ScanContextBuilder::AddOption(const std::string& key, - const std::string& value) { - impl_->options_[key] = value; - return *this; -} - ScanContextBuilder& ScanContextBuilder::WithMemoryPool( const std::shared_ptr& memory_pool) { impl_->memory_pool_ = memory_pool; @@ -217,15 +197,10 @@ Result> ScanContextBuilder::Finish() { } if (impl_->catalog_ != nullptr && impl_->specific_file_system_ == nullptr && impl_->fs_scheme_to_identifier_map_.empty()) { - // A catalog issuing per-table temporary credentials only hands them out through - // GetTableFileSystem, so the scan uses the credentials of this table. A scheme map - // routes by scheme instead, so the catalog is not asked when one is set. + // A scheme map takes precedence over catalog-issued per-table credentials. PAIMON_ASSIGN_OR_RAISE(impl_->specific_file_system_, impl_->catalog_->GetTableFileSystem(impl_->identifier_.value())); } - PAIMON_ASSIGN_OR_RAISE(std::string branch, - BranchManager::ResolveBranch(impl_->identifier_, impl_->options_, - /*explicit_branch=*/std::nullopt, "scan")); std::shared_ptr executor = impl_->executor_ ? impl_->executor_ : CreateDefaultExecutor(); auto ctx = std::make_unique( @@ -233,8 +208,8 @@ Result> ScanContextBuilder::Finish() { std::make_shared(impl_->predicates_, impl_->partition_filters_, impl_->bucket_filter_), impl_->global_index_result_, impl_->realtime_context_, impl_->memory_pool_, executor, - impl_->specific_file_system_, impl_->table_schema_, impl_->options_, impl_->cache_, - impl_->catalog_, impl_->identifier_, branch, impl_->fs_scheme_to_identifier_map_); + impl_->specific_file_system_, impl_->table_schema_, impl_->cache_, impl_->catalog_, + impl_->identifier_, impl_->fs_scheme_to_identifier_map_); impl_->Reset(); return ctx; } diff --git a/src/paimon/core/operation/scan_context_test.cpp b/src/paimon/core/operation/scan_context_test.cpp index 7a0e9e1bc..e18e60b1a 100644 --- a/src/paimon/core/operation/scan_context_test.cpp +++ b/src/paimon/core/operation/scan_context_test.cpp @@ -90,8 +90,6 @@ TEST(ScanContextTest, TestSchemeMapWithoutFileSystem) { } TEST(ScanContextTest, TestSchemeMapOverridesCatalogFileSystem) { - // A scheme map routes by scheme instead of a per-table file system, so the catalog is not - // asked for one when a map is set. auto table_fs = std::make_shared(); auto catalog = std::make_shared(); catalog->SetTableFileSystem(table_fs); @@ -112,32 +110,10 @@ TEST(ScanContextTest, TestWithCatalogNullRejected) { "cannot scan through a null catalog"); } -TEST(ScanContextTest, TestBranchResolution) { - // Finish() resolves the branch out of the identifier and the options and stores it, so the - // factory reads a resolved branch instead of resolving it on the spot. - ScanContextBuilder builder("table_root_path"); - ASSERT_OK_AND_ASSIGN(auto option_ctx, builder.AddOption(Options::BRANCH, "dev").Finish()); - ASSERT_EQ("dev", option_ctx->GetBranch()); - - auto catalog = std::make_shared(); - ASSERT_OK_AND_ASSIGN( - auto identifier_ctx, - builder.WithCatalog(catalog, Identifier("db1", "t1$branch_data")).Finish()); - ASSERT_EQ("data", identifier_ctx->GetBranch()); - - // Names which disagree are refused rather than silently chosen between. - ScanContextBuilder mismatch_builder("table_root_path"); - ASSERT_NOK_WITH_MSG(mismatch_builder.WithCatalog(catalog, Identifier("db1", "t1$branch_data")) - .AddOption(Options::BRANCH, "rt") - .Finish(), - "a scan is aimed at one branch, but both 'data' and 'rt' were named"); -} - TEST(ScanContextTest, TestDefaultValue) { ScanContextBuilder builder("table_root_path"); ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish()); ASSERT_EQ(ctx->GetPath(), "table_root_path"); - ASSERT_EQ("main", ctx->GetBranch()); ASSERT_FALSE(ctx->GetTableId()); ASSERT_FALSE(ctx->IsStreamingMode()); ASSERT_FALSE(ctx->GetLimit()); @@ -147,7 +123,6 @@ TEST(ScanContextTest, TestDefaultValue) { ASSERT_FALSE(ctx->GetScanFilters()->GetBucketFilter()); ASSERT_FALSE(ctx->GetScanFilters()->GetPredicate()); ASSERT_TRUE(ctx->GetScanFilters()->GetPartitionFilters().empty()); - ASSERT_TRUE(ctx->GetOptions().empty()); ASSERT_FALSE(ctx->GetGlobalIndexResult()); ASSERT_FALSE(ctx->GetSpecificFileSystem()); ASSERT_TRUE(ctx->GetFileSystemSchemeToIdentifierMap().empty()); @@ -168,7 +143,7 @@ TEST(ScanContextTest, TestSetContent) { auto global_index_result = BitmapGlobalIndexResult::FromRanges(row_ranges); builder.SetGlobalIndexResult(global_index_result); builder.SetLimit(1000); - builder.AddOption("key", "value"); + builder.WithStreamingMode(true); builder.WithMemoryPool(memory_pool); builder.WithExecutor(executor); @@ -191,22 +166,10 @@ TEST(ScanContextTest, TestSetContent) { ASSERT_TRUE(ctx->GetSpecificTableSchema().has_value()); ASSERT_EQ("table-schema-json", ctx->GetSpecificTableSchema().value()); std::map expected_options = {{"key", "value"}}; - ASSERT_EQ(expected_options, ctx->GetOptions()); ASSERT_EQ(fs, ctx->GetSpecificFileSystem()); ASSERT_TRUE(ctx->GetCache()); } -TEST(ScanContextTest, TestSetOptionsOverridesAddedOptions) { - ScanContextBuilder builder("table_root_path"); - builder.AddOption("old", "value"); - builder.SetOptions({{"key1", "value1"}, {"key2", "value2"}}); - - ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish()); - - std::map expected_options = {{"key1", "value1"}, {"key2", "value2"}}; - ASSERT_EQ(expected_options, ctx->GetOptions()); -} - TEST(ScanContextTest, TestDefaultExecutorIsCreatedPerContext) { // A builder without WithExecutor() gives every context a default executor // of its own; nothing is shared across contexts. diff --git a/src/paimon/core/operation/write_context.cpp b/src/paimon/core/operation/write_context.cpp index ad20baf0a..328c2043c 100644 --- a/src/paimon/core/operation/write_context.cpp +++ b/src/paimon/core/operation/write_context.cpp @@ -37,20 +37,15 @@ namespace paimon { WriteContext::WriteContext( const std::string& root_path, const std::string& commit_user, bool is_streaming_mode, bool ignore_num_bucket_check, bool ignore_previous_files, bool enable_multi_thread_spill, - const std::optional& write_id, const std::string& branch, - const std::optional& named_branch, - const std::vector& write_schema, const std::shared_ptr& memory_pool, - const std::shared_ptr& executor, const std::string& temp_directory, - const std::shared_ptr& specific_file_system, + const std::optional& write_id, const std::vector& write_schema, + const std::shared_ptr& memory_pool, const std::shared_ptr& executor, + const std::string& temp_directory, const std::shared_ptr& specific_file_system, const std::map& fs_scheme_to_identifier_map, const std::shared_ptr& realtime_context, - const std::map& options, const std::optional& table_schema, const std::shared_ptr& catalog, const std::optional& identifier) : root_path_(root_path), commit_user_(commit_user), - branch_(branch), - named_branch_(named_branch), is_streaming_mode_(is_streaming_mode), ignore_num_bucket_check_(ignore_num_bucket_check), ignore_previous_files_(ignore_previous_files), @@ -63,7 +58,6 @@ WriteContext::WriteContext( specific_file_system_(specific_file_system), fs_scheme_to_identifier_map_(fs_scheme_to_identifier_map), realtime_context_(realtime_context), - options_(options), table_schema_(table_schema), catalog_(catalog), identifier_(identifier) {} @@ -83,7 +77,6 @@ class WriteContextBuilder::Impl { memory_pool_ = GetDefaultPool(); executor_ = CreateDefaultExecutor(); temp_directory_.clear(); - branch_.reset(); write_schema_.clear(); fs_scheme_to_identifier_map_.clear(); specific_file_system_.reset(); @@ -91,15 +84,11 @@ class WriteContextBuilder::Impl { catalog_.reset(); identifier_.reset(); table_schema_ = std::nullopt; - options_.clear(); } private: std::string root_path_; std::string commit_user_; - /// Unset until `WithBranch()` names one, so that a branch named only by the identifier or - /// the `branch` option is not overruled by the default. - std::optional branch_; std::optional write_id_; bool is_streaming_mode_ = false; bool ignore_num_bucket_check_ = false; @@ -115,7 +104,6 @@ class WriteContextBuilder::Impl { std::shared_ptr catalog_; std::optional identifier_; std::optional table_schema_; - std::map options_; }; WriteContextBuilder::WriteContextBuilder(const std::string& root_path, @@ -127,18 +115,6 @@ WriteContextBuilder::WriteContextBuilder(const std::string& root_path, WriteContextBuilder::~WriteContextBuilder() = default; -WriteContextBuilder& WriteContextBuilder::AddOption(const std::string& key, - const std::string& value) { - impl_->options_[key] = value; - return *this; -} - -WriteContextBuilder& WriteContextBuilder::SetOptions( - const std::map& opts) { - impl_->options_ = opts; - return *this; -} - WriteContextBuilder& WriteContextBuilder::WithStreamingMode(bool is_streaming_mode) { impl_->is_streaming_mode_ = is_streaming_mode; return *this; @@ -175,11 +151,6 @@ WriteContextBuilder& WriteContextBuilder::WithWriteId(int32_t write_id) { return *this; } -WriteContextBuilder& WriteContextBuilder::WithBranch(const std::string& branch) { - impl_->branch_ = branch; - return *this; -} - WriteContextBuilder& WriteContextBuilder::WithWriteSchema( const std::vector& write_schema) { impl_->write_schema_ = write_schema; @@ -235,29 +206,9 @@ Result> WriteContextBuilder::Finish() { PAIMON_RETURN_NOT_OK( CatalogUtils::CheckNotSystemTable(impl_->identifier_.value(), "write")); } - std::optional identifier_branch; - if (impl_->identifier_) { - PAIMON_ASSIGN_OR_RAISE(identifier_branch, impl_->identifier_->GetBranchName()); - } - PAIMON_ASSIGN_OR_RAISE( - std::string branch, - BranchManager::ResolveBranch(impl_->identifier_, impl_->options_, impl_->branch_, "write")); - if (impl_->catalog_ != nullptr && !BranchManager::IsMainBranch(branch)) { - // Refused here rather than at the commit of what is written to the branch. - PAIMON_RETURN_NOT_OK(BranchManager::CheckCatalogAddressableBranch(branch)); - if (!identifier_branch) { - PAIMON_ASSIGN_OR_RAISE(std::string table_name, impl_->identifier_->GetDataTableName()); - return Status::Invalid(fmt::format( - "a write through a catalog addresses a branch by the table identifier, so name " - "branch '{}' there as '{}$branch_{}'", - branch, table_name, branch)); - } - } if (impl_->catalog_ != nullptr && impl_->specific_file_system_ == nullptr && impl_->fs_scheme_to_identifier_map_.empty()) { - // A catalog issuing per-table temporary credentials only hands them out through - // GetTableFileSystem, so the write uses the credentials of this table. A scheme map - // routes by scheme instead, so the catalog is not asked when one is set. + // A scheme map takes precedence over catalog-issued per-table credentials. PAIMON_ASSIGN_OR_RAISE(impl_->specific_file_system_, impl_->catalog_->GetTableFileSystem(impl_->identifier_.value())); } @@ -266,20 +217,12 @@ Result> WriteContextBuilder::Finish() { PAIMON_RETURN_NOT_OK_FROM_ARROW( arrow::SetCpuThreadPoolCapacity(impl_->spill_thread_number_)); } - // The branch the naming settings named, kept beside the resolved one so that a table can - // tell a context naming no branch - which it serves on its own - from one naming another. - std::optional named_branch; - if (impl_->branch_ || identifier_branch || - impl_->options_.find(Options::BRANCH) != impl_->options_.end()) { - named_branch = branch; - } auto ctx = std::make_unique( impl_->root_path_, impl_->commit_user_, impl_->is_streaming_mode_, impl_->ignore_num_bucket_check_, impl_->ignore_previous_files_, enable_multi_thread_spill, - impl_->write_id_, branch, named_branch, impl_->write_schema_, impl_->memory_pool_, - impl_->executor_, impl_->temp_directory_, impl_->specific_file_system_, - impl_->fs_scheme_to_identifier_map_, impl_->realtime_context_, impl_->options_, - impl_->table_schema_, impl_->catalog_, impl_->identifier_); + impl_->write_id_, impl_->write_schema_, impl_->memory_pool_, impl_->executor_, + impl_->temp_directory_, impl_->specific_file_system_, impl_->fs_scheme_to_identifier_map_, + impl_->realtime_context_, impl_->table_schema_, impl_->catalog_, impl_->identifier_); impl_->Reset(); return ctx; } diff --git a/src/paimon/core/operation/write_context_test.cpp b/src/paimon/core/operation/write_context_test.cpp index a0137bf59..ea551eeeb 100644 --- a/src/paimon/core/operation/write_context_test.cpp +++ b/src/paimon/core/operation/write_context_test.cpp @@ -43,12 +43,10 @@ TEST(WriteContextTest, TestDefaultValue) { ASSERT_FALSE(ctx->IgnorePreviousFiles()); ASSERT_FALSE(ctx->EnableMultiThreadSpill()); ASSERT_EQ(ctx->GetWriteId(), std::nullopt); - ASSERT_EQ(ctx->GetBranch(), "main"); ASSERT_TRUE(ctx->GetWriteSchema().empty()); ASSERT_TRUE(ctx->GetMemoryPool()); ASSERT_TRUE(ctx->GetExecutor()); ASSERT_TRUE(ctx->GetTempDirectory().empty()); - ASSERT_TRUE(ctx->GetOptions().empty()); ASSERT_TRUE(ctx->GetFileSystemSchemeToIdentifierMap().empty()); ASSERT_FALSE(ctx->GetSpecificFileSystem()); ASSERT_FALSE(ctx->GetSpecificTableSchema()); @@ -72,8 +70,7 @@ TEST(WriteContextTest, TestWithCatalog) { } TEST(WriteContextTest, TestWithCatalogResolvesTableFileSystem) { - // WithCatalog resolves the per-table file system when Finish() builds the context, which the - // factory then reads instead of asking the catalog again. + // Finish() resolves the per-table file system once. auto table_fs = std::make_shared(); auto catalog = std::make_shared(); catalog->SetTableFileSystem(table_fs); @@ -84,7 +81,6 @@ TEST(WriteContextTest, TestWithCatalogResolvesTableFileSystem) { ASSERT_EQ(catalog->TableFileSystemRequests().size(), 1U); ASSERT_EQ(catalog->TableFileSystemRequests().front(), Identifier("db1", "t1")); - // A scheme map routes by scheme instead, so the catalog is not asked for a file system. WriteContextBuilder map_builder("table_root_path", "commit_user_1"); ASSERT_OK_AND_ASSIGN(auto map_ctx, map_builder.WithCatalog(catalog, Identifier("db1", "t1")) .WithFileSystemSchemeToIdentifierMap({{"file", "local"}}) @@ -93,87 +89,6 @@ TEST(WriteContextTest, TestWithCatalogResolvesTableFileSystem) { ASSERT_EQ(catalog->TableFileSystemRequests().size(), 1U); } -TEST(WriteContextTest, TestCatalogAddressesBranchByIdentifier) { - auto catalog = std::make_shared(); - WriteContextBuilder builder("table_root_path", "commit_user_1"); - ASSERT_OK_AND_ASSIGN(auto ctx, builder.WithCatalog(catalog, Identifier("db1", "t1$branch_dev")) - .WithBranch("dev") - .AddOption(Options::BRANCH, "dev") - .Finish()); - ASSERT_EQ(ctx->GetBranch(), "dev"); - - WriteContextBuilder identifier_builder("table_root_path", "commit_user_1"); - ASSERT_OK_AND_ASSIGN( - auto identifier_ctx, - identifier_builder.WithCatalog(catalog, Identifier("db1", "t1$branch_dev")).Finish()); - ASSERT_EQ(identifier_ctx->GetBranch(), "dev"); - - for (int32_t branch_source = 0; branch_source < 2; ++branch_source) { - SCOPED_TRACE(branch_source); - WriteContextBuilder option_only_builder("table_root_path", "commit_user_1"); - option_only_builder.WithCatalog(catalog, Identifier("db1", "t1")); - if (branch_source == 0) { - option_only_builder.WithBranch("dev"); - } else { - option_only_builder.AddOption(Options::BRANCH, "dev"); - } - ASSERT_NOK_WITH_MSG(option_only_builder.Finish(), - "name branch 'dev' there as 't1$branch_dev'"); - } - - WriteContextBuilder mixed_builder("table_root_path", "commit_user_1"); - ASSERT_NOK_WITH_MSG(mixed_builder.WithCatalog(catalog, Identifier("db1", "t1$branch_dev")) - .WithBranch("release") - .Finish(), - "but both 'dev' and 'release' were named"); - - WriteContextBuilder main_builder("table_root_path", "commit_user_1"); - ASSERT_OK(main_builder.WithCatalog(catalog, Identifier("db1", "t1$branch_main")) - .WithBranch("") - .AddOption(Options::BRANCH, "main") - .Finish()); - - WriteContextBuilder upper_main_builder("table_root_path", "commit_user_1"); - ASSERT_NOK_WITH_MSG( - upper_main_builder.WithCatalog(catalog, Identifier("db1", "t1$branch_MAIN")).Finish(), - "a catalog names branch 'MAIN' as it names the main branch"); - - WriteContextBuilder constructed_builder("table_root_path", "commit_user_1"); - ASSERT_OK_AND_ASSIGN( - auto constructed_ctx, - constructed_builder.WithCatalog(catalog, Identifier("db1", "t1", "dev")).Finish()); - ASSERT_EQ(constructed_ctx->GetBranch(), "dev"); - WriteContextBuilder system_table_builder("table_root_path", "commit_user_1"); - ASSERT_NOK_WITH_MSG( - system_table_builder.WithCatalog(catalog, Identifier("db1", "t1", "dev$options")).Finish(), - "Cannot 'write' for system table"); -} - -TEST(WriteContextTest, TestBranch) { - WriteContextBuilder option_builder("table_root_path", "commit_user_1"); - ASSERT_OK_AND_ASSIGN(auto option_ctx, option_builder.AddOption(Options::BRANCH, "rt").Finish()); - ASSERT_EQ("rt", option_ctx->GetBranch()); - - WriteContextBuilder upper_main_builder("table_root_path", "commit_user_1"); - ASSERT_OK_AND_ASSIGN(auto upper_main_ctx, upper_main_builder.WithBranch("MAIN").Finish()); - ASSERT_EQ("MAIN", upper_main_ctx->GetBranch()); - - // An empty branch selects the main branch and stays accepted. - WriteContextBuilder main_builder("table_root_path", "commit_user_1"); - main_builder.WithBranch(""); - ASSERT_OK_AND_ASSIGN(auto ctx, main_builder.Finish()); - ASSERT_EQ(BranchManager::DEFAULT_MAIN_BRANCH, ctx->GetBranch()); - WriteContextBuilder empty_builder("table_root_path", "commit_user_1"); - ASSERT_NOK_WITH_MSG(empty_builder.WithBranch("").AddOption(Options::BRANCH, "dev").Finish(), - "but both 'dev' and 'main' were named"); - - // The branch names a directory under the root path, so a value that is not a single path - // component is rejected when the context is built. - WriteContextBuilder escaping_builder("table_root_path", "commit_user_1"); - escaping_builder.WithBranch("rt/../../../../../outside"); - ASSERT_NOK_WITH_MSG(escaping_builder.Finish(), "branch name cannot contain path separators"); -} - TEST(WriteContextTest, TestSetContent) { WriteContextBuilder builder("table_root_path", "commit_user_1"); @@ -185,6 +100,7 @@ TEST(WriteContextTest, TestSetContent) { {"oss", "jindo"}}; ASSERT_OK_AND_ASSIGN(auto ctx, + builder.WithStreamingMode(true) .WithIgnoreNumBucketCheck(true) .WithIgnorePreviousFiles(true) @@ -192,12 +108,10 @@ TEST(WriteContextTest, TestSetContent) { .WithExecutor(executor) .WithTempDirectory("/tmp/with-all") .WithWriteId(123) - .WithBranch("test_branch") .WithWriteSchema(write_schema) .WithFileSystemSchemeToIdentifierMap(fs_scheme_to_identifier_map) .WithFileSystem(file_system) .SetTableSchema("table-schema-json") - .AddOption("key", "value") .Finish()); ASSERT_TRUE(ctx->IsStreamingMode()); @@ -208,25 +122,12 @@ TEST(WriteContextTest, TestSetContent) { ASSERT_EQ(ctx->GetExecutor(), executor); ASSERT_EQ(ctx->GetTempDirectory(), "/tmp/with-all"); ASSERT_EQ(ctx->GetWriteId(), 123); - ASSERT_EQ(ctx->GetBranch(), "test_branch"); ASSERT_EQ(ctx->GetWriteSchema(), write_schema); ASSERT_EQ(ctx->GetFileSystemSchemeToIdentifierMap(), fs_scheme_to_identifier_map); ASSERT_EQ(ctx->GetSpecificFileSystem(), file_system); ASSERT_TRUE(ctx->GetSpecificTableSchema().has_value()); ASSERT_EQ("table-schema-json", ctx->GetSpecificTableSchema().value()); std::map expected_options = {{"key", "value"}}; - ASSERT_EQ(expected_options, ctx->GetOptions()); -} - -TEST(WriteContextTest, TestSetOptionsOverridesAddedOptions) { - WriteContextBuilder builder("table_root_path", "commit_user_1"); - builder.AddOption("old", "value"); - builder.SetOptions({{"key1", "value1"}, {"key2", "value2"}}); - - ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish()); - - std::map expected_options = {{"key1", "value1"}, {"key2", "value2"}}; - ASSERT_EQ(expected_options, ctx->GetOptions()); } TEST(WriteContextTest, TestSetWriteBufferSpillThreadNumber) { diff --git a/src/paimon/core/schema/schema_validation.cpp b/src/paimon/core/schema/schema_validation.cpp index 9fcb5f370..9640a4173 100644 --- a/src/paimon/core/schema/schema_validation.cpp +++ b/src/paimon/core/schema/schema_validation.cpp @@ -75,21 +75,9 @@ namespace paimon { namespace { constexpr char kDeletionVectorsMergeOnRead[] = "deletion-vectors.merge-on-read"; +constexpr char kDeletionVectorsModifiable[] = "deletion-vectors.modifiable"; -/// The options a copy is refused to change, mirroring the Java `SchemaManagerUtils# -/// checkAlterTableOption` over the immutable options the C++ side has constants for. They name -/// the physical layout the data on the table's path was written by: a `bucket-key` the options -/// rename, say, re-hashes the bucket pruning while the files stay hashed by the old key, so the -/// scans a copy serves would filter out the buckets its rows are actually in. The check runs over -/// every option the call gives, before the merge, so it holds on every path that merges options -/// into a table. Unlike the Java set, `merge-engine`, `bucket`, `row-tracking.enabled` and -/// `data-evolution.enabled` stay servable as dynamic options here, and `type`/`branch` are dropped -/// by `Copy` before this point. -// TODO(yonghao.fyh): refuse the rest of Java's immutable set, which this list does not cover yet: -// `blob-descriptor-field` (only its deprecated `blob.stored-descriptor-fields` fallback is above), -// `data-file.path-directory`, `sequence.snapshot-ordering`, `rowkind.field`, `primary-key`, -// `primary-key.nullable`, `partition`, `dynamic-bucket.initial-buckets`, `video-frame-field`, and -// the four servable keys named above, should the C++ side stop serving them as dynamic options. +// Layout options protected by both operator factories and table copies. const char* const kImmutableOptions[] = { Options::BUCKET_KEY, Options::BUCKET_FUNCTION_TYPE, @@ -104,6 +92,23 @@ const char* const kImmutableOptions[] = { Options::PK_CLUSTERING_OVERRIDE, }; +// Copy-only immutable options; low-level factories retain their initialization semantics. +const char* const kCopyImmutableOptions[] = { + Options::BLOB_DESCRIPTOR_FIELD, + Options::MERGE_ENGINE, + Options::ROW_TRACKING_ENABLED, + Options::DATA_EVOLUTION_ENABLED, + "bucket-function.provider", + "data-file.path-directory", + "sequence.snapshot-ordering", + "rowkind.field", + "primary-key", + "primary-key.nullable", + "partition", + "dynamic-bucket.initial-buckets", + "video-frame-field", +}; + bool ContainsBlobField(const std::shared_ptr& field) { if (BlobUtils::IsBlobField(field)) { return true; @@ -394,7 +399,7 @@ Status SchemaValidation::ValidateTableSchema(const TableSchema& schema) { PAIMON_RETURN_NOT_OK(ValidateBucket(schema, options)); PAIMON_RETURN_NOT_OK(ValidateRealtime(schema, options)); // PAIMON_RETURN_NOT_OK(ValidateDefaultValues(schema)); - // PAIMON_RETURN_NOT_OK(ValidateStartupMode(options)); + PAIMON_RETURN_NOT_OK(ValidateStartupMode(options)); PAIMON_RETURN_NOT_OK(ValidateFieldsPrefix(schema, options)); PAIMON_RETURN_NOT_OK(ValidateSequenceField(schema, options)); PAIMON_RETURN_NOT_OK(ValidateSequenceGroup(schema, options)); @@ -432,10 +437,10 @@ Status SchemaValidation::ValidateTableSchema(const TableSchema& schema) { Status SchemaValidation::CheckImmutableOptions( const TableSchema& schema, const std::map& dynamic_options) { - const std::map& schema_options = schema.Options(); + const auto& schema_options = schema.Options(); for (const auto& [key, value] : dynamic_options) { - auto schema_option = schema_options.find(key); - if (schema_option != schema_options.end() && schema_option->second == value) { + auto old_value = schema_options.find(key); + if (old_value != schema_options.end() && old_value->second == value) { continue; } if (std::find(std::begin(kImmutableOptions), std::end(kImmutableOptions), key) != @@ -455,6 +460,97 @@ Status SchemaValidation::CheckImmutableOptions( return Status::OK(); } +Status SchemaValidation::CheckCopyOptions( + const TableSchema& schema, const std::map& dynamic_options) { + PAIMON_RETURN_NOT_OK(CheckImmutableOptions(schema, dynamic_options)); + const std::map& schema_options = schema.Options(); + for (const auto& [key, value] : dynamic_options) { + auto schema_option = schema_options.find(key); + if (schema_option != schema_options.end() && schema_option->second == value) { + continue; + } + if (schema_option == schema_options.end() && (key == "primary-key" || key == "partition")) { + std::vector keys; + for (const auto& item : StringUtils::Split(value, ",")) { + std::string trimmed = item; + StringUtils::Trim(&trimmed); + if (!trimmed.empty()) { + keys.push_back(trimmed); + } + } + if (keys == (key == "primary-key" ? schema.PrimaryKeys() : schema.PartitionKeys())) { + continue; + } + } + if (std::find(std::begin(kCopyImmutableOptions), std::end(kCopyImmutableOptions), key) != + std::end(kCopyImmutableOptions)) { + return Status::Invalid(fmt::format("Change '{}' is not supported yet.", key)); + } + if (key == Options::BUCKET) { + PAIMON_ASSIGN_OR_RAISE(int32_t old_bucket, + OptionsUtils::GetValueFromMap(schema_options, key, -1)); + PAIMON_ASSIGN_OR_RAISE(int32_t new_bucket, + OptionsUtils::GetValueFromMap(dynamic_options, key)); + if (old_bucket == -1) { + return Status::Invalid("Cannot change bucket when it is -1."); + } + if (new_bucket == -1) { + return Status::Invalid("Cannot change bucket to -1."); + } + } + if (key == Options::DELETION_VECTORS_ENABLED) { + PAIMON_ASSIGN_OR_RAISE(bool modifiable, + OptionsUtils::GetValueFromMap( + schema_options, kDeletionVectorsModifiable, false)); + PAIMON_ASSIGN_OR_RAISE(bool old_enabled, + OptionsUtils::GetValueFromMap(schema_options, key, false)); + PAIMON_ASSIGN_OR_RAISE(bool new_enabled, + OptionsUtils::GetValueFromMap(dynamic_options, key)); + if (!modifiable && old_enabled != new_enabled) { + return Status::Invalid(fmt::format( + "Cannot change deletion vectors mode from {} to {}. Set '{}' to true on the " + "table before changing deletion vectors mode.", + old_enabled, new_enabled, kDeletionVectorsModifiable)); + } + } + if (key == "ignore-update-before") { + PAIMON_ASSIGN_OR_RAISE(bool old_ignore_delete, + OptionsUtils::GetValueFromMap(schema_options, key, false)); + PAIMON_ASSIGN_OR_RAISE(bool new_ignore_delete, + OptionsUtils::GetValueFromMap(dynamic_options, key)); + if (old_ignore_delete && !new_ignore_delete) { + return Status::Invalid(fmt::format("Cannot change {} from true to false.", key)); + } + } + } + return Status::OK(); +} + +Status SchemaValidation::ValidateStartupMode(const CoreOptions& options) { + const auto& values = options.ToMap(); + const bool snapshot = values.count(Options::SCAN_SNAPSHOT_ID) != 0; + const bool tag = values.count(Options::SCAN_TAG_NAME) != 0; + const bool timestamp = values.count(Options::SCAN_TIMESTAMP) != 0; + const bool timestamp_millis = values.count(Options::SCAN_TIMESTAMP_MILLIS) != 0; + const int32_t selectors = snapshot + tag + timestamp + timestamp_millis; + const StartupMode mode = options.GetStartupMode(); + bool valid = selectors == 0; + if (mode == StartupMode::FromSnapshot()) { + valid = selectors == 1 && (snapshot || tag); + } else if (mode == StartupMode::FromSnapshotFull()) { + valid = selectors == 1 && snapshot; + } else if (mode == StartupMode::FromTimestamp()) { + valid = selectors == 1 && (timestamp || timestamp_millis); + } + if (!valid) { + return Status::Invalid(fmt::format( + "Invalid time travel options for scan.mode '{}': set exactly one compatible selector " + "for from-snapshot/from-snapshot-full/from-timestamp, and none for latest/latest-full.", + mode.ToString())); + } + return Status::OK(); +} + Status SchemaValidation::ValidateNoDuplicateField(const std::vector& field_names, const std::string& error_message_intro) { auto duplicate_field_names = ObjectUtils::DuplicateItems(field_names); diff --git a/src/paimon/core/schema/schema_validation.h b/src/paimon/core/schema/schema_validation.h index ba169c057..3e69ee0f9 100644 --- a/src/paimon/core/schema/schema_validation.h +++ b/src/paimon/core/schema/schema_validation.h @@ -51,15 +51,17 @@ class SchemaValidation { static Status ValidateTableSchema(const TableSchema& schema); - /// Whether a copy's dynamic options stay within what the data on the table's path was laid out - /// with - the C++ reading of the Java `AbstractFileStoreTable#checkImmutability`. An option - /// that says what the schema already says moves nothing, an immutable one that says anything - /// else is refused, and `ignore-delete` carries the Java check's directional rule of its own: - /// it may only ever be turned on, never back off. The immutable options themselves are the - /// Java `CoreOptions.IMMUTABLE_OPTIONS`, as far as the C++ side has constants for them. + /// Checks layout options protected by low-level factories and Copy alike. static Status CheckImmutableOptions(const TableSchema& schema, const std::map& dynamic_options); + /// Checks Java table Copy immutability, including bucket-mode transitions. + static Status CheckCopyOptions(const TableSchema& schema, + const std::map& dynamic_options); + + /// Checks selector exclusivity and compatibility with the effective scan.mode. + static Status ValidateStartupMode(const CoreOptions& options); + /// Validates real-time options after operation-specific options have been merged with the /// table schema options. static Status ValidateRealtime(const TableSchema& schema, const CoreOptions& options); @@ -130,9 +132,6 @@ class SchemaValidation { static Status ValidateDefaultValues(const TableSchema& schema) { return Status::NotImplemented("validate default values not implemented"); } - static Status ValidateStartupMode(const CoreOptions& options) { - return Status::NotImplemented("validate startup mode not implemented"); - } static Status ValidateFieldsPrefix(const TableSchema& schema, const CoreOptions& options); static Status ValidateSequenceField(const TableSchema& schema, const CoreOptions& options); static Status ValidateSequenceGroup(const TableSchema& schema, const CoreOptions& options); diff --git a/src/paimon/core/schema/table_schema.cpp b/src/paimon/core/schema/table_schema.cpp index e8a3b7b65..1c0cfd333 100644 --- a/src/paimon/core/schema/table_schema.cpp +++ b/src/paimon/core/schema/table_schema.cpp @@ -200,6 +200,11 @@ Result> TableSchema::MergeOptions( for (const auto& [key, value] : dynamic_options) { options[key] = value; } + return Copy(options); +} + +Result> TableSchema::Copy( + const std::map& options) const { auto merged = std::unique_ptr( new TableSchema(version_, id_, fields_, highest_field_id_, partition_keys_, primary_keys_, options, comment_, time_millis_)); diff --git a/src/paimon/core/schema/table_schema.h b/src/paimon/core/schema/table_schema.h index aff4ee920..ee817c030 100644 --- a/src/paimon/core/schema/table_schema.h +++ b/src/paimon/core/schema/table_schema.h @@ -102,14 +102,14 @@ class TableSchema : public DataSchema, public Jsonizable { return options_; } - /// A copy of this schema with `dynamic_options` on top of the options it carries, which is - /// the precedence every copy of a table promises. Everything else - id, fields, keys, - /// comment - is copied as it is. - /// - /// @param dynamic_options Options given at the call, which win over the schema's own. - /// @return A result containing the merged schema, or an error status. + /// Copies the schema with dynamic_options overriding its current options. + /// Other schema attributes are preserved. Result> MergeOptions( const std::map& dynamic_options) const; + + /// Copies this schema with the complete replacement option map. + Result> Copy( + const std::map& options) const; const std::vector& Fields() const { return fields_; } @@ -151,8 +151,7 @@ class TableSchema : public DataSchema, public Jsonizable { Result> OriginalBucketKeys() const; - /// Derives `bucket_keys_` and `num_bucket_` from the options the schema carries: every - /// constructor path must call this, as both members serve scans and copies. + /// Derives cached bucket keys and count after construction or option changes. Status DeriveBucketInfo(); static Result> AssignFieldIdsRecursively( diff --git a/src/paimon/core/table/append_only_file_store_table.h b/src/paimon/core/table/append_only_file_store_table.h index 5dee43d11..43e29bda1 100644 --- a/src/paimon/core/table/append_only_file_store_table.h +++ b/src/paimon/core/table/append_only_file_store_table.h @@ -34,15 +34,9 @@ class IOManager; class SplitGenerator; class TableSchema; -/// The managed table a schema without primary keys describes, Java Paimon's -/// `AppendOnlyFileStoreTable`. Its write appends the rows it is given, and its scan, read and -/// commit walk the files as they were written. The operator flows are the ones `FileStoreTable` -/// carries; the pieces those flows hand down are built here. +/// Managed table without primary keys; uses the shared FileStoreTable operator flows. class AppendOnlyFileStoreTable : public FileStoreTable { public: - /// The constructor `FileStoreTable::Create` builds an append-only table through: the store - /// carries the storage state, `schema` is the schema the table was loaded with, and the base - /// class holds both. AppendOnlyFileStoreTable(const std::shared_ptr& store, const std::shared_ptr& schema, const Identifier& identifier, const std::string& uuid); diff --git a/src/paimon/core/table/file_store.h b/src/paimon/core/table/file_store.h index 58812a76e..12d200091 100644 --- a/src/paimon/core/table/file_store.h +++ b/src/paimon/core/table/file_store.h @@ -45,22 +45,14 @@ class IndexManifestFile; class ManifestFile; class ManifestList; -/// Holds the storage a managed table reads and writes through: the file system, root path, -/// branch, options and managers. Java Paimon carries the same state in `AbstractFileStore`. -/// `FileStoreTable` builds one when it loads and asks it for the pieces each operator opens with. -/// -/// The factories below take the options of the call, so an operator can serve a cache of its own -/// over the options the table was loaded with. Each call builds a fresh instance and never a -/// cached one: a snapshot manager a write armed with a catalog loader must not serve a scan. +/// Storage state and factories for managed-table operators. +/// Operator factories create fresh instances so catalog snapshot loaders are not shared +/// between scans, writes and commits. class FileStore { public: - /// Builds the store over options resolved before the call - `FileStoreTable::Create` resolves - /// them once - so the store consumes the resolution and parses nothing itself. - /// @param root_path Root path of the table in the file system. - /// @param core_options The final options the table serves. The store reads and writes through - /// their file system and serves their branch, normalized here; both managers are built - /// over the two, so no manager is ever left reading through a backend its table moved - /// off. + /// Builds storage from already resolved CoreOptions. + /// @param root_path Table root path. + /// @param core_options Effective options, including the file system and branch. static std::shared_ptr Create(const std::string& root_path, CoreOptions core_options); @@ -68,20 +60,14 @@ class FileStore { return branch_; } - /// The schema manager the operators read through. It carries the schema cache of `root_path` - /// on `CurrentBranch()`. const std::shared_ptr& GetSchemaManager() const { return schema_manager_; } - /// The snapshot manager a scan reads through. A write or a commit builds its own: see - /// `NewSnapshotManager`. const std::shared_ptr& GetSnapshotManager() const { return snapshot_manager_; } - /// The file system the table's data is accessed through: the one the options resolved, which - /// is the one the caller opened the table through when it opened it through one of its own. const std::shared_ptr& GetFileSystem() const { return file_system_; } @@ -90,60 +76,41 @@ class FileStore { return root_path_; } - /// The final options the table serves, as `Create` took them. Every operator reads them; a - /// copy or a branch switch resolves its own, from the inputs the table keeps. const CoreOptions& GetOptions() const { return core_options_; } - /// Builds the path factory an operator names its metadata paths through, over this store's - /// root path. The caller names the partition keys of the schema the arrow schema was converted - /// from: an operator serving the table's own schema names this store's partition keys, while - /// a read of a fallback branch names the keys of that branch's schema, which is not this - /// store's. The caller names the file format identifier: a write names the format of its - /// level-0 files, which the options may set per level, while a scan, a read and a commit name - /// the table's format. + /// Uses the caller's partition keys and format identifier, which may differ for fallback + /// reads or per-level writes. Result> NewPathFactory( const std::shared_ptr& arrow_schema, const std::vector& partition_keys, const CoreOptions& options, const std::string& file_format_identifier, const std::shared_ptr& memory_pool) const; - /// Builds the snapshot manager a write or a commit reads snapshots through, over this store's - /// root path and branch. The caller names the file system, from the options its operator - /// serves. A catalog that version-manages the table hands out its own snapshots, so its loader - /// is attached and the catalog is captured to keep it alive. Such a catalog names the table by - /// an identifier, which the context must have supplied. + /// Creates an operator-local snapshot manager with an optional catalog loader. + /// Version-managed catalogs require catalog_identifier. Result> NewSnapshotManager( const CoreOptions& options, const std::shared_ptr& catalog, const std::optional& catalog_identifier) const; - /// Builds the manifest list a scan or a commit reads the table's manifest entries through. Result> NewManifestList( const CoreOptions& options, const std::shared_ptr& path_factory, const std::shared_ptr& memory_pool) const; - /// Builds the manifest file a scan or a commit reads a single manifest's entries through, - /// pruned by `partition_schema`. Result> NewManifestFile( const CoreOptions& options, const std::shared_ptr& path_factory, const std::shared_ptr& partition_schema, const std::shared_ptr& memory_pool) const; - /// Builds the index manifest file a write, a scan or a commit reads the table's index - /// manifest through. Result> NewIndexManifestFile( const CoreOptions& options, const std::shared_ptr& path_factory, const std::shared_ptr& memory_pool) const; - /// Builds the index file handler a write maintains its deletion vectors and indexes through, - /// or a scan reads them through. Result> NewIndexFileHandler( const CoreOptions& options, const std::shared_ptr& path_factory, const std::shared_ptr& memory_pool) const; - /// Builds the partition computer a commit names the partitions it writes with, over the - /// caller's partition keys and arrow schema. Result> NewPartitionComputer( const std::shared_ptr& arrow_schema, const std::vector& partition_keys, const CoreOptions& options, @@ -157,7 +124,6 @@ class FileStore { std::string branch_; std::shared_ptr schema_manager_; std::shared_ptr snapshot_manager_; - /// The final options the table serves, as `Create` took them. CoreOptions core_options_; }; diff --git a/src/paimon/core/table/file_store_table.cpp b/src/paimon/core/table/file_store_table.cpp index 1b5dae4fa..808334130 100644 --- a/src/paimon/core/table/file_store_table.cpp +++ b/src/paimon/core/table/file_store_table.cpp @@ -25,6 +25,7 @@ #include #include #include +#include #include "fmt/format.h" #include "paimon/catalog/catalog.h" @@ -74,6 +75,7 @@ #include "paimon/core/utils/field_mapping.h" #include "paimon/core/utils/file_store_path_factory.h" #include "paimon/core/utils/snapshot_manager.h" +#include "paimon/core/utils/tag_manager.h" #include "paimon/file_store_commit.h" #include "paimon/file_store_write.h" #include "paimon/format/file_format.h" @@ -97,8 +99,7 @@ namespace paimon { namespace { -/// The escape hatch that lets an integration test commit on an object store, read where -/// `NewCommit` refuses one below. +/// Allows object-store commits in integration tests. constexpr const char kObjectStoreCommitInteTestOption[] = "enable-object-store-commit-in-inte-test"; } // namespace @@ -129,14 +130,22 @@ Result> FileStoreTable::Create( } else { options[Options::BRANCH] = normalized_branch; } + std::shared_ptr effective_schema = schema; + if (options != schema->Options()) { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr normalized_schema, + schema->Copy(options)); + effective_schema = std::shared_ptr(std::move(normalized_schema)); + } PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(options, specified_file_system, fs_scheme_map)); std::shared_ptr store = FileStore::Create(root_path, std::move(core_options)); std::shared_ptr table; if (!schema->PrimaryKeys().empty()) { - table = std::make_shared(store, schema, identifier, uuid); + table = + std::make_shared(store, effective_schema, identifier, uuid); } else { - table = std::make_shared(store, schema, identifier, uuid); + table = + std::make_shared(store, effective_schema, identifier, uuid); } table->dynamic_options_ = dynamic_options; table->specified_file_system_ = specified_file_system; @@ -144,71 +153,113 @@ Result> FileStoreTable::Create( return table; } -Result> FileStoreTable::Copy( - const std::map& dynamic_options) const { - std::shared_ptr loaded_schema = std::static_pointer_cast(schema_); - // `type` is immutable the way Java Paimon holds it: the kind of a copy is named by the primary - // keys of the schema it serves, never by this option. A value naming another type is refused, - // and one naming the same type - whatever its case, an absent `type` reading as `table` - is - // dropped, as Java's unchanged-key check drops it. - auto type_option = dynamic_options.find(Options::TYPE); - if (type_option != dynamic_options.end()) { +Result> FileStoreTable::CopyTableSchema( + const std::shared_ptr& fs, const std::string& root_path, + const std::string& current_branch, const std::shared_ptr& schema, + const std::map& applied_options, + const std::map& dynamic_options) { + std::map overrides = dynamic_options; + auto type_option = overrides.find(Options::TYPE); + if (type_option != overrides.end()) { PAIMON_ASSIGN_OR_RAISE(TableType schema_type, - TableTypeDefine::FromOptions(loaded_schema->Options())); - std::map named{{Options::TYPE, type_option->second}}; - PAIMON_ASSIGN_OR_RAISE(TableType named_type, TableTypeDefine::FromOptions(named)); + TableTypeDefine::FromOptions(schema->Options())); + PAIMON_ASSIGN_OR_RAISE(TableType named_type, TableTypeDefine::FromOptions( + {{Options::TYPE, type_option->second}})); if (named_type != schema_type) { - return Status::Invalid(fmt::format("Change '{}' is not supported yet.", Options::TYPE)); + return Status::Invalid("Change 'type' is not supported yet."); + } + overrides.erase(type_option); + } + PAIMON_RETURN_NOT_OK(SchemaValidation::CheckCopyOptions(*schema, overrides)); + for (const std::string& key : {std::string("primary-key"), std::string("partition")}) { + if (schema->Options().count(key) == 0) { + // The immutable check accepts these only when they normalize to the schema keys. + overrides.erase(key); + } + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merged, schema->MergeOptions(overrides)); + PAIMON_ASSIGN_OR_RAISE(CoreOptions options, CoreOptions::FromMap(merged->Options(), fs)); + PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateStartupMode(options)); + + // Snapshot lookup failures defer to scanning; a resolved snapshot with a missing schema + // fails Copy. + const std::shared_ptr resolved_fs = options.GetFileSystem(); + SnapshotManager snapshots(resolved_fs, root_path, current_branch); + auto resolve_snapshot = [&]() -> Result> { + const int32_t selectors = options.GetScanSnapshotId().has_value() + + options.GetScanTagName().has_value() + + options.GetScanTimestampMillis().has_value(); + if (selectors == 0) { + return std::optional(); + } + if (options.GetScanSnapshotId()) { + PAIMON_ASSIGN_OR_RAISE(Snapshot snapshot, + snapshots.LoadSnapshot(*options.GetScanSnapshotId())); + return std::optional(std::move(snapshot)); + } + if (options.GetScanTagName()) { + TagManager tags(resolved_fs, root_path, current_branch); + PAIMON_ASSIGN_OR_RAISE(Tag tag, tags.GetOrThrow(*options.GetScanTagName())); + PAIMON_ASSIGN_OR_RAISE(Snapshot snapshot, tag.TrimToSnapshot()); + return std::optional(std::move(snapshot)); + } + return snapshots.EarlierOrEqualTimeMillis(*options.GetScanTimestampMillis()); + }; + auto snapshot = resolve_snapshot(); + if (snapshot.ok() && snapshot.value()) { + SchemaManager schemas(resolved_fs, root_path, current_branch); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr historical, + schemas.ReadSchema(snapshot.value()->SchemaId())); + auto historical_options = merged->Options(); + // Restore historical column declarations and aliases unless explicitly overridden. + for (const std::vector& keys : + {std::vector{"vector-field"}, + {Options::BLOB_FIELD}, + {Options::BLOB_DESCRIPTOR_FIELD, Options::FALLBACK_BLOB_DESCRIPTOR_FIELD}, + {Options::BLOB_VIEW_FIELD}, + {"video-frame-field"}}) { + bool explicitly_set = false; + for (const auto& key : keys) { + explicitly_set |= applied_options.count(key) != 0 || overrides.count(key) != 0; + } + if (explicitly_set) { + continue; + } + for (const auto& key : keys) { + historical_options.erase(key); + auto value = historical->Options().find(key); + if (value != historical->Options().end()) { + historical_options.emplace(*value); + } + } } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr historical_copy, + historical->Copy(historical_options)); + merged = std::move(historical_copy); } - // `branch` names the branch the copy serves, the way it routes a load: a copy on the branch - // this table serves keeps this table's schema, while one on another branch reads that branch's - // own latest schema and refuses a branch that holds none, as `SwitchToBranch` does. - std::string target_branch = CurrentBranch(); - auto branch_option = dynamic_options.find(Options::BRANCH); - if (branch_option != dynamic_options.end()) { - target_branch = BranchManager::NormalizeBranch(branch_option->second); - } - std::map overridable = dynamic_options; - overridable.erase(Options::TYPE); - overridable.erase(Options::BRANCH); - std::shared_ptr base_schema = loaded_schema; - // An `Identifier` is immutable once built, so the branch one is held beside this table's - // and the copy carries whichever names the branch it serves. - std::optional branch_identifier; - if (target_branch != CurrentBranch()) { - // The branch is read through the file system this table serves, the same read - // `SwitchToBranch` makes, and then served through the one its own schema resolves. - SchemaManager schema_manager(file_system_, root_path_, target_branch); - PAIMON_ASSIGN_OR_RAISE(std::optional> latest, - schema_manager.Latest()); - if (!latest) { - return Status::Invalid(fmt::format("branch {} does not exist", target_branch)); + PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateTableSchema(*merged)); + return std::shared_ptr(std::move(merged)); +} + +Result> FileStoreTable::Copy( + const std::map& dynamic_options) const { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr copied_schema, + CopyTableSchema(file_system_, root_path_, CurrentBranch(), + std::static_pointer_cast(schema_), + dynamic_options_, dynamic_options)); + auto accumulated = dynamic_options_; + for (const auto& [key, value] : dynamic_options) { + if (key != Options::TYPE) { + accumulated.insert_or_assign(key, value); } - base_schema = *latest; - PAIMON_ASSIGN_OR_RAISE(Identifier branch_named, - CatalogUtils::BranchIdentifier(identifier_, target_branch)); - branch_identifier.emplace(std::move(branch_named)); - } - std::map accumulated = dynamic_options_; - accumulated.erase(Options::TYPE); - accumulated.erase(Options::BRANCH); - for (const auto& [key, value] : overridable) { - accumulated.insert_or_assign(key, value); - } - std::map schema_overridable = accumulated; - if (target_branch != CurrentBranch() && target_branch != BranchManager::DEFAULT_MAIN_BRANCH) { - schema_overridable[Options::BRANCH] = target_branch; - } - PAIMON_RETURN_NOT_OK(SchemaValidation::CheckImmutableOptions(*base_schema, accumulated)); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merged_schema, - base_schema->MergeOptions(schema_overridable)); - // A copy resolves its file system from its own merged options, so it takes over no manager of - // the table it was copied from: the store builds both over the resolution the copy names. - return FileStoreTable::Create( - specified_file_system_, root_path_, std::shared_ptr(std::move(merged_schema)), - branch_identifier ? *branch_identifier : identifier_, CatalogUuid().value_or(""), - target_branch, fs_scheme_map_, accumulated); + } + auto branch = copied_schema->Options().find(Options::BRANCH); + const std::string target_branch = branch == copied_schema->Options().end() + ? BranchManager::DEFAULT_MAIN_BRANCH + : BranchManager::NormalizeBranch(branch->second); + return FileStoreTable::Create(specified_file_system_, root_path_, copied_schema, identifier_, + CatalogUuid().value_or(""), target_branch, fs_scheme_map_, + accumulated); } Result> FileStoreTable::SwitchToBranch(const std::string& branch) const { @@ -219,9 +270,6 @@ Result> FileStoreTable::SwitchToBranch(const std::string& CatalogUuid().value_or(""), CurrentBranch(), fs_scheme_map_, dynamic_options_); } - // The branch is read through the file system this table serves, and then served through the - // one its own schema resolves: the store builds the managers over that resolution, so none of - // this table's is carried over. SchemaManager schema_manager(file_system_, root_path_, target_branch); PAIMON_ASSIGN_OR_RAISE(std::optional> latest, schema_manager.Latest()); @@ -260,40 +308,6 @@ Status FileStoreTable::RestoreRealtimeCommittedProgress( return Status::OK(); } -Status FileStoreTable::CheckOptionsAgreeWithTable( - const TableSchema& table_schema, - const std::map& context_options) const { - std::map table_options = table_schema.Options(); - for (const auto& [key, value] : dynamic_options_) { - table_options.insert_or_assign(key, value); - } - for (const auto& [key, value] : context_options) { - auto table_option = table_options.find(key); - if (table_option != table_options.end()) { - if (table_option->second != value) { - return Status::Invalid( - fmt::format("option '{}' is '{}' in the context but '{}' on the table", key, - SensitiveConfigUtils::RedactValue(key, value), - SensitiveConfigUtils::RedactValue(key, table_option->second))); - } - continue; - } - // `branch` and `type` route a load to the table it names and are folded into it there - by - // the load or by `Copy` - so the spelling a context gives them is checked against the - // identifier and the schema, not against these keys. - if (key == Options::BRANCH || key == Options::TYPE) { - continue; - } - return Status::Invalid(fmt::format( - "option '{}' is '{}' in the context but the table was not loaded with it: the " - "operator serves the table's own configuration, so the context value would be " - "dropped in silence; load the table with this option or apply it through " - "Table::Copy()", - key, SensitiveConfigUtils::RedactValue(key, value))); - } - return Status::OK(); -} - Status FileStoreTable::CheckContextNamesThisTable( const std::string& kind, const std::string& context_root_path, const std::optional& context_identifier, @@ -371,7 +385,6 @@ Result> FileStoreTable::NewWrite( PAIMON_ASSIGN_OR_RAISE(std::optional catalog_identifier, CatalogUtils::BranchIdentifier(ctx->GetIdentifier(), CurrentBranch())); std::shared_ptr schema = std::static_pointer_cast(schema_); - PAIMON_RETURN_NOT_OK(CheckOptionsAgreeWithTable(*schema, ctx->GetOptions())); CoreOptions options = store_->GetOptions(); auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(schema->Fields()); PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*schema, options)); @@ -414,9 +427,8 @@ Result> FileStoreTable::NewScan( std::unique_ptr context) const { PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "scan")); std::shared_ptr table_schema = std::static_pointer_cast(schema_); - PAIMON_RETURN_NOT_OK(CheckOptionsAgreeWithTable(*table_schema, context->GetOptions())); PAIMON_ASSIGN_OR_RAISE(bool read_optimized, - OptionsUtils::GetValueFromMap(context->GetOptions(), + OptionsUtils::GetValueFromMap(store_->GetOptions().ToMap(), kReadOptimizedScanOption, false)); CoreOptions core_options = store_->GetOptions(); core_options.WithCache(context->GetCache()); @@ -539,10 +551,10 @@ Result> FileStoreTable::NewRead( std::shared_ptr read_context = std::move(context); PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(read_context, "read")); std::shared_ptr loaded_schema = std::static_pointer_cast(schema_); - PAIMON_RETURN_NOT_OK(CheckOptionsAgreeWithTable(*loaded_schema, read_context->GetOptions())); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr internal_context, - InternalReadContext::CreateForTable(read_context, loaded_schema, - store_->GetOptions(), root_path_)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr internal_context, + InternalReadContext::CreateForTable(read_context, loaded_schema, store_->GetOptions(), + root_path_, dynamic_options_)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, CreateTableRead(internal_context, read_context->GetMemoryPool(), @@ -606,7 +618,6 @@ Result> FileStoreTable::NewCommit( std::shared_ptr schema = std::static_pointer_cast(schema_); std::shared_ptr arrow_schema = DataField::ConvertDataFieldsToArrowSchema(schema->Fields()); - PAIMON_RETURN_NOT_OK(CheckOptionsAgreeWithTable(*schema, ctx->GetOptions())); CoreOptions options = store_->GetOptions(); PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*schema, options)); assert(options.GetFileSystem()); @@ -615,7 +626,7 @@ Result> FileStoreTable::NewCommit( PAIMON_ASSIGN_OR_RAISE(bool is_object_store, FileSystem::IsObjectStore(root_path_)); if (is_object_store && !ctx->UseRESTCatalogCommit() && AsVersionManaged(ctx->GetCatalog()) == nullptr && - ctx->GetOptions().find(kObjectStoreCommitInteTestOption) == ctx->GetOptions().end()) { + options.ToMap().count(kObjectStoreCommitInteTestOption) == 0) { return Status::NotImplemented( "commit operation does not support object store file system for now"); } diff --git a/src/paimon/core/table/file_store_table.h b/src/paimon/core/table/file_store_table.h index 9737dada0..fc44b583d 100644 --- a/src/paimon/core/table/file_store_table.h +++ b/src/paimon/core/table/file_store_table.h @@ -43,40 +43,16 @@ class ManifestList; class RealtimeContext; class SplitGenerator; -/// The managed table a catalog hands out: one with snapshots and manifests, described by the -/// schema under its root path. A format table - a directory of plain data files - is a -/// `FormatTable`, the other kind of `Table`. -/// -/// The four operators share one flow, the one Java Paimon's `AbstractFileStoreTable` carries. What -/// the table was loaded with - schema, file system, root path, branch and managers - decides what -/// every operator serves, so it is read once here and handed down. The schema's primary keys decide -/// the kind of managed table it is, `AppendOnlyFileStoreTable` or `PrimaryKeyFileStoreTable`, which -/// the `Create` factory below picks. That kind then builds the pieces the flow hands down: the -/// write itself, the scans, the read and the split generator. +/// Managed table backed by snapshots and manifests. +/// Shares operator setup across append-only and primary-key tables, selected from the schema. class FileStoreTable : public Table { public: - /// Builds the managed table the schema describes, as the kind its primary keys name: an - /// `AppendOnlyFileStoreTable` when there are none, a `PrimaryKeyFileStoreTable` when there are. - /// Java Paimon's `FileStoreTableFactory` picks the kind the same way. A schema change can - /// neither add nor drop primary keys, so every schema the root path ever held names one kind. - /// The final options are resolved here once, before the store is built: the schema's own - /// options as `branch` restyles them - the main branch carries no `branch` option, another - /// branch names itself - are parsed, and a parse failure is this call's error. Every operator - /// and both managers are built over that one resolution. - /// @param specified_file_system The file system the caller opened the table through. It - /// wins over the access configuration in the schema's options. Null leaves the - /// resolution to that configuration, and to the default local file system when the - /// options name none. Kept so `Copy` and `SwitchToBranch` resolve through it again. - /// @param schema The schema the table was loaded with, a `TableSchema` as every managed - /// table's is. Its primary keys name the kind and its options are what the operators - /// serve. Null is refused: a path a load found no schema under hands back no table, - /// and the failure belongs to the call that opens one. - /// @param branch The branch the table serves, which the managers below read and write on. - /// @param fs_scheme_map The scheme-to-identifier map the caller's contexts resolve custom - /// schemes through, which the options resolve through as well. Kept so `Copy` and - /// `SwitchToBranch` resolve through it again. - /// @param dynamic_options The options a copy was given over the ones its schema stored, which - /// `Copy` accumulates. See the member of the same name. + /// Creates an append-only or primary-key table according to the schema's primary keys. + /// @param specified_file_system Explicit file system, overriding option-based resolution. + /// @param schema Loaded table schema; must not be null. + /// @param branch Branch used by all operators and metadata managers. + /// @param fs_scheme_map URI scheme routing for option-based file system resolution. + /// @param dynamic_options Overrides accumulated by Copy and reused for fallback reads. static Result> Create( const std::shared_ptr& specified_file_system, const std::string& root_path, const std::shared_ptr& schema, const Identifier& identifier, @@ -104,25 +80,21 @@ class FileStoreTable : public Table { return identifier_.GetFullName(); } - /// Returns a copy that serves `dynamic_options`. The root path stays as it is: it names this - /// table, and an option does not rename it. The `type` option is immutable the way Java - /// Paimon's is - a value naming the type the schema's primary keys already name, whatever its - /// case, is dropped, and any other value is refused - and the `branch` option names the branch - /// the copy serves: one on the branch this table serves keeps this table's schema, while one - /// on another branch reads that branch's own latest schema, carries the identifier of that - /// branch the way a load's routing does, and refuses a branch holding no schema. The copy - /// resolves the file system the caller opened this table through, if any, together with its - /// own merged options, and builds its managers over that resolution rather than taking this - /// table's over. + /// Applies dynamic options to the current schema, retaining the table's identifier. Time + /// travel selects the schema of the requested snapshot. Use SwitchToBranch to load another + /// branch's latest schema. Result> Copy( const std::map& dynamic_options) const override; - /// Returns the table of `branch`, as Java Paimon's `switchToBranch` does. The branch switched - /// to is served as it is: its schema is read from its own directory and its managers are built - /// over the file system that schema resolves, so neither this table's dynamic options nor its - /// managers are carried over. Switching to the branch this table already serves hands back a - /// copy of it with the schema shared and the managers built over the same resolution again. A - /// branch with no schema does not exist. + /// Shared schema-copy semantics for managed tables and their system-table views. + static Result> CopyTableSchema( + const std::shared_ptr& fs, const std::string& root_path, + const std::string& current_branch, const std::shared_ptr& schema, + const std::map& applied_options, + const std::map& dynamic_options); + + /// Loads the latest schema from branch and discards this table's dynamic options. + /// Switching to the current branch preserves its effective schema and options. Result> SwitchToBranch(const std::string& branch) const; Result> NewScan(std::unique_ptr context) const override; @@ -136,54 +108,36 @@ class FileStoreTable : public Table { std::unique_ptr context) const override; private: - /// Checks the identity a context carries - the path it names, the identifier it gives a - /// catalog, the table id it records - against this table's own. The store below serves this - /// table's files, so a context naming another table would write this table's metadata and - /// publish it as the other's; a field that contradicts the table is refused by name, the way - /// `CheckContextCarriesNoOptions` refuses an option a context carries. A field the context - /// does not carry is no contradiction: a write or a commit from a bare path names no catalog - /// object. An identifier that names a branch this table does not serve names a branch of - /// another table, as the operator serves the branch the table was loaded with alone. - /// @param kind The operator the context was handed to, which the error names. - /// @param context_table_id The id `CommitContext` records for a catalog commit; a write - /// context carries none, so `NewWrite` passes null. + /// Rejects paths, identifiers or UUIDs that contradict this table's identity. + /// Missing context fields are allowed. + /// @param kind Operation name used in errors. + /// @param context_table_id Catalog commit UUID; nullopt for writes. Status CheckContextNamesThisTable(const std::string& kind, const std::string& context_root_path, const std::optional& context_identifier, const std::optional& context_table_id) const; - /// Builds the read `NewRead` hands back, from the internal context it read the schema into. The - /// path factory comes from the store. The reader is picked by the schema the read serves - the - /// primary keys it carries - not by the table's own kind. + /// Creates a reader for the internal context's schema and primary keys. Result> CreateTableRead( const std::shared_ptr& internal_context, const std::shared_ptr& memory_pool, const std::shared_ptr& executor) const; protected: - /// The base constructor the kinds of managed table build through. The store carries the - /// storage they hand down; `schema` is the schema the table was loaded with, which the `Table` - /// base holds and the operators read. See `Create` for what each piece is. FileStoreTable(const std::shared_ptr& store, const std::shared_ptr& schema, const Identifier& identifier, const std::string& uuid); - /// Restores the committed progress of a real-time write from the latest snapshot the snapshot - /// manager reads. It tells the write where to pick up, and both kinds ask for it the same way. + /// Restores real-time write progress from the latest snapshot. static Status RestoreRealtimeCommittedProgress( const std::shared_ptr& realtime_context, const std::shared_ptr& snapshot_manager, const CoreOptions& options); - /// Returns the scan restricted to the table's root path, which the kinds of file store scan - /// read from. static std::unique_ptr WithTablePath(std::unique_ptr&& scan, const std::string& root_path); - /// Builds the write of this kind of table. `NewWrite` hands the pieces of the shared flow to - /// it. The schema and the branch the write serves are the table's own. - /// @param io_manager The spill manager the context's temp directory names, or null when the - /// context named none. - /// @param ignore_previous_files Whether the write starts over rather than appends, which the - /// flow resolved with this kind's own rules. + /// Creates the table-specific writer. + /// @param io_manager Optional spill manager. + /// @param ignore_previous_files Whether to start over instead of appending. virtual Result> CreateWrite( const WriteContext& ctx, const std::shared_ptr& arrow_schema, const std::shared_ptr& partition_schema, const CoreOptions& options, @@ -191,8 +145,6 @@ class FileStoreTable : public Table { const std::shared_ptr& snapshot_manager, const std::shared_ptr& io_manager, bool ignore_previous_files) const = 0; - /// Builds the scan of this kind of table. `NewScan` hands it the manifest list and the manifest - /// file it built, over the table's own schema and managers. virtual Result> NewFileStoreScan( const std::shared_ptr& manifest_list, const std::shared_ptr& manifest_file, @@ -200,12 +152,9 @@ class FileStoreTable : public Table { const std::shared_ptr& executor, const std::shared_ptr& memory_pool, const ScanContext* context) const = 0; - /// Builds the split generator of this kind of table, Java Paimon's `splitGenerator()`. virtual Result> NewSplitGenerator( const CoreOptions& core_options) const = 0; - /// Builds the manifest entry scan supplier of this kind of table. `NewCommit` hands it the - /// manifest list and the manifest file it built. virtual CommitScanner::ScanSupplier NewCommitScanSupplier( const std::shared_ptr& snapshot_manager, const std::shared_ptr& manifest_list, @@ -214,23 +163,16 @@ class FileStoreTable : public Table { const std::shared_ptr& executor, const std::shared_ptr& pool) const = 0; - /// The storage this table reads and writes through. See `FileStore`. std::shared_ptr store_; - /// The dynamic options the table was loaded with, over the ones its schema stored. `Copy` - /// accumulates them into the copy it derives. They stay separate from the schema they were - /// merged into because a fallback reader needs them on their own: it serves another branch's - /// schema and merges the same options over it again. + /// Accumulated overrides kept separately so fallback reads can apply them to another schema. std::map dynamic_options_; - /// The file system the caller opened this table through, as the options were resolved with - - /// null when the load had none. A copy and a branch switch resolve their own options, and the - /// resolution that built this table's store is one-way: neither the file system it produced - /// nor the options hand the inputs back, so they are kept here for the next resolution. + /// Original explicit file system, or null for option-based resolution; reused by Copy and + /// SwitchToBranch. std::shared_ptr specified_file_system_; - /// The scheme-to-identifier map the options were resolved with, kept for the same next - /// resolution `specified_file_system_` is kept for. + /// Original URI scheme routing, reused by Copy and SwitchToBranch. std::map fs_scheme_map_; }; diff --git a/src/paimon/core/table/file_store_table_factory.cpp b/src/paimon/core/table/file_store_table_factory.cpp index 756273a91..a1bf0895d 100644 --- a/src/paimon/core/table/file_store_table_factory.cpp +++ b/src/paimon/core/table/file_store_table_factory.cpp @@ -69,8 +69,6 @@ Result> ManagedTableOf( latest_schema = *latest; } } - // A path no schema was found under names no table: the failure belongs to the call that - // opens one, rather than to the first operator a handle without a schema would reach. if (latest_schema == nullptr) { return Status::Invalid(fmt::format("not found latest schema in branch {}", BranchManager::NormalizeBranch(branch))); @@ -117,10 +115,28 @@ Result> CatalogMainBranchSchemaJson( } template -Result> ManagedTable(const Context& ctx, const char* kind) { +Result> ManagedTable(const Context& ctx, + const std::map& options, + const char* kind, bool catalog_write) { PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx.GetCatalog())); + if (ctx.GetCatalog() && !ctx.GetIdentifier()) { + return Status::Invalid(fmt::format("a catalog {} requires a table identifier", kind)); + } + PAIMON_ASSIGN_OR_RAISE(std::string branch, + BranchManager::ResolveBranch(ctx.GetIdentifier(), options, kind)); + if (catalog_write && ctx.GetCatalog() && !BranchManager::IsMainBranch(branch)) { + PAIMON_RETURN_NOT_OK(BranchManager::CheckCatalogAddressableBranch(branch)); + PAIMON_ASSIGN_OR_RAISE(std::optional identifier_branch, + ctx.GetIdentifier()->GetBranchName()); + if (!identifier_branch) { + return Status::Invalid( + fmt::format("a {} through a catalog addresses a branch by the table identifier; " + "name branch '{}' in the identifier", + kind, branch)); + } + } PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, - CoreOptions::FromMap(ctx.GetOptions(), ctx.GetSpecificFileSystem(), + CoreOptions::FromMap(options, ctx.GetSpecificFileSystem(), ctx.GetFileSystemSchemeToIdentifierMap())); PAIMON_ASSIGN_OR_RAISE(std::optional system_table_path, SystemTableLoader::TryParsePath(ctx.GetPath())); @@ -131,38 +147,39 @@ Result> ManagedTable(const Context& ctx, const char* kind "table->NewScan/NewRead/NewWrite/NewCommit on it", ctx.GetPath())); } - // A schema the caller gave replaces the catalog's; the caller answers for its correctness. std::optional catalog_table_schema = ctx.GetSpecificTableSchema(); if (!catalog_table_schema) { + PAIMON_ASSIGN_OR_RAISE(std::optional catalog_identifier, + CatalogUtils::BranchIdentifier(ctx.GetIdentifier(), branch)); PAIMON_ASSIGN_OR_RAISE( - std::optional catalog_identifier, - CatalogUtils::BranchIdentifier(ctx.GetIdentifier(), ctx.GetBranch())); - PAIMON_ASSIGN_OR_RAISE(catalog_table_schema, - CatalogMainBranchSchemaJson(ctx.GetCatalog(), catalog_identifier, - ctx.GetBranch(), kind)); + catalog_table_schema, + CatalogMainBranchSchemaJson(ctx.GetCatalog(), catalog_identifier, branch, kind)); } return ManagedTableOf(tmp_options.GetFileSystem(), ctx.GetSpecificFileSystem(), ctx.GetPath(), - ctx.GetBranch(), ctx.GetOptions(), catalog_table_schema, - ctx.GetIdentifier(), ctx.GetTableId().value_or(""), - ctx.GetFileSystemSchemeToIdentifierMap()); + branch, options, catalog_table_schema, ctx.GetIdentifier(), + ctx.GetTableId().value_or(""), ctx.GetFileSystemSchemeToIdentifierMap()); } } // namespace -Result> FileStoreTableFactory::Create(const ScanContext& ctx) { - return ManagedTable(ctx, "scan"); +Result> FileStoreTableFactory::Create( + const ScanContext& ctx, const std::map& options) { + return ManagedTable(ctx, options, "scan", false); } -Result> FileStoreTableFactory::Create(const ReadContext& ctx) { - return ManagedTable(ctx, "read"); +Result> FileStoreTableFactory::Create( + const ReadContext& ctx, const std::map& options) { + return ManagedTable(ctx, options, "read", false); } -Result> FileStoreTableFactory::Create(const WriteContext& ctx) { - return ManagedTable(ctx, "write"); +Result> FileStoreTableFactory::Create( + const WriteContext& ctx, const std::map& options) { + return ManagedTable(ctx, options, "write", true); } -Result> FileStoreTableFactory::Create(const CommitContext& ctx) { - return ManagedTable(ctx, "commit"); +Result> FileStoreTableFactory::Create( + const CommitContext& ctx, const std::map& options) { + return ManagedTable(ctx, options, "commit", true); } } // namespace paimon diff --git a/src/paimon/core/table/file_store_table_factory.h b/src/paimon/core/table/file_store_table_factory.h index 91113a96f..480a09418 100644 --- a/src/paimon/core/table/file_store_table_factory.h +++ b/src/paimon/core/table/file_store_table_factory.h @@ -18,7 +18,9 @@ #pragma once +#include #include +#include #include "paimon/result.h" @@ -30,23 +32,24 @@ class ScanContext; class Table; class WriteContext; -/// Loads the data table a context names, as Java Paimon's `FileStoreTableFactory` does: the schema -/// under the context's path describes it, and a path holding none is refused. A system table -/// behind a `T$system` path and a format table are refused - a catalog hands those back through -/// `GetTable`, and their operators come from the table itself - so the static `Create` of all four -/// operators answers the one question this place asks and they cannot disagree. +/// Loads managed tables for the static operator Create methods. +/// System and format tables must be loaded through Catalog::GetTable(). class FileStoreTableFactory { public: FileStoreTableFactory() = delete; ~FileStoreTableFactory() = delete; - static Result> Create(const ScanContext& context); + static Result> Create(const ScanContext& context, + const std::map& options); - static Result> Create(const ReadContext& context); + static Result> Create(const ReadContext& context, + const std::map& options); - static Result> Create(const WriteContext& context); + static Result> Create(const WriteContext& context, + const std::map& options); - static Result> Create(const CommitContext& context); + static Result> Create(const CommitContext& context, + const std::map& options); }; } // namespace paimon diff --git a/src/paimon/core/table/format/format_table.cpp b/src/paimon/core/table/format/format_table.cpp index b5b41ff5e..f200a85cf 100644 --- a/src/paimon/core/table/format/format_table.cpp +++ b/src/paimon/core/table/format/format_table.cpp @@ -205,9 +205,13 @@ Result> FormatTable::CopySelf( for (const auto& [key, value] : dynamic_options) { options[key] = value; } - return Create(file_system_, root_path_, identifier_, - std::static_pointer_cast(schema_), location_carries_paimon_metadata_, - options); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr copied, + Create(file_system_, root_path_, identifier_, std::static_pointer_cast(schema_), + location_carries_paimon_metadata_, options)); + // Java merges file.format into the option map but retains the physical format of the table. + copied->format_ = format_; + return copied; } Result> FormatTable::Copy( @@ -228,8 +232,6 @@ Result> FormatTable::NewScan( std::unique_ptr context) const { std::shared_ptr shared_context = std::move(context); PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(shared_context, "scan")); - PAIMON_RETURN_NOT_OK( - ContextUtils::CheckContextCarriesNoOptions("scan", shared_context->GetOptions())); std::shared_ptr table = SharedSelf(); if (shared_context->IsStreamingMode()) { return Status::Invalid( @@ -273,8 +275,6 @@ Result> FormatTable::NewRead( std::unique_ptr context) const { std::shared_ptr read_context = std::move(context); PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(read_context, "read")); - PAIMON_RETURN_NOT_OK( - ContextUtils::CheckContextCarriesNoOptions("read", read_context->GetOptions())); if (read_context->HasVectorOrFullTextSearch()) { return Status::NotImplemented( "VectorSearch or FullTextSearch is not supported for format tables"); @@ -290,8 +290,6 @@ Result> FormatTable::NewRead( Result> FormatTable::NewWrite( std::unique_ptr context) const { PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "write")); - PAIMON_RETURN_NOT_OK( - ContextUtils::CheckContextCarriesNoOptions("write", context->GetOptions())); std::shared_ptr table = SharedSelf(); if (context->GetCatalog() != nullptr) { return Status::Invalid( @@ -322,8 +320,6 @@ Result> FormatTable::NewWrite( Result> FormatTable::NewCommit( std::unique_ptr context) const { PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "commit")); - PAIMON_RETURN_NOT_OK( - ContextUtils::CheckContextCarriesNoOptions("commit", context->GetOptions())); std::shared_ptr given_table = SharedSelf(); if (!context->IgnoreEmptyCommit()) { return Status::Invalid( diff --git a/src/paimon/core/table/format/format_table_test.cpp b/src/paimon/core/table/format/format_table_test.cpp index f4c7e6651..21e809680 100644 --- a/src/paimon/core/table/format/format_table_test.cpp +++ b/src/paimon/core/table/format/format_table_test.cpp @@ -763,37 +763,16 @@ TEST(FormatTableTest, TestCatalogManagedPartitionsAreRejected) { ASSERT_TRUE(dynamic_table.status().IsNotImplemented()) << dynamic_table.status().ToString(); } -TEST(FormatTableTest, TestTheTableOperatorsRefuseAContextOption) { - std::unique_ptr dir = UniqueTestDirectory::Create(); +TEST(FormatTableTest, TestCopyValidatesDynamicOptions) { + auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); ASSERT_OK_AND_ASSIGN(std::shared_ptr table, CreateTable(dir->GetFileSystem(), dir->Str(), {"dt"})); - const std::map refused = { - {Options::METASTORE_PARTITIONED_TABLE, "true"}}; - - ScanContextBuilder scan_builder(dir->Str()); - scan_builder.SetOptions(refused); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); - ASSERT_NOK_WITH_MSG(table->NewScan(std::move(scan_context)), - "the scan context carries option 'metastore.partitioned-table'"); - - ReadContextBuilder read_builder(dir->Str()); - read_builder.SetOptions(refused); - ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); - ASSERT_NOK_WITH_MSG(table->NewRead(std::move(read_context)), - "the read context carries option 'metastore.partitioned-table'"); - - WriteContextBuilder write_builder(dir->Str(), "test-user"); - write_builder.SetOptions(refused); - ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); - ASSERT_NOK_WITH_MSG(table->NewWrite(std::move(write_context)), - "the write context carries option 'metastore.partitioned-table'"); - - CommitContextBuilder commit_builder(dir->Str(), "test-user"); - commit_builder.SetOptions(refused); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); - ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(commit_context)), - "the commit context carries option 'metastore.partitioned-table'"); + ASSERT_NOK_WITH_MSG(table->Copy({{Options::METASTORE_PARTITIONED_TABLE, "true"}}), + "is not supported by paimon-cpp yet"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, + ScanContextBuilder(dir->Str()).Finish()); + ASSERT_OK(table->NewScan(std::move(context)).status()); } TEST(FormatTableTest, TestValueOnlyPartitionCannotBeNamedAfterThisTablesMetadata) { @@ -2807,10 +2786,8 @@ TEST(FormatTableTest, TestALoadedFormatTableIsServedThroughItsOwnOperators) { // what kind of table it is. ScanContextBuilder path_builder(data_dir->Str()); ASSERT_OK_AND_ASSIGN(std::unique_ptr path_context, path_builder.Finish()); - ASSERT_NOK(TableScan::Create(std::move(path_context))); + ASSERT_NOK(TableScan::Create(std::move(path_context), {})); - // The way to serve a different value is a copy that carries it, so a file is rolled per - // row rather than one file holding all three. ASSERT_OK_AND_ASSIGN(std::shared_ptr
rolled, external->Copy({{Options::TARGET_FILE_ROW_NUM, "1"}})); WriteContextBuilder write_builder(external->Location(), "test-user"); @@ -2920,13 +2897,13 @@ TEST(FormatTableTest, TestTheTableEntryPointsRefuseWhatAFormatTableCannotHonour) CreateTable(dir->GetFileSystem(), dir->Str(), {"dt"})); std::shared_ptr
table = format_table; - ASSERT_NOK_WITH_MSG(TableScan::Create(nullptr), "scan context is null pointer"); + ASSERT_NOK_WITH_MSG(TableScan::Create(nullptr, {}), "scan context is null pointer"); ASSERT_NOK_WITH_MSG(table->NewScan(nullptr), "scan context is null pointer"); - ASSERT_NOK_WITH_MSG(TableRead::Create(nullptr), "read context is null pointer"); + ASSERT_NOK_WITH_MSG(TableRead::Create(nullptr, {}), "read context is null pointer"); ASSERT_NOK_WITH_MSG(table->NewRead(nullptr), "read context is null pointer"); - ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(nullptr), "write context is null pointer"); + ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(nullptr, {}), "write context is null pointer"); ASSERT_NOK_WITH_MSG(table->NewWrite(nullptr), "write context is null pointer"); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(nullptr), "commit context is null pointer"); + ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(nullptr, {}), "commit context is null pointer"); ASSERT_NOK_WITH_MSG(table->NewCommit(nullptr), "commit context is null pointer"); auto streaming_scan_context = [&dir]() -> Result> { @@ -2935,7 +2912,8 @@ TEST(FormatTableTest, TestTheTableEntryPointsRefuseWhatAFormatTableCannotHonour) return builder.Finish(); }; ASSERT_OK_AND_ASSIGN(std::unique_ptr streaming_context, streaming_scan_context()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(streaming_context)), "names a format table"); + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(streaming_context), {}), + "names a format table"); ASSERT_OK_AND_ASSIGN(std::unique_ptr streaming_context_again, streaming_scan_context()); ASSERT_NOK_WITH_MSG(table->NewScan(std::move(streaming_context_again)), @@ -2950,7 +2928,7 @@ TEST(FormatTableTest, TestTheTableEntryPointsRefuseWhatAFormatTableCannotHonour) return builder.Finish(); }; ASSERT_OK_AND_ASSIGN(std::unique_ptr vector_context, vector_read_context()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(vector_context)), "names a format table"); + ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(vector_context), {}), "names a format table"); ASSERT_OK_AND_ASSIGN(std::unique_ptr vector_context_again, vector_read_context()); ASSERT_NOK_WITH_MSG(table->NewRead(std::move(vector_context_again)), "not supported for format tables"); @@ -2963,7 +2941,8 @@ TEST(FormatTableTest, TestTheTableEntryPointsRefuseWhatAFormatTableCannotHonour) return builder.WithCatalog(catalog, Identifier("db", "tbl")).Finish(); }; ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog_write, catalog_write_context()); - ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(catalog_write)), "names a format table"); + ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(catalog_write), {}), + "names a format table"); ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog_write_again, catalog_write_context()); ASSERT_NOK_WITH_MSG(table->NewWrite(std::move(catalog_write_again)), @@ -2974,7 +2953,8 @@ TEST(FormatTableTest, TestTheTableEntryPointsRefuseWhatAFormatTableCannotHonour) return builder.WithCatalog(catalog, Identifier("db", "fmt")).Finish(); }; ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog_commit, catalog_commit_context()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(catalog_commit)), "names a format table"); + ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(catalog_commit), {}), + "names a format table"); ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog_commit_again, catalog_commit_context()); ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(catalog_commit_again)), @@ -2986,7 +2966,7 @@ TEST(FormatTableTest, TestTheTableEntryPointsRefuseWhatAFormatTableCannotHonour) return builder.Finish(); }; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_id, write_id_context()); - ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_id)), "names a format table"); + ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_id), {}), "names a format table"); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_id_again, write_id_context()); ASSERT_NOK_WITH_MSG(table->NewWrite(std::move(write_id_again)), "a write id would name nothing"); @@ -3000,19 +2980,21 @@ TEST(FormatTableTest, TestTheGenericEntryPointsRefuseAFormatTablePath) { ScanContextBuilder scan_builder(dir->Str()); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), "names a format table"); + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context), {}), "names a format table"); ReadContextBuilder read_builder(dir->Str()); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), "names a format table"); + ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context), {}), "names a format table"); WriteContextBuilder write_builder(dir->Str(), "test-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_context)), "names a format table"); + ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_context), {}), + "names a format table"); CommitContextBuilder commit_builder(dir->Str(), "test-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(commit_context)), "names a format table"); + ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(commit_context), {}), + "names a format table"); ScanContextBuilder served_builder(dir->Str()); ASSERT_OK_AND_ASSIGN(std::unique_ptr served_context, served_builder.Finish()); @@ -3022,7 +3004,7 @@ TEST(FormatTableTest, TestTheGenericEntryPointsRefuseAFormatTablePath) { ASSERT_EQ(plan->Splits().size(), 0u); } -TEST(FormatTableTest, TestABranchNamedThroughOptionsIsRefusedByTheTableOperators) { +TEST(FormatTableTest, TestBranchOptionsAreRefusedByFormatTables) { std::unique_ptr dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); ASSERT_OK_AND_ASSIGN(std::shared_ptr table, @@ -3037,32 +3019,6 @@ TEST(FormatTableTest, TestABranchNamedThroughOptionsIsRefusedByTheTableOperators FormatTable::Create(dir->GetFileSystem(), dir->Str(), Identifier("db", "tbl"), branch), "has no branches"); - ScanContextBuilder scan_builder(dir->Str()); - scan_builder.SetOptions(branch); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); - ASSERT_NOK_WITH_MSG(table->NewScan(std::move(scan_context)), - "the scan context carries option 'branch'"); - - ReadContextBuilder read_builder(dir->Str()); - read_builder.SetOptions(branch); - ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); - ASSERT_NOK_WITH_MSG(table->NewRead(std::move(read_context)), - "the read context carries option 'branch'"); - - WriteContextBuilder write_builder(dir->Str(), "test-user"); - write_builder.SetOptions(branch); - ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); - ASSERT_NOK_WITH_MSG(table->NewWrite(std::move(write_context)), - "the write context carries option 'branch'"); - - // The commit is the one that matters most: an overwrite clears what it replaces before it - // publishes anything, so the refusal has to come before the deleting does. - CommitContextBuilder commit_builder(dir->Str(), "test-user"); - commit_builder.SetOptions(branch); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); - ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(commit_context)), - "the commit context carries option 'branch'"); - ASSERT_OK_AND_ASSIGN( std::unique_ptr scan, FormatTableScan::Create(table, /*partition_filter=*/{}, /*limit=*/std::nullopt)); @@ -3095,6 +3051,41 @@ TEST(FormatTableTest, TestATableCopiesItselfWithTheOptionsGivenAtTheCall) { ASSERT_NOK_WITH_MSG(table->Copy({{Options::BRANCH, "dev"}}), "has no branches"); } +TEST(FormatTableTest, TestCopyRetainsPhysicalFileFormat) { + for (const std::string& format : {std::string("parquet"), std::string("orc")}) { + SCOPED_TRACE(format); + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN(std::shared_ptr original, + CreateTable(dir->GetFileSystem(), dir->Str(), {"dt"}, + {{Options::FILE_FORMAT, format}})); + ASSERT_OK_AND_ASSIGN(std::unique_ptr first, + MakeBatch({1}, {"alice"}, "20240101", {{"dt", "20240101"}})); + ASSERT_OK(WriteAndCommit(original, std::move(first))); + const std::string hinted_format = format == "orc" ? "parquet" : "orc"; + ASSERT_OK_AND_ASSIGN(std::shared_ptr
copy, + original->Copy({{Options::FILE_FORMAT, hinted_format}})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
another, + copy->Copy({{Options::READ_BATCH_SIZE, "1"}})); + auto copied = std::dynamic_pointer_cast(another); + ASSERT_TRUE(copied); + EXPECT_EQ(copied->GetFormat(), original->GetFormat()); + EXPECT_EQ(copied->Options().at(Options::FILE_FORMAT), hinted_format); + ASSERT_OK_AND_ASSIGN(std::unique_ptr second, + MakeBatch({2}, {"bob"}, "20240101", {{"dt", "20240101"}})); + ASSERT_OK(WriteAndCommit(copied, std::move(second))); + for (const auto& table : {original, copied}) { + ASSERT_OK_AND_ASSIGN( + std::unique_ptr scan, + FormatTableScan::Create(table, /*partition_filter=*/{}, /*limit=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_OK_AND_ASSIGN(std::vector rows, ReadAll(table, plan->Splits())); + std::sort(rows.begin(), rows.end()); + EXPECT_EQ(rows, (std::vector{"1|alice|20240101", "2|bob|20240101"})); + } + } +} + TEST(FormatTableTest, TestAFormatTableSchemaOnABranchIsRefusedRatherThanServedFromTheTableRoot) { std::unique_ptr dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); @@ -3111,17 +3102,23 @@ TEST(FormatTableTest, TestAFormatTableSchemaOnABranchIsRefusedRatherThanServedFr MakeSchema(), /*partition_keys=*/{"dt"}, /*primary_keys=*/{}, {{Options::TYPE, "format-table"}, {Options::FILE_FORMAT, "parquet"}})); + std::map read_builder_options; ReadContextBuilder read_builder(dir->Str()); - read_builder.WithBranch("dev"); + read_builder_options[Options::BRANCH] = "dev"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), "names a format table"); + ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context), read_builder_options), + "names a format table"); // A scan takes its branch from the option rather than from a `WithBranch()` of its own, which // is the same branch by another name and gets the same answer. + std::map scan_builder_options; ScanContextBuilder scan_builder(dir->Str()); - scan_builder.SetOptions({{Options::BRANCH, "dev"}}); + scan_builder_options = {{Options::BRANCH, "dev"}}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), "names a format table"); + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context), scan_builder_options), + "names a format table"); ScanContextBuilder main_builder(dir->Str()); ASSERT_OK_AND_ASSIGN(std::unique_ptr main_context, main_builder.Finish()); @@ -3140,11 +3137,15 @@ TEST(FormatTableTest, TestTheTableOperatorsCarryTheContextThrough) { // `target-file-row-num` is stored nowhere in this table's schema, so a file would hold every // row. Given at the call it has to roll a file per row, as it does for a managed table. + std::map write_builder_options; WriteContextBuilder write_builder(dir->Str(), "test-user"); - write_builder.SetOptions({{Options::TARGET_FILE_ROW_NUM, "1"}}); + write_builder_options = {{Options::TARGET_FILE_ROW_NUM, "1"}}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_23, + table->Copy(write_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr write, - table->NewWrite(std::move(write_context))); + configured_table_23->NewWrite(std::move(write_context))); for (int32_t i = 0; i < 3; i++) { ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, MakeBatch({i}, {"name"}, "20240101", {})); @@ -3162,19 +3163,25 @@ TEST(FormatTableTest, TestTheTableOperatorsCarryTheContextThrough) { ASSERT_OK(generic_commit->Commit(messages)); ASSERT_OK(write->Close()); + std::map branch_builder_options; ReadContextBuilder branch_builder(dir->Str()); - branch_builder.WithBranch("nosuchbranch"); + branch_builder_options[Options::BRANCH] = "nosuchbranch"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr branch_context, branch_builder.Finish()); - ASSERT_NOK(TableRead::Create(std::move(branch_context))); + ASSERT_NOK(TableRead::Create(std::move(branch_context), branch_builder_options)); // `type` is structural, so an option given at the call must not decide what kind of table // this is. Checked by reading the rows back rather than by counting splits: three small files // are packed into one split, so a split count says nothing about how many there are. + std::map retyped_builder_options; ScanContextBuilder retyped_builder(dir->Str()); - retyped_builder.SetOptions({{Options::TYPE, "table"}}); + retyped_builder_options = {{Options::TYPE, "table"}}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr retyped_context, retyped_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_24, + table->Copy(retyped_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr retyped_scan, - table->NewScan(std::move(retyped_context))); + configured_table_24->NewScan(std::move(retyped_context))); ASSERT_OK_AND_ASSIGN(std::shared_ptr retyped_plan, retyped_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(std::vector retyped_rows, ReadAll(table, retyped_plan->Splits())); diff --git a/src/paimon/core/table/primary_key_file_store_table.h b/src/paimon/core/table/primary_key_file_store_table.h index 121081735..8ab8c05cf 100644 --- a/src/paimon/core/table/primary_key_file_store_table.h +++ b/src/paimon/core/table/primary_key_file_store_table.h @@ -34,15 +34,9 @@ class IOManager; class SplitGenerator; class TableSchema; -/// The managed table a schema with primary keys describes, Java Paimon's -/// `PrimaryKeyFileStoreTable`. Its write merges by key, and its scan, read and commit read -/// through the merge. The operator flows are the ones `FileStoreTable` carries; the pieces -/// those flows hand down are built here. +/// Managed table with primary keys; uses the shared FileStoreTable operator flows. class PrimaryKeyFileStoreTable : public FileStoreTable { public: - /// The constructor `FileStoreTable::Create` builds a primary key table through: the store - /// carries the storage state, `schema` is the schema the table was loaded with, and the base - /// class holds both. PrimaryKeyFileStoreTable(const std::shared_ptr& store, const std::shared_ptr& schema, const Identifier& identifier, const std::string& uuid); diff --git a/src/paimon/core/table/sink/commit_message_test.cpp b/src/paimon/core/table/sink/commit_message_test.cpp index 8fd31c4c8..de6d5da23 100644 --- a/src/paimon/core/table/sink/commit_message_test.cpp +++ b/src/paimon/core/table/sink/commit_message_test.cpp @@ -1230,16 +1230,18 @@ TEST(CommitMessageTest, TestSerialize) { /*primary_keys=*/{}, options, /*ignore_if_exists=*/false)); + std::map context_builder_options; WriteContextBuilder context_builder(PathUtil::JoinPath(dir->Str(), "foo.db/bar"), "commit_user1"); + context_builder_options[Options::FILE_FORMAT] = "mock_format"; + context_builder_options[Options::TARGET_FILE_SIZE] = "1024"; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.AddOption(Options::FILE_FORMAT, "mock_format") - .AddOption(Options::TARGET_FILE_SIZE, "1024") - .Finish()); + + context_builder.Finish()); std::string root_path = write_context->GetRootPath(); ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(std::move(write_context), context_builder_options)); for (size_t i = 0; i < 10240; i++) { auto f0_array = std::make_shared(); diff --git a/src/paimon/core/table/source/append_count_reader_test.cpp b/src/paimon/core/table/source/append_count_reader_test.cpp index 3b191a3d0..82bf53eef 100644 --- a/src/paimon/core/table/source/append_count_reader_test.cpp +++ b/src/paimon/core/table/source/append_count_reader_test.cpp @@ -51,14 +51,16 @@ class AppendCountReaderTest : public testing::Test { protected: Result>> CreateSplits( const std::string& table_path, const std::optional& snapshot_id) { + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); if (snapshot_id.has_value()) { - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, - std::to_string(snapshot_id.value())); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = + std::to_string(snapshot_id.value()); } PAIMON_ASSIGN_OR_RAISE(auto scan_context, scan_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context), + scan_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto plan, table_scan->CreatePlan()); if (snapshot_id.has_value() && (!plan->SnapshotId().has_value() || plan->SnapshotId().value() != snapshot_id)) { @@ -136,7 +138,7 @@ TEST_F(AppendCountReaderTest, TestAppendOnlyTableReadCreateCountReaderPredicateN ReadContextBuilder read_context_builder(table_path); read_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context), {})); ASSERT_OK_AND_ASSIGN(auto splits, CreateSplits(table_path, /*snapshot_id=*/5)); ASSERT_NOK_WITH_MSG(table_read->CreateCountReader(splits), diff --git a/src/paimon/core/table/source/pk_count_reader_test.cpp b/src/paimon/core/table/source/pk_count_reader_test.cpp index 39c0b6b50..feee28344 100644 --- a/src/paimon/core/table/source/pk_count_reader_test.cpp +++ b/src/paimon/core/table/source/pk_count_reader_test.cpp @@ -51,14 +51,16 @@ class PKCountReaderTest : public testing::Test { protected: Result>> CreateSplits( const std::string& table_path, const std::optional& snapshot_id) { + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); if (snapshot_id.has_value()) { - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, - std::to_string(snapshot_id.value())); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = + std::to_string(snapshot_id.value()); } PAIMON_ASSIGN_OR_RAISE(auto scan_context, scan_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context), + scan_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto plan, table_scan->CreatePlan()); if (snapshot_id.has_value()) { if (!plan->SnapshotId().has_value() || plan->SnapshotId().value() != snapshot_id) { @@ -78,7 +80,7 @@ class PKCountReaderTest : public testing::Test { PAIMON_ASSIGN_OR_RAISE(auto table_schema, schema_manager.ReadSchema(0)); PAIMON_ASSIGN_OR_RAISE( auto internal_context, - InternalReadContext::Create(read_context, table_schema, table_schema->Options())); + InternalReadContext::Create(read_context, table_schema, table_schema->Options(), {})); return std::shared_ptr(std::move(internal_context)); } diff --git a/src/paimon/core/table/source/realtime_table_scan.h b/src/paimon/core/table/source/realtime_table_scan.h index 318319139..46227d327 100644 --- a/src/paimon/core/table/source/realtime_table_scan.h +++ b/src/paimon/core/table/source/realtime_table_scan.h @@ -41,9 +41,7 @@ class TableSchema; /// Adds process-local memory splits to a normal data-table batch scan. class RealtimeTableScan : public TableScan { public: - /// The checks a scan that asks for a real-time union read opens with, run before one of these - /// is built: the table has to be able to serve it, and the options have to name a scan one - /// can be served over. A context that asks for none of it passes straight through. + /// Validates table and scan options for real-time union reads; a no-op for ordinary scans. static Status ValidateScan(const TableSchema& table_schema, const CoreOptions& core_options, const ScanContext& context, bool read_optimized); diff --git a/src/paimon/core/table/source/snapshot/static_from_tag_starting_scanner.h b/src/paimon/core/table/source/snapshot/static_from_tag_starting_scanner.h index a10b11b85..3229a76ef 100644 --- a/src/paimon/core/table/source/snapshot/static_from_tag_starting_scanner.h +++ b/src/paimon/core/table/source/snapshot/static_from_tag_starting_scanner.h @@ -36,7 +36,8 @@ class StaticFromTagStartingScanner : public StartingScanner { Result> Scan( const std::shared_ptr& snapshot_reader) override { - const TagManager tag_manager(snapshot_manager_->Fs(), snapshot_manager_->RootPath()); + const TagManager tag_manager(snapshot_manager_->Fs(), snapshot_manager_->RootPath(), + snapshot_manager_->Branch()); PAIMON_ASSIGN_OR_RAISE(const Tag tag, tag_manager.GetOrThrow(tag_name_)); PAIMON_ASSIGN_OR_RAISE(const Snapshot snapshot, tag.TrimToSnapshot()); PAIMON_ASSIGN_OR_RAISE( diff --git a/src/paimon/core/table/source/table_read.cpp b/src/paimon/core/table/source/table_read.cpp index 2ccbe142b..648713628 100644 --- a/src/paimon/core/table/source/table_read.cpp +++ b/src/paimon/core/table/source/table_read.cpp @@ -25,12 +25,11 @@ namespace paimon { -Result> TableRead::Create(std::unique_ptr ctx) { +Result> TableRead::Create( + std::unique_ptr ctx, const std::map& options) { PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "read")); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, FileStoreTableFactory::Create(*ctx)); - // The load folded the context's options into the table, so the operator below serves them - // from there and the context carries none of its own. - ctx->ClearOptions(); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, + FileStoreTableFactory::Create(*ctx, options)); return table->NewRead(std::move(ctx)); } diff --git a/src/paimon/core/table/source/table_read_test.cpp b/src/paimon/core/table/source/table_read_test.cpp index c7724529b..8659f871a 100644 --- a/src/paimon/core/table/source/table_read_test.cpp +++ b/src/paimon/core/table/source/table_read_test.cpp @@ -61,7 +61,7 @@ TEST(TableReadTest, TestReadWithInvalidContext) { ReadContextBuilder context_builder(path); context_builder.SetReadFieldNames({"f0", "f1", "non-exist"}); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), + ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context), {}), "Get field non-exist failed: not exist in table schema"); } { @@ -72,7 +72,7 @@ TEST(TableReadTest, TestReadWithInvalidContext) { context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); ASSERT_NOK_WITH_MSG( - TableRead::Create(std::move(read_context)), + TableRead::Create(std::move(read_context), {}), "field f3 has field type BIGINT in literal, mismatch field type DOUBLE in predicate"); } { @@ -82,7 +82,7 @@ TEST(TableReadTest, TestReadWithInvalidContext) { ReadContextBuilder context_builder(path); context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), + ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context), {}), "schema type double mismatches predicate field type BIGINT"); } { @@ -94,7 +94,7 @@ TEST(TableReadTest, TestReadWithInvalidContext) { context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); ASSERT_NOK_WITH_MSG( - TableRead::Create(std::move(read_context)), + TableRead::Create(std::move(read_context), {}), "field f3 has field idx 0 in input schema, mismatch field idx 2 in predicate"); } { @@ -104,7 +104,7 @@ TEST(TableReadTest, TestReadWithInvalidContext) { ReadContextBuilder context_builder(path); context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), + ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context), {}), "literal cannot be null in predicate, field name f3"); } { @@ -112,7 +112,7 @@ TEST(TableReadTest, TestReadWithInvalidContext) { ReadContextBuilder context_builder(path); context_builder.SetReadFieldNames({"f3", "f1", "f3"}); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), + ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context), {}), "validate schema failed: read schema has duplicate field f3"); } } @@ -122,7 +122,7 @@ TEST(TableReadTest, TestReadWithSpecifiedInvalidSchema) { ReadContextBuilder context_builder(path); context_builder.SetReadFieldNames({"field_no_exist"}); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), + ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context), {}), "Get field field_no_exist failed: not exist in table schema"); } @@ -145,14 +145,14 @@ TEST(TableReadTest, TestFullTextMatchExecutionIsNotImplemented) { builder.SetReadFieldNames({"f0", "f1"}).SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); auto result = table_entry ? table->NewRead(std::move(context)) - : TableRead::Create(std::move(context)); + : TableRead::Create(std::move(context), {}); ASSERT_NOK_WITH_MSG(result, "FullTextMatch predicate execution is not implemented yet"); ASSERT_TRUE(result.status().IsNotImplemented()); builder.SetReadFieldNames({"f0", "f1", "_f0_INDEX_SCORE"}).SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(context, builder.Finish()); result = table_entry ? table->NewRead(std::move(context)) - : TableRead::Create(std::move(context)); + : TableRead::Create(std::move(context), {}); ASSERT_NOK_WITH_MSG(result, "Score-producing FullTextMatch cannot appear under OR"); } } @@ -174,7 +174,7 @@ TEST(TableReadTest, TestPerFieldScoreExecutionIsNotImplemented) { builder.SetFullTextSearch(search).SetReadFieldNames({"f1", "_f0_INDEX_SCORE"}); ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); auto result = table_entry ? table->NewRead(std::move(context)) - : TableRead::Create(std::move(context)); + : TableRead::Create(std::move(context), {}); ASSERT_NOK_WITH_MSG(result, "Per-field index score output is not implemented yet"); ASSERT_TRUE(result.status().IsNotImplemented()); } @@ -183,12 +183,16 @@ TEST(TableReadTest, TestPerFieldScoreExecutionIsNotImplemented) { TEST(TableReadTest, TestCreateKeyValueTableRead) { std::string path = paimon::test::GetDataDir() + "/orc/pk_table_with_dv_cardinality.db/pk_table_with_dv_cardinality/"; + std::map context_builder_options; ReadContextBuilder context_builder(path); context_builder.SetReadFieldNames({"f0", "f1", "f2", "f3"}); - context_builder.AddOption("read.batch-size", "2"); - context_builder.AddOption("orc.read.enable-lazy-decoding", "true"); + context_builder_options["read.batch-size"] = "2"; + + context_builder_options["orc.read.enable-lazy-decoding"] = "true"; + ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); auto key_value_table_read = dynamic_cast(table_read.get()); ASSERT_TRUE(key_value_table_read); } @@ -201,7 +205,7 @@ TEST(TableReadTest, TestCreateCountReaderRejectsFileIndexSearch) { ReadContextBuilder context_builder(path); context_builder.SetReadFieldNames({"f0"}).SetFullTextSearch(search); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context), {})); ASSERT_NOK_WITH_MSG(table_read->CreateCountReader({}), "VectorSearch or FullTextSearch"); } { @@ -213,7 +217,7 @@ TEST(TableReadTest, TestCreateCountReaderRejectsFileIndexSearch) { ReadContextBuilder context_builder(path); context_builder.SetReadFieldNames({"f0"}).SetVectorSearch(search); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context), {})); ASSERT_NOK_WITH_MSG(table_read->CreateCountReader({}), "VectorSearch or FullTextSearch"); } } @@ -227,7 +231,8 @@ TEST(TableReadTest, TestPrimaryKeyFileIndexSearchDoesNotFallBackToMergeReader) { ReadContextBuilder context_builder(path); context_builder.SetReadFieldNames({"f0"}).SetVectorSearch(search); ASSERT_OK_AND_ASSIGN(std::unique_ptr context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, TableRead::Create(std::move(context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, + TableRead::Create(std::move(context), {})); BinaryRow partition = BinaryRowGenerator::GenerateRow({10}, GetDefaultPool().get()); DataSplitImpl::Builder builder(partition, /*bucket=*/0, /*bucket_path=*/"", /*data_files=*/{}); @@ -247,12 +252,16 @@ TEST(TableReadTest, TestPrimaryKeyFileIndexSearchDoesNotFallBackToMergeReader) { TEST(TableReadTest, TestCreateAppendOnlyTableRead) { std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; + std::map context_builder_options; ReadContextBuilder context_builder(path); context_builder.SetReadFieldNames({"f0", "f1", "f2", "f3"}); - context_builder.AddOption("read.batch-size", "2"); - context_builder.AddOption("orc.read.enable-lazy-decoding", "true"); + context_builder_options["read.batch-size"] = "2"; + + context_builder_options["orc.read.enable-lazy-decoding"] = "true"; + ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); auto append_only_table_read = dynamic_cast(table_read.get()); ASSERT_TRUE(append_only_table_read); } @@ -260,12 +269,16 @@ TEST(TableReadTest, TestCreateAppendOnlyTableRead) { TEST(TableReadTest, TestMergeOptions) { std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; { + std::map context_builder_options; ReadContextBuilder context_builder(path); context_builder.SetReadFieldNames({"f0", "f1", "f2", "f3"}); - context_builder.AddOption("read.batch-size", "2"); - context_builder.AddOption("orc.read.enable-lazy-decoding", "true"); + context_builder_options["read.batch-size"] = "2"; + + context_builder_options["orc.read.enable-lazy-decoding"] = "true"; + ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); auto append_only_table_read = dynamic_cast(table_read.get()); ASSERT_TRUE(append_only_table_read); auto* typed_split_read = @@ -282,11 +295,14 @@ TEST(TableReadTest, TestMergeOptions) { ASSERT_EQ(expected_options, core_options.ToMap()); } { + std::map context_builder_options; ReadContextBuilder context_builder(path); context_builder.SetReadFieldNames({"f0", "f1", "f2", "f3"}); - context_builder.AddOption("bucket", "10"); + context_builder_options["bucket"] = "10"; + ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); auto append_only_table_read = dynamic_cast(table_read.get()); ASSERT_TRUE(append_only_table_read); auto* typed_split_read = @@ -307,15 +323,16 @@ TEST(TableReadTest, TestDependencyReadWithoutCatalogUsesReadOptions) { ASSERT_OK_AND_ASSIGN(std::string schema_json, table_schema->ToJsonString()); for (bool explicit_header : {false, true}) { + std::map builder_options; ReadContextBuilder builder("warehouse/db.db/table"); builder.SetTableSchema(schema_json); if (explicit_header) { - builder.AddOption(read_via_option, "read-view"); + builder_options[read_via_option] = "read-view"; } ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, builder.Finish()); // Exercise the actual table/read merge before creating the dependency context. ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(std::move(read_context), builder_options)); auto* append_only_table_read = dynamic_cast(table_read.get()); ASSERT_NE(append_only_table_read, nullptr); const auto& internal_context = append_only_table_read->context_; diff --git a/src/paimon/core/table/source/table_scan.cpp b/src/paimon/core/table/source/table_scan.cpp index 14c78ab3b..9684d609f 100644 --- a/src/paimon/core/table/source/table_scan.cpp +++ b/src/paimon/core/table/source/table_scan.cpp @@ -27,12 +27,11 @@ namespace paimon { -Result> TableScan::Create(std::unique_ptr context) { +Result> TableScan::Create( + std::unique_ptr context, const std::map& options) { PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "scan")); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, FileStoreTableFactory::Create(*context)); - // The load folded the context's options into the table, so the operator below serves them - // from there and the context carries none of its own. - context->ClearOptions(); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, + FileStoreTableFactory::Create(*context, options)); return table->NewScan(std::move(context)); } diff --git a/src/paimon/core/table/source/table_scan_test.cpp b/src/paimon/core/table/source/table_scan_test.cpp index 4fd0f9873..59cae2906 100644 --- a/src/paimon/core/table/source/table_scan_test.cpp +++ b/src/paimon/core/table/source/table_scan_test.cpp @@ -63,10 +63,12 @@ TEST(TableScanTest, TestDefaultMetricsSnapshot) { TEST(TableScanTest, TestNoSnapshot) { std::string path = paimon::test::GetDataDir() + "/orc/append_table_with_nested_type.db/append_table_with_nested_type/"; + std::map builder_options; ScanContextBuilder builder(path); - builder.AddOption(Options::FILE_FORMAT, "orc"); + builder_options[Options::FILE_FORMAT] = "orc"; + ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(context), builder_options)); ASSERT_OK_AND_ASSIGN(auto plan, table_scan->CreatePlan()); ASSERT_FALSE(plan->SnapshotId()); ASSERT_TRUE(plan->Splits().empty()); @@ -74,23 +76,27 @@ TEST(TableScanTest, TestNoSnapshot) { TEST(TableScanTest, TestNonExistTable) { std::string path = paimon::test::GetDataDir() + "/non-exist.db/non-exist/"; + std::map builder_options; ScanContextBuilder builder(path); - builder.AddOption(Options::FILE_FORMAT, "orc"); + builder_options[Options::FILE_FORMAT] = "orc"; + ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(context)), "not found latest schema"); + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(context), builder_options), + "not found latest schema"); } TEST(TableScanTest, TestStreamingSnapshotsRemoved) { auto dir = UniqueTestDirectory::Create("local"); ASSERT_TRUE(dir); ASSERT_TRUE(TestUtil::CopyDirectory(GetDataDir() + "/orc/append_09.db/append_09", dir->Str())); + std::map builder_options; ScanContextBuilder builder(dir->Str()); - builder.AddOption(Options::FILE_FORMAT, "orc") - .AddOption(Options::SCAN_MODE, "latest") - .WithStreamingMode(true); + builder_options[Options::FILE_FORMAT] = "orc"; + builder_options[Options::SCAN_MODE] = "latest"; + builder.WithStreamingMode(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(std::move(context))); + TableScan::Create(std::move(context), builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr initial_plan, table_scan->CreatePlan()); ASSERT_TRUE(initial_plan->Splits().empty()); ASSERT_OK_AND_ASSIGN(std::shared_ptr waiting_plan, table_scan->CreatePlan()); @@ -106,14 +112,15 @@ TEST(TableScanTest, TestStreamingNextSnapshotBoundary) { std::string path = GetDataDir() + "/orc/append_09.db/append_09"; for (const std::string next_snapshot_id : {"6", "7"}) { SCOPED_TRACE(next_snapshot_id); + std::map builder_options; ScanContextBuilder builder(path); - builder.AddOption(Options::FILE_FORMAT, "orc") - .AddOption(Options::SCAN_MODE, "from-snapshot") - .AddOption(Options::SCAN_SNAPSHOT_ID, next_snapshot_id) - .WithStreamingMode(true); + builder_options[Options::FILE_FORMAT] = "orc"; + builder_options[Options::SCAN_MODE] = "from-snapshot"; + builder_options[Options::SCAN_SNAPSHOT_ID] = next_snapshot_id; + builder.WithStreamingMode(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(std::move(context))); + TableScan::Create(std::move(context), builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr initial_plan, table_scan->CreatePlan()); ASSERT_TRUE(initial_plan->Splits().empty()); @@ -134,10 +141,12 @@ TEST(TableScanTest, TestStreamingNextSnapshotBoundary) { TEST(TableScanTest, TestPkSchemaEvolutionScan) { std::string path = paimon::test::GetDataDir() + "/orc/pk_table_with_alter_table.db/pk_table_with_alter_table/"; + std::map builder_options; ScanContextBuilder builder(path); - builder.AddOption(Options::FILE_FORMAT, "orc"); + builder_options[Options::FILE_FORMAT] = "orc"; + ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(context), builder_options)); ASSERT_OK_AND_ASSIGN(auto plan, table_scan->CreatePlan()); ASSERT_TRUE(plan->SnapshotId()); ASSERT_FALSE(plan->Splits().empty()); @@ -172,12 +181,15 @@ TEST(TableScanTest, TestReadOptimizedPrimaryKeyStreamingScanUnsupported) { ASSERT_OK_AND_ASSIGN( std::shared_ptr table, SystemTableLoader::LoadFromPath(core_options.GetFileSystem(), path, options)); + std::map builder_options; ScanContextBuilder builder(path); - builder.AddOption(Options::FILE_FORMAT, "orc"); + builder_options[Options::FILE_FORMAT] = "orc"; + builder.WithStreamingMode(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); - ASSERT_NOK_WITH_MSG(table->NewScan(std::move(context)), + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_1, table->Copy(builder_options)); + ASSERT_NOK_WITH_MSG(configured_table_1->NewScan(std::move(context)), "read-optimized system table does not support streaming scan for primary " "key table"); } diff --git a/src/paimon/core/table/system/audit_log_system_table.cpp b/src/paimon/core/table/system/audit_log_system_table.cpp index e86ca34ca..4da5a803a 100644 --- a/src/paimon/core/table/system/audit_log_system_table.cpp +++ b/src/paimon/core/table/system/audit_log_system_table.cpp @@ -42,6 +42,7 @@ #include "paimon/common/utils/checked_cast.h" #include "paimon/core/core_options.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/file_store_table.h" #include "paimon/core/table/source/data_split_impl.h" #include "paimon/core/table/source/key_value_table_read.h" #include "paimon/core/utils/context_utils.h" @@ -401,8 +402,7 @@ Result> AuditLogSystemTable::NewScan( if (data_table_fs == nullptr && context->GetFileSystemSchemeToIdentifierMap().empty()) { data_table_fs = GetFileSystem(); } - builder.SetOptions(options_) - .WithStreamingMode(context->IsStreamingMode()) + builder.WithStreamingMode(context->IsStreamingMode()) .WithMemoryPool(context->GetMemoryPool()) .WithExecutor(context->GetExecutor()) .WithFileSystem(data_table_fs) @@ -420,7 +420,11 @@ Result> AuditLogSystemTable::NewScan( builder.SetLimit(context->GetLimit().value()); } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr data_context, builder.Finish()); - return TableScan::Create(std::move(data_context)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr data_table, + CreateDataTable(table_schema_, options_, data_context->GetSpecificFileSystem(), + data_context->GetFileSystemSchemeToIdentifierMap())); + return data_table->NewScan(std::move(data_context)); } Result> AuditLogSystemTable::NewRead( @@ -446,8 +450,12 @@ Result> AuditLogSystemTable::NewChangelogRead( PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(read_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr data_context, CreateDataReadContext(context, read_options)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr data_table, + CreateDataTable(table_schema_, read_options, data_context->GetSpecificFileSystem(), + data_context->GetFileSystemSchemeToIdentifierMap())); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr data_read, - TableRead::Create(std::move(data_context))); + data_table->NewRead(std::move(data_context))); auto* key_value_read = dynamic_cast(data_read.get()); if (!key_value_read) { return Status::Invalid("audit_log system table requires key-value table read"); @@ -470,9 +478,7 @@ Result> AuditLogSystemTable::CreateDataReadContext( if (data_table_fs == nullptr && context->GetFileSystemSchemeToIdentifierMap().empty()) { data_table_fs = GetFileSystem(); } - builder.SetOptions(read_options) - .SetReadFieldNames(base_read_schema->field_names()) - .WithBranch(core_options.GetBranch()) + builder.SetReadFieldNames(base_read_schema->field_names()) .WithMemoryPool(context->GetMemoryPool()) .WithExecutor(context->GetExecutor()) .WithFileSystem(data_table_fs) diff --git a/src/paimon/core/table/system/audit_log_system_table.h b/src/paimon/core/table/system/audit_log_system_table.h index 3aad57a8e..bc15c385f 100644 --- a/src/paimon/core/table/system/audit_log_system_table.h +++ b/src/paimon/core/table/system/audit_log_system_table.h @@ -54,9 +54,7 @@ class AuditLogSystemTable : public SystemTable { public: static constexpr const char* kName = "audit_log"; - /// @param data_table_name Logical object name of the data table underneath, branch included, - /// which this view answers `$audit_log` with - the name a catalog - /// knows the data table by, not the name of the directory it lives in. + /// @param data_table_name Logical table name, including any branch suffix. AuditLogSystemTable(std::shared_ptr fs, std::string table_path, std::shared_ptr table_schema, std::string data_table_name, std::map options); @@ -66,8 +64,7 @@ class AuditLogSystemTable : public SystemTable { Result> NewRead(std::unique_ptr context) const override; protected: - /// For a derived changelog view that answers with a kind of its own: `BinlogSystemTable` names - /// itself `T$binlog`, not the `T$audit_log` the public constructor composes. + /// Constructor for derived changelog views with a different system-table name. AuditLogSystemTable(std::shared_ptr fs, std::string table_path, std::shared_ptr table_schema, std::map options, std::string name); diff --git a/src/paimon/core/table/system/binlog_system_table.h b/src/paimon/core/table/system/binlog_system_table.h index 5d486e457..fcdcb84b6 100644 --- a/src/paimon/core/table/system/binlog_system_table.h +++ b/src/paimon/core/table/system/binlog_system_table.h @@ -36,9 +36,7 @@ class BinlogSystemTable : public AuditLogSystemTable { public: static constexpr const char* kName = "binlog"; - /// @param data_table_name Logical object name of the data table underneath, branch included, - /// which this view answers `$binlog` with - the name a catalog knows - /// the data table by, not the name of the directory it lives in. + /// @param data_table_name Logical table name, including any branch suffix. BinlogSystemTable(std::shared_ptr fs, std::string table_path, std::shared_ptr table_schema, std::string data_table_name, std::map options); diff --git a/src/paimon/core/table/system/global_system_tables.cpp b/src/paimon/core/table/system/global_system_tables.cpp index d86849d47..c04900638 100644 --- a/src/paimon/core/table/system/global_system_tables.cpp +++ b/src/paimon/core/table/system/global_system_tables.cpp @@ -308,9 +308,6 @@ Result> GlobalSystemTableLoader::Load( } PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, LoadForSchema(table_name, context)); - // The inputs this table was built from are recorded for `Copy`, which re-enters this loader - // with the context unchanged: Java Paimon's global system tables copy over the options they - // already carry and ignore the dynamic ones a copy is given. SystemTable::Origin origin; origin.kind = StringUtils::ToLowerCase(table_name); origin.context = std::make_shared(context); @@ -329,9 +326,6 @@ Result> GlobalSystemTableLoader::LoadForSchema( } PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, entry.factory(context)); - // TODO(yonghao.fyh): the loader-driven AdoptOwnSchema() is an init-style coupling; - // consider overriding LatestSchema() on SystemTable, as FormatTable does, to serve - // its own read-only schema and drop this call from every loader. PAIMON_RETURN_NOT_OK(system_table->AdoptOwnSchema()); return system_table; } diff --git a/src/paimon/core/table/system/global_system_tables.h b/src/paimon/core/table/system/global_system_tables.h index 280f57479..6ac4fbb30 100644 --- a/src/paimon/core/table/system/global_system_tables.h +++ b/src/paimon/core/table/system/global_system_tables.h @@ -33,10 +33,7 @@ class FileSystem; /// Context passed to global system table constructors, providing catalog-level /// access for enumerating databases, tables, and reading metadata. struct GlobalSystemTableContext { - /// The catalog this table reads its rows through. Rows are built lazily, so a table handed back - /// to a caller keeps shared ownership of it. Empty when the table is transient and only its - /// schema is read. `GlobalSystemTableLoader::Load` refuses a table that enumerates data tables - /// when this is empty. + /// Shared catalog for lazy row loading; may be empty for schema-only tables. std::shared_ptr catalog; std::shared_ptr fs; std::string warehouse; @@ -113,14 +110,11 @@ class GlobalSystemTableLoader { static Result IsSupported(const std::string& table_name, const std::map& catalog_options); - /// Loads a global system table for a caller to read. A table that enumerates data tables needs - /// the catalog in the context, so it is refused here when the context carries none. + /// Loads a global system table. Views that enumerate tables require a catalog in context. static Result> Load(const std::string& table_name, const GlobalSystemTableContext& context); - /// Loads a global system table for the schema it exposes alone, the way - /// `Catalog::LoadTableSchema` does. No rows are built, so a context that carries no catalog is - /// accepted here and refused by `Load` instead. + /// Loads only the system table schema; no catalog is required. static Result> LoadForSchema( const std::string& table_name, const GlobalSystemTableContext& context); diff --git a/src/paimon/core/table/system/read_optimized_system_table.cpp b/src/paimon/core/table/system/read_optimized_system_table.cpp index 4f529de13..edc444369 100644 --- a/src/paimon/core/table/system/read_optimized_system_table.cpp +++ b/src/paimon/core/table/system/read_optimized_system_table.cpp @@ -26,6 +26,7 @@ #include "arrow/c/bridge.h" #include "paimon/common/types/data_field.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/file_store_table.h" #include "paimon/core/table/source/read_optimized_scan_options.h" #include "paimon/core/utils/context_utils.h" #include "paimon/defs.h" @@ -70,8 +71,7 @@ Result> ReadOptimizedSystemTable::NewScan( if (data_table_fs == nullptr && context->GetFileSystemSchemeToIdentifierMap().empty()) { data_table_fs = GetFileSystem(); } - builder.SetOptions(options) - .WithStreamingMode(context->IsStreamingMode()) + builder.WithStreamingMode(context->IsStreamingMode()) .WithMemoryPool(context->GetMemoryPool()) .WithExecutor(context->GetExecutor()) .WithFileSystem(data_table_fs) @@ -94,26 +94,21 @@ Result> ReadOptimizedSystemTable::NewScan( builder.SetTableSchema(context->GetSpecificTableSchema().value()); } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr base_context, builder.Finish()); - return TableScan::Create(std::move(base_context)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr data_table, + CreateDataTable(table_schema_, options, base_context->GetSpecificFileSystem(), + base_context->GetFileSystemSchemeToIdentifierMap())); + return data_table->NewScan(std::move(base_context)); } Result> ReadOptimizedSystemTable::CreateDataReadContext( const std::shared_ptr& context) const { - auto options = options_; - std::string branch = context->GetBranch(); - // SystemTableLoader injects Options::BRANCH when parsing paths such as `T$branch_dev$ro`. - auto branch_iter = options.find(Options::BRANCH); - if (branch_iter != options.end()) { - branch = branch_iter->second; - } ReadContextBuilder builder(table_path_); std::shared_ptr data_table_fs = context->GetSpecificFileSystem(); if (data_table_fs == nullptr && context->GetFileSystemSchemeToIdentifierMap().empty()) { data_table_fs = GetFileSystem(); } - builder.SetOptions(options) - .WithBranch(branch) - .SetPredicate(context->GetPredicate()) + builder.SetPredicate(context->GetPredicate()) .EnablePredicateFilter(context->EnablePredicateFilter()) .EnablePrefetch(context->EnablePrefetch()) .SetPrefetchBatchCount(context->GetPrefetchBatchCount()) @@ -150,7 +145,11 @@ Result> ReadOptimizedSystemTable::NewRead( PAIMON_RETURN_NOT_OK(CheckNoSearch(context)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr base_context, CreateDataReadContext(context)); - return TableRead::Create(std::move(base_context)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr data_table, + CreateDataTable(table_schema_, options_, base_context->GetSpecificFileSystem(), + base_context->GetFileSystemSchemeToIdentifierMap())); + return data_table->NewRead(std::move(base_context)); } } // namespace paimon diff --git a/src/paimon/core/table/system/read_optimized_system_table.h b/src/paimon/core/table/system/read_optimized_system_table.h index 5efb0f289..164b2c79c 100644 --- a/src/paimon/core/table/system/read_optimized_system_table.h +++ b/src/paimon/core/table/system/read_optimized_system_table.h @@ -34,9 +34,7 @@ class ReadOptimizedSystemTable : public SystemTable { public: static constexpr const char* kName = "ro"; - /// @param data_table_name Logical object name of the data table underneath, branch included, - /// which this view answers `$ro` with - the name a catalog knows the - /// data table by, not the name of the directory it lives in. + /// @param data_table_name Logical table name, including any branch suffix. ReadOptimizedSystemTable(const std::shared_ptr& fs, std::string table_path, std::shared_ptr table_schema, std::string data_table_name, std::map options); diff --git a/src/paimon/core/table/system/system_table.cpp b/src/paimon/core/table/system/system_table.cpp index a75aa1a2b..70767c5b1 100644 --- a/src/paimon/core/table/system/system_table.cpp +++ b/src/paimon/core/table/system/system_table.cpp @@ -30,12 +30,15 @@ #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" #include "paimon/commit_context.h" +#include "paimon/common/utils/options_utils.h" #include "paimon/common/utils/path_util.h" #include "paimon/common/utils/string_utils.h" +#include "paimon/core/catalog/catalog_utils.h" #include "paimon/core/options/table_type.h" #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/schema_validation.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/file_store_table.h" #include "paimon/core/table/system/audit_log_system_table.h" #include "paimon/core/table/system/binlog_system_table.h" #include "paimon/core/table/system/global_system_tables.h" @@ -82,9 +85,12 @@ const std::vector& SystemTableRegistry() { {OptionsSystemTable::kName, [](const std::shared_ptr& fs, const std::string& table_path, const std::shared_ptr& table_schema, - const std::map& /*dynamic_options*/, + const std::map& dynamic_options, const std::string& /*data_table_name*/) -> Result> { - return std::make_shared(fs, table_path, table_schema); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merged, + table_schema->MergeOptions(dynamic_options)); + return std::make_shared( + fs, table_path, std::shared_ptr(std::move(merged))); }}, {AuditLogSystemTable::kName, [](const std::shared_ptr& fs, const std::string& table_path, @@ -216,81 +222,56 @@ Status SystemTable::AdoptOwnSchema() { return Status::OK(); } +Result> SystemTable::CreateDataTable( + const std::shared_ptr& table_schema, + const std::map& options, + const std::shared_ptr& specific_file_system, + const std::map& fs_scheme_map) const { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merged_schema, + table_schema->MergeOptions(options)); + const std::string branch = BranchManager::NormalizeBranch(LoadBranch(merged_schema->Options())); + return FileStoreTable::Create(specific_file_system, GetRootPath(), + std::shared_ptr(std::move(merged_schema)), + CatalogUtils::IdentifierFromPath(GetRootPath(), branch), "", + branch, fs_scheme_map, options); +} + Result> SystemTable::Copy( const std::map& dynamic_options) const { if (!origin_) { - // A table no loader built - one of the schema-only handles `LoadForSchema` hands out - has - // no recorded inputs to rebuild from, the same defensive case the refused write and - // commit below guard. return Status::Invalid(Name(), " was not loaded by a system table loader: copying it is " "not supported"); } if (origin_->context) { - // Java Paimon's global system tables copy over the options they already carry and ignore - // the dynamic ones (`CatalogOptionsTable#copy`), so the copy re-enters the same loader. + // Global system tables ignore dynamic options when copied. return GlobalSystemTableLoader::Load(origin_->kind, *origin_->context); } - const std::shared_ptr& loaded_schema = origin_->table_schema; - // `type` is immutable the way `FileStoreTable::Copy` holds it: the kind of a copy is named by - // the primary keys of the schema it serves, never by this option. A value naming another - // type is refused, and one naming the same type is dropped. - auto type_option = dynamic_options.find(Options::TYPE); - if (type_option != dynamic_options.end()) { - PAIMON_ASSIGN_OR_RAISE(TableType schema_type, - TableTypeDefine::FromOptions(loaded_schema->Options())); - std::map named{{Options::TYPE, type_option->second}}; - PAIMON_ASSIGN_OR_RAISE(TableType named_type, TableTypeDefine::FromOptions(named)); - if (named_type != schema_type) { - return Status::Invalid(fmt::format("Change '{}' is not supported yet.", Options::TYPE)); + if (origin_->kind == AuditLogSystemTable::kName || origin_->kind == BinlogSystemTable::kName) { + PAIMON_ASSIGN_OR_RAISE( + bool sequence_number, + OptionsUtils::GetValueFromMap( + dynamic_options, Options::TABLE_READ_SEQUENCE_NUMBER_ENABLED, false)); + if (sequence_number) { + return Status::Invalid("table-read.sequence-number.enabled is not supported by hint."); } } - // `branch` names the branch the copy serves, the way it routes a load: a copy on the branch - // this view serves keeps the data table's schema, while one on another branch reads that - // branch's own latest schema and refuses a branch that holds none. - std::string current_branch = LoadBranch(MergeOptions(loaded_schema, origin_->dynamic_options)); - std::string target_branch = current_branch; - auto branch_option = dynamic_options.find(Options::BRANCH); - if (branch_option != dynamic_options.end()) { - target_branch = BranchManager::NormalizeBranch(branch_option->second); - } - std::map accumulated = origin_->dynamic_options; - accumulated.erase(Options::TYPE); - accumulated.erase(Options::BRANCH); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr current_schema, + origin_->table_schema->MergeOptions(origin_->dynamic_options)); + const std::string current_branch = LoadBranch(current_schema->Options()); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr copied_schema, + FileStoreTable::CopyTableSchema(GetFileSystem(), GetRootPath(), current_branch, + std::shared_ptr(std::move(current_schema)), + origin_->dynamic_options, dynamic_options)); + auto accumulated = origin_->dynamic_options; for (const auto& [key, value] : dynamic_options) { - if (key != Options::TYPE && key != Options::BRANCH) { + if (key != Options::TYPE) { accumulated.insert_or_assign(key, value); } } - std::shared_ptr base_schema = loaded_schema; - std::string data_table_name = origin_->data_table_name; - if (target_branch != current_branch) { - // The branch is read through the file system this table serves, and then served through - // the schema that branch holds, the same read `FileStoreTable::Copy` makes. - SchemaManager schema_manager(GetFileSystem(), GetRootPath(), target_branch); - PAIMON_ASSIGN_OR_RAISE(std::optional> latest, - schema_manager.Latest()); - if (!latest) { - return Status::Invalid(fmt::format("branch {} does not exist", target_branch)); - } - base_schema = *latest; - // The copy answers with the logical name of the branch it now serves, composed the way - // `LoadFromPath` composes one: the data table's name with the branch it addresses. - const std::string& recorded_name = - data_table_name.empty() ? PathUtil::GetName(GetRootPath()) : data_table_name; - PAIMON_ASSIGN_OR_RAISE( - std::string data_table_base, - Identifier(Identifier::kUnknownDatabase, recorded_name).GetDataTableName()); - data_table_name = Identifier(Identifier::kUnknownDatabase, data_table_base, target_branch) - .GetObjectName(); - } - std::map schema_overridable = accumulated; - if (target_branch != BranchManager::DEFAULT_MAIN_BRANCH) { - schema_overridable[Options::BRANCH] = target_branch; - } - PAIMON_RETURN_NOT_OK(SchemaValidation::CheckImmutableOptions(*base_schema, accumulated)); - return SystemTableLoader::Load(origin_->kind, GetFileSystem(), GetRootPath(), base_schema, - schema_overridable, data_table_name); + return SystemTableLoader::Load(origin_->kind, GetFileSystem(), GetRootPath(), copied_schema, + accumulated, origin_->data_table_name); } Result> SystemTable::NewWrite( @@ -314,16 +295,11 @@ Result> SystemTableLoader::Load( const std::map& dynamic_options, const std::string& data_table_name) { std::optional factory = FindSystemTableFactory(system_table_name); if (factory) { - // A caller that knows no logical name for the data table - a load from a bare path with no - // branch - leaves the views to answer with the directory name, as before. const std::string effective_name = data_table_name.empty() ? PathUtil::GetName(table_path) : data_table_name; PAIMON_ASSIGN_OR_RAISE( std::shared_ptr system_table, factory.value()(fs, table_path, table_schema, dynamic_options, effective_name)); - // The inputs this table was built from are recorded for `Copy`, which merges new dynamic - // options into them and rebuilds through this same loader, the way Java Paimon's system - // tables re-wrap `dataTable.copy(dynamicOptions)`. SystemTable::Origin origin; origin.kind = StringUtils::ToLowerCase(system_table_name); origin.table_schema = table_schema; @@ -399,9 +375,7 @@ Result> SystemTableLoader::LoadFromPath( if (parsed.branch) { options[Options::BRANCH] = parsed.branch.value(); } - // The logical name of the data table this path addresses, branch included: a `T$branch_dev$ro` - // path names the view `T$branch_dev$ro`, not the `T$ro` the bare directory name would give - // it. The branch constructor folds a spelling of the main branch into the bare name. + // Preserve branch-qualified logical names when deriving identifiers from paths. const Identifier data_identifier(Identifier::kUnknownDatabase, PathUtil::GetName(parsed.table_path), parsed.branch.value_or(std::string())); diff --git a/src/paimon/core/table/system/system_table.h b/src/paimon/core/table/system/system_table.h index 92281ad07..9cab11cd8 100644 --- a/src/paimon/core/table/system/system_table.h +++ b/src/paimon/core/table/system/system_table.h @@ -33,6 +33,7 @@ namespace paimon { class FileSystem; +class FileStoreTable; class ReadContext; class ScanContext; class TableSchema; @@ -51,13 +52,8 @@ struct SystemTablePath { bool is_global = false; }; -/// Base class for the table-scoped system tables such as `T$options` and `T$snapshots`. A -/// `T$system` identifier names a read-only view that exposes a schema of its own and serves scans -/// and reads from the data table it belongs to. -/// -/// A system table of a data table answers `$` for `Name()`. One of the `sys` -/// database answers the bare kind name. `FullName()` returns the same as `Name()` in both cases, -/// which is what Java Paimon's `Table.fullName()` defaults to. +/// Read-only table-scoped or global system view. +/// Table-scoped names are $; global names are . FullName() equals Name(). class SystemTable : public Table, public std::enable_shared_from_this { public: std::string Name() const override { @@ -68,15 +64,10 @@ class SystemTable : public Table, public std::enable_shared_from_this> ArrowSchema() const = 0; - /// Copies this system table with `dynamic_options` on top of the ones it was loaded with, - /// rebuilding it through the loader that created it - the way Java Paimon's system tables - /// re-wrap `dataTable.copy(dynamicOptions)`. A global system table of the `sys` database - /// copies over the options it already carries and ignores the dynamic ones, as Java's do. - /// Writing and committing stay refused: this view is read-only, and the data table - /// underneath serves them through its own `Table`. + /// Copies a table-scoped view with dynamic option overrides. Global views ignore overrides. Result> Copy( const std::map& dynamic_options) const override; @@ -87,48 +78,41 @@ class SystemTable : public Table, public std::enable_shared_from_this context) const override; protected: - /// @param fs File system the data table underneath is read through. - /// @param table_path Root path of the data table this system table belongs to. - /// @param name The name this table answers with: `ComposeName(data_table_name, kind)` for a - /// table-scoped system table, the bare kind name for one of the `sys` database. + /// @param fs Data table file system. + /// @param table_path Data table root path. + /// @param name System table name. SystemTable(const std::shared_ptr& fs, const std::string& table_path, std::string name); - /// Composes the name of a table-scoped system table out of the logical name of the data table - /// underneath - the object name a catalog knows it by, branch included - and the kind. Java - /// Paimon's `OptionsTable.name()` composes it the same way, from `location.getName()`; the - /// metadata tables answer with the directory name, while the views that serve the data table - /// answer with its logical name, the way Java's `ReadOptimizedTable` answers with the name of - /// the table it wraps. + /// Composes $, preserving any branch suffix. static std::string ComposeName(const std::string& data_table_name, const std::string& kind); - /// Refuses a read whose context asks for a search: a root TopK request or a `FullTextMatch` - /// predicate. Neither is evaluated on the rows a system table serves, so answering one - /// would return plain rows and quietly drop the search and its top K. + /// Rejects TopK and full-text search requests, which system-table readers cannot evaluate. static Status CheckNoSearch(const std::shared_ptr& context); - /// Fills in this table's own read-only `SystemTableSchema`. A loader calls it once the object - /// exists, because `ArrowSchema()` is virtual and cannot run from a constructor. + /// Initializes the system schema after construction, when ArrowSchema() can be called. Status AdoptOwnSchema(); + /// Builds the data table using this view's schema, including one selected by time travel. + Result> CreateDataTable( + const std::shared_ptr& table_schema, + const std::map& options, + const std::shared_ptr& specific_file_system, + const std::map& fs_scheme_map) const; + friend class SystemTableLoader; friend class GlobalSystemTableLoader; private: - /// The inputs the loader that created this table used, recorded so `Copy` rebuilds through - /// the same loader. A table-scoped system table records the `SystemTableLoader::Load` - /// arguments; a global one records the `GlobalSystemTableLoader::Load` ones instead. + /// Loader inputs retained for Copy. struct Origin { - /// System table kind, lower-case, the way the registry knows it. std::string kind; /// Data table schema the view was loaded with. Null for a global system table. std::shared_ptr table_schema; - /// Dynamic options the table was loaded with, on top of the schema's own. std::map dynamic_options; - /// Logical name of the data table underneath, branch included. May be empty for a - /// table-scoped table, whose loader then falls back to the directory name. + /// Logical data table name, including its branch; empty uses the directory name. std::string data_table_name; - /// Context a global system table was loaded with. Null for a table-scoped table. + /// Global table context; null for table-scoped views. std::shared_ptr context; }; diff --git a/src/paimon/core/table/system/system_table_test.cpp b/src/paimon/core/table/system/system_table_test.cpp index d46f2426e..2c063c278 100644 --- a/src/paimon/core/table/system/system_table_test.cpp +++ b/src/paimon/core/table/system/system_table_test.cpp @@ -233,8 +233,6 @@ TEST(SystemTableTest, TestReadOptimizedSystemTableRegistration) { /*dynamic_options=*/{}, /*data_table_name=*/"")); ASSERT_EQ(system_table->Name(), "table$ro"); - // A view loaded with the logical name of its data table answers with it, branch included: - // reloading the name a branch view answers with addresses the same branch again. ASSERT_OK_AND_ASSIGN(std::shared_ptr branch_view, SystemTableLoader::Load(ReadOptimizedSystemTable::kName, /*fs=*/nullptr, "/tmp/uuid-dir", table_schema, @@ -247,10 +245,6 @@ TEST(SystemTableTest, TestReadOptimizedSystemTableRegistration) { ASSERT_EQ(arrow_schema->field(1)->type()->id(), arrow::Type::INT32); } -// A copy of a system table merges the dynamic options into the ones it was loaded with and -// rebuilds through the same loader, the way Java Paimon's system tables re-wrap -// `dataTable.copy(dynamicOptions)`: the copy keeps the view's identity while being a table of -// its own, and an option the data table's own `Copy` would refuse is refused here too. TEST(SystemTableTest, TestCopyRebuildsThroughTheLoader) { std::map options = {{Options::FILE_SYSTEM, "local"}, {Options::FILE_FORMAT, "orc"}}; @@ -265,15 +259,46 @@ TEST(SystemTableTest, TestCopyRebuildsThroughTheLoader) { EXPECT_NE(copy, system_table); EXPECT_TRUE(std::dynamic_pointer_cast(copy) != nullptr); - // The dynamic options a copy is given reach the data table underneath it, so an immutable one - // is refused the way the data table's own `Copy` refuses it, and `type` names the kind of that - // table, which no copy changes. ASSERT_NOK_WITH_MSG(system_table->Copy({{Options::BUCKET_KEY, "other"}}), "Change 'bucket-key' is not supported yet."); ASSERT_NOK_WITH_MSG(system_table->Copy({{Options::TYPE, "format-table"}}), "Change 'type' is not supported yet."); } +TEST(SystemTableTest, TestCopyOptionsAndAuditSequenceNumber) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr schema, + CreateTableSchemaForTest({{Options::FILE_SYSTEM, "local"}, {Options::FILE_FORMAT, "orc"}})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr options, + SystemTableLoader::Load(OptionsSystemTable::kName, dir->GetFileSystem(), + dir->Str(), schema, {}, "tbl")); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
copied, + options->Copy({{Options::READ_BATCH_SIZE, "2"}})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
again, copied->Copy({{"custom", "value"}})); + auto options_copy = std::dynamic_pointer_cast(again); + ASSERT_TRUE(options_copy); + ASSERT_OK_AND_ASSIGN(std::vector rows, options_copy->BuildRows()); + std::map values; + for (const auto& row : rows) { + values.emplace(std::string(row.GetStringView(0)), std::string(row.GetStringView(1))); + } + EXPECT_EQ(values.at(Options::READ_BATCH_SIZE), "2"); + EXPECT_EQ(values.at("custom"), "value"); + EXPECT_EQ(schema->Options().count(Options::READ_BATCH_SIZE), 0); + for (const std::string& kind : + {std::string(AuditLogSystemTable::kName), std::string(BinlogSystemTable::kName)}) { + ASSERT_OK_AND_ASSIGN( + std::shared_ptr view, + SystemTableLoader::Load(kind, dir->GetFileSystem(), dir->Str(), schema, {}, "tbl")); + ASSERT_NOK_WITH_MSG(view->Copy({{Options::TABLE_READ_SEQUENCE_NUMBER_ENABLED, "true"}}), + "is not supported by hint"); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::shared_ptr
allowed, + view->Copy({{Options::TABLE_READ_SEQUENCE_NUMBER_ENABLED, "false"}})); + } +} + TEST(SystemTableTest, TestReadOptimizedSystemTablePathParsing) { ASSERT_OK_AND_ASSIGN(std::optional parsed, SystemTableLoader::TryParsePath("/tmp/db.db/t$branch_audit$ro")); @@ -301,7 +326,7 @@ TEST(SystemTableTest, TestNewReadPropagatesWarmupLevel) { for (WarmupLevel level : {WarmupLevel::NONE, WarmupLevel::RAW, WarmupLevel::DECODED}) { ReadContextBuilder builder("/tmp/table"); - builder.SetOptions(options).SetWarmupLevel(level); + builder.SetWarmupLevel(level); ASSERT_OK_AND_ASSIGN(std::unique_ptr caller_unique_context, builder.Finish()); std::shared_ptr caller_context(std::move(caller_unique_context)); ASSERT_EQ(level, caller_context->GetWarmupLevel()); @@ -359,19 +384,21 @@ TEST(SystemTableTest, TestTheOperatorEntriesRefuseASystemTablePath) { ScanContextBuilder scan_builder(system_table_path); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), "names a system table"); + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context), {}), "names a system table"); ReadContextBuilder read_builder(system_table_path); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), "names a system table"); + ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context), {}), "names a system table"); WriteContextBuilder write_builder(system_table_path, "commit_user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_context)), "names a system table"); + ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_context), {}), + "names a system table"); CommitContextBuilder commit_builder(system_table_path, "commit_user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(commit_context)), "names a system table"); + ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(commit_context), {}), + "names a system table"); } TEST(SystemTableTest, TestNewReadRefusesASearch) { diff --git a/src/paimon/core/table/table_test.cpp b/src/paimon/core/table/table_test.cpp index bc1dbcf9b..b2589be4b 100644 --- a/src/paimon/core/table/table_test.cpp +++ b/src/paimon/core/table/table_test.cpp @@ -31,14 +31,19 @@ #include "paimon/cache/cache.h" #include "paimon/commit_context.h" #include "paimon/commit_message.h" +#include "paimon/common/data/blob_utils.h" #include "paimon/common/utils/checked_cast.h" #include "paimon/core/catalog/file_system_catalog.h" #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/snapshot.h" #include "paimon/core/table/file_store_table.h" #include "paimon/core/table/file_store_table_factory.h" #include "paimon/core/table/source/data_split_impl.h" #include "paimon/core/table/source/fallback_data_split.h" +#include "paimon/core/table/system/metadata_system_tables.h" +#include "paimon/core/table/system/system_table.h" +#include "paimon/core/utils/tag_manager.h" #include "paimon/defs.h" #include "paimon/file_store_commit.h" #include "paimon/file_store_write.h" @@ -160,8 +165,6 @@ TEST(TableTest, TestTableCarriesManagersOfItsBranch) { auto file_store_copy = std::dynamic_pointer_cast(copy); ASSERT_TRUE(file_store_copy != nullptr); EXPECT_EQ(file_store_copy->CurrentBranch(), "dev"); - // A copy builds its managers over its own resolution rather than taking this table's over, and - // so serves the same branch through managers of its own. ASSERT_TRUE(file_store_copy->GetSchemaManager() != nullptr); ASSERT_TRUE(file_store_copy->GetSnapshotManager() != nullptr); EXPECT_NE(file_store_copy->GetSchemaManager(), file_store_table->GetSchemaManager()); @@ -180,8 +183,6 @@ TEST(TableTest, TestCopyKeepsTheBranchTheTableServes) { CreateManagedTable(dir->GetFileSystem(), dir->Str(), identifier, /*branch=*/"main", {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); - // A copy whose `branch` option names the branch the table serves stays on it, the option - // folded away, and still takes the other dynamic options. ASSERT_OK_AND_ASSIGN(std::shared_ptr
copy, table->Copy({{Options::BRANCH, "main"}, {Options::BUCKET, "2"}})); auto file_store_copy = std::dynamic_pointer_cast(copy); @@ -192,11 +193,15 @@ TEST(TableTest, TestCopyKeepsTheBranchTheTableServes) { ASSERT_TRUE(copied_schema != nullptr); EXPECT_EQ(copied_schema->Options().at(Options::BUCKET), "2"); { + std::map scan_builder_options; ScanContextBuilder scan_builder(dir->Str()); - scan_builder.AddOption(Options::BRANCH, "main"); + scan_builder_options[Options::BRANCH] = "main"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, scan_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_1, + file_store_copy->Copy(scan_builder_options)); ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr scan, - file_store_copy->NewScan(std::move(ctx))); + configured_table_1->NewScan(std::move(ctx))); } ASSERT_OK_AND_ASSIGN(std::shared_ptr
branch_table, @@ -209,15 +214,19 @@ TEST(TableTest, TestCopyKeepsTheBranchTheTableServes) { ASSERT_TRUE(branch_file_store_copy != nullptr); EXPECT_EQ(branch_file_store_copy->CurrentBranch(), "dev"); { + std::map scan_builder_options; ScanContextBuilder scan_builder(dir->Str() + "/branch_tbl"); - scan_builder.AddOption(Options::BRANCH, "dev"); + scan_builder_options[Options::BRANCH] = "dev"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, scan_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_2, + branch_file_store_copy->Copy(scan_builder_options)); ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr scan, - branch_file_store_copy->NewScan(std::move(ctx))); + configured_table_2->NewScan(std::move(ctx))); } } -TEST(TableTest, TestCopySwitchesToTheBranchItNames) { +TEST(TableTest, TestCopyBranchRetainsCurrentSchemaAndIdentifier) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); const std::shared_ptr fs = dir->GetFileSystem(); @@ -226,7 +235,6 @@ TEST(TableTest, TestCopySwitchesToTheBranchItNames) { ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, CreateManagedTable(fs, path, identifier, /*branch=*/"main", {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); - // The dev branch holds its own schema, told apart from the main branch's by its bucket. SchemaManager dev_schema_manager(fs, path, "dev"); auto schema = arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false)}); ASSERT_OK_AND_ASSIGN( @@ -234,20 +242,19 @@ TEST(TableTest, TestCopySwitchesToTheBranchItNames) { dev_schema_manager.CreateTable(schema, /*partition_keys=*/{}, /*primary_keys=*/{}, {{Options::BUCKET, "4"}, {Options::BUCKET_KEY, "id"}})); - // A copy on another branch serves that branch's own latest schema, addressed as its table. + // Branch overrides retain the current schema and identifier. ASSERT_OK_AND_ASSIGN(std::shared_ptr
dev_copy, table->Copy({{Options::BRANCH, "dev"}})); auto dev_file_store_copy = std::dynamic_pointer_cast(dev_copy); ASSERT_TRUE(dev_file_store_copy != nullptr); EXPECT_EQ("dev", dev_file_store_copy->CurrentBranch()); - EXPECT_EQ("tbl$branch_dev", dev_copy->Name()); + EXPECT_EQ(table->Name(), dev_copy->Name()); std::shared_ptr dev_copied_schema = std::dynamic_pointer_cast(dev_copy->LatestSchema()); ASSERT_TRUE(dev_copied_schema != nullptr); - EXPECT_EQ(dev_copied_schema->Options().at(Options::BUCKET), "4"); + EXPECT_EQ(dev_copied_schema->Options().at(Options::BUCKET), "1"); EXPECT_EQ(dev_copied_schema->Options().at(Options::BRANCH), "dev"); EXPECT_EQ("dev", dev_file_store_copy->GetSnapshotManager()->Branch()); - // The copy's other dynamic options land on the branch's schema on top. ASSERT_OK_AND_ASSIGN(std::shared_ptr
dev_copy_restyled, table->Copy({{Options::BRANCH, "dev"}, {Options::BUCKET, "2"}})); std::shared_ptr restyled_schema = @@ -256,9 +263,9 @@ TEST(TableTest, TestCopySwitchesToTheBranchItNames) { EXPECT_EQ(restyled_schema->Options().at(Options::BUCKET), "2"); EXPECT_EQ(restyled_schema->Options().at(Options::BUCKET_KEY), "id"); - // A branch holding no schema does not exist. - ASSERT_NOK_WITH_MSG(table->Copy({{Options::BRANCH, "missing"}}), - "branch missing does not exist"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
missing, + table->Copy({{Options::BRANCH, "missing"}})); + EXPECT_EQ(table->LatestSchema()->FieldNames(), missing->LatestSchema()->FieldNames()); } TEST(TableTest, TestACopyHandsItsDynamicOptionsToTheFallbackReader) { @@ -303,9 +310,7 @@ TEST(TableTest, TestAReadContextMatchingTheSchemaKeepsItsOptionsForTheFallbackRe const std::shared_ptr fs = dir->GetFileSystem(); const std::string path = dir->Str() + "/tbl"; - // The main schema already carries the sequence-number setting, so a read context repeating - // it with the same value does not restyle the load; the setting still has to reach the - // fallback branch, whose schema omits it, or that reader refuses the sequence number. + // Persisted sequence-number options must also reach the fallback reader. SchemaManager schema_manager(fs, path, "main"); auto schema = arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false)}); ASSERT_OK_AND_ASSIGN( @@ -322,12 +327,14 @@ TEST(TableTest, TestAReadContextMatchingTheSchemaKeepsItsOptionsForTheFallbackRe /*primary_keys=*/{"id"}, {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + std::map read_builder_options; ReadContextBuilder read_builder(path); - read_builder.AddOption(Options::KEY_VALUE_SEQUENCE_NUMBER_ENABLED, "true"); + read_builder_options[Options::KEY_VALUE_SEQUENCE_NUMBER_ENABLED] = "true"; + read_builder.SetReadFieldNames({"id", "_SEQUENCE_NUMBER"}); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr read, - TableRead::Create(std::move(read_context))); + TableRead::Create(std::move(read_context), read_builder_options)); } TEST(TableTest, TestACopyOfACopyServesTheLatestValueOfARepeatedDynamicOption) { @@ -381,7 +388,6 @@ TEST(TableTest, TestACopySwitchingBranchesKeepsItsAccumulatedDynamicOptions) { std::unique_ptr created, schema_manager.CreateTable(schema, /*partition_keys=*/{}, /*primary_keys=*/{"id"}, {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); - // The dev branch holds its own schema, one the sequence-number switch never reached. SchemaManager dev_schema_manager(fs, path, "dev"); ASSERT_OK_AND_ASSIGN( [[maybe_unused]] std::unique_ptr dev_schema, @@ -393,9 +399,7 @@ TEST(TableTest, TestACopySwitchingBranchesKeepsItsAccumulatedDynamicOptions) { /*uuid=*/"", /*branch=*/"main", /*fs_scheme_map=*/{}, /*dynamic_options=*/{})); - // A copy onto another branch still applies the options a previous copy accumulated - the dev - // schema omits the sequence-number switch, yet the copy serves it enabled, or a read of - // _SEQUENCE_NUMBER through it fails while the record names the switch as effective. + // Verify accumulated options also reach the reader after changing branches. ASSERT_OK_AND_ASSIGN(std::shared_ptr
with_sequence, table->Copy({{Options::KEY_VALUE_SEQUENCE_NUMBER_ENABLED, "true"}})); ASSERT_OK_AND_ASSIGN(std::shared_ptr
dev_copy, @@ -413,7 +417,6 @@ TEST(TableTest, TestACopySwitchingBranchesKeepsItsAccumulatedDynamicOptions) { ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr read, dev_copy->NewRead(std::move(read_context))); - // A key both the accumulated options and this call name takes this call's value. ASSERT_OK_AND_ASSIGN( std::shared_ptr
off, with_sequence->Copy( @@ -583,49 +586,6 @@ TEST(TableTest, TestAFallbackReaderServesItsBranchsOwnPartitionKeys) { EXPECT_TRUE(BatchReader::IsEofBatch(eof_batch)); } -TEST(TableTest, TestAFallbackReaderRefusesReadContextOptionsAbsentFromTheTable) { - auto dir = UniqueTestDirectory::Create(); - ASSERT_TRUE(dir); - const std::shared_ptr fs = dir->GetFileSystem(); - - auto make_table = - [&](const std::string& path, - const std::map& options) -> Result> { - SchemaManager schema_manager(fs, path, "main"); - auto schema = arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false)}); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr created, - schema_manager.CreateTable(schema, /*partition_keys=*/{}, - /*primary_keys=*/{}, options)); - if (options.count(Options::SCAN_FALLBACK_BRANCH) != 0) { - SchemaManager fallback_schema_manager(fs, path, "fb"); - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr fallback_created, - fallback_schema_manager.CreateTable(schema, /*partition_keys=*/{}, - /*primary_keys=*/{}, {})); - } - return FileStoreTable::Create(fs, path, std::shared_ptr(std::move(created)), - Identifier("db", "tbl"), - /*uuid=*/"", /*branch=*/"main", /*fs_scheme_map=*/{}, - /*dynamic_options=*/{}); - }; - ASSERT_OK_AND_ASSIGN(std::shared_ptr
plain, make_table(dir->Str() + "/plain", {})); - ASSERT_OK_AND_ASSIGN( - std::shared_ptr
with_fallback, - make_table(dir->Str() + "/fallback", {{Options::SCAN_FALLBACK_BRANCH, "fb"}})); - - for (const std::shared_ptr
& table : {plain, with_fallback}) { - SCOPED_TRACE(table->GetRootPath()); - ReadContextBuilder read_builder(table->GetRootPath()); - read_builder.AddOption(Options::FILE_FORMAT, "bogus"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); - // `file.format` is a key `CoreOptions` knows, so neither the plain reader nor the - // fallback one may drop it in silence: both refuse it rather than read the table's own - // format behind the caller's back. - ASSERT_NOK_WITH_MSG(table->NewRead(std::move(read_context)), - "the table was not loaded with it"); - } -} - TEST(TableTest, TestACopyServesItsS3OptionThroughManagersOfItsOwn) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); @@ -640,9 +600,7 @@ TEST(TableTest, TestACopyServesItsS3OptionThroughManagersOfItsOwn) { auto file_store_table = std::dynamic_pointer_cast(table); ASSERT_TRUE(file_store_table != nullptr); - // An option that moves the file system the schema named and one that leaves it where it was - // both come back as a copy of their own: a copy never reads metadata through the managers of - // the table it was copied from, whichever resolution it ends up serving. + // Copies rebuild managers even when the resolved file system is unchanged. for (const std::string& endpoint : {std::string("http://new"), std::string("http://old")}) { SCOPED_TRACE(endpoint); ASSERT_OK_AND_ASSIGN(std::shared_ptr
copy, table->Copy({{"s3.endpoint", endpoint}})); @@ -692,8 +650,7 @@ TEST(TableTest, TestSwitchToBranch) { dev_file_store_table->SwitchToBranch("dev")); auto same_file_store_table = std::dynamic_pointer_cast(same_table); ASSERT_TRUE(same_file_store_table != nullptr); - // Switching to the branch a table already serves shares its schema and resolves again: the - // managers it hands back are instances of their own, reading the same path on the same branch. + // Switching to the same branch still rebuilds its managers. EXPECT_EQ(same_table->LatestSchema(), dev_table->LatestSchema()); EXPECT_NE(same_file_store_table->GetSchemaManager(), dev_file_store_table->GetSchemaManager()); EXPECT_NE(same_file_store_table->GetSnapshotManager(), @@ -728,8 +685,6 @@ TEST(TableTest, TestCreateRefusesATableWithoutASchema) { dev_schema_manager.CreateTable(arrow::schema({arrow::field("c0", arrow::int32())}), /*partition_keys=*/{}, /*primary_keys=*/{}, {})); - // A path its own branch holds no schema under names no table: the call that opens one is - // refused, whatever another branch of it holds. ASSERT_NOK_WITH_MSG(FileStoreTable::Create(/*file_system=*/nullptr, path, /*schema=*/nullptr, Identifier("db", "tbl"), /*uuid=*/"", /*branch=*/BranchManager::DEFAULT_MAIN_BRANCH, @@ -811,13 +766,9 @@ TEST(TableTest, TestOperatorsServeTheBranchTheTableWasLoadedWith) { ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); EXPECT_FALSE(plan->Splits().empty()); - ScanContextBuilder main_scan_builder(dir->Str()); - main_scan_builder.AddOption(Options::BRANCH, "main"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr main_scan_ctx, main_scan_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr main_scan, - table->NewScan(std::move(main_scan_ctx))); - ASSERT_OK_AND_ASSIGN(std::shared_ptr main_plan, main_scan->CreatePlan()); - EXPECT_FALSE(main_plan->Splits().empty()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
main_copy, + table->Copy({{Options::BRANCH, "main"}})); + EXPECT_EQ(table->LatestSchema()->FieldNames(), main_copy->LatestSchema()->FieldNames()); } TEST(TableTest, TestGetTableWithUnknownDatabase) { @@ -1058,7 +1009,7 @@ TEST(TableTest, TestNewScanServesTheSchemaTheTableWasLoadedWith) { table->NewScan(std::move(held_ctx))); ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, ScanContextBuilder(dir->Str()).Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(ctx)), "not found latest schema"); + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(ctx), {}), "not found latest schema"); } TEST(TableTest, TestNewScanServesTheFileSystemTheTableWasOpenedWith) { @@ -1105,7 +1056,7 @@ TEST(TableTest, TestNewScanServesTheFileSystemTheTableWasOpenedWith) { ASSERT_OK_AND_ASSIGN( std::unique_ptr ctx, ScanContextBuilder(dir->Str()).WithFileSystem(std::make_shared()).Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(ctx)), "not found latest schema"); + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(ctx), {}), "not found latest schema"); } TEST(TableTest, TestNewScanServesTheRootPathTheTableWasOpenedWith) { @@ -1150,7 +1101,7 @@ TEST(TableTest, TestNewScanServesTheRootPathTheTableWasOpenedWith) { ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, ScanContextBuilder(dir->Str() + "/not-the-table").Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(ctx)), "not found latest schema"); + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(ctx), {}), "not found latest schema"); } TEST(TableTest, TestNewReadCreatesATableRead) { @@ -1263,6 +1214,72 @@ TEST(TableTest, TestNewWriteAndCommitRefuseAContextThatNamesAnotherTable) { table->NewCommit(std::move(ctx))); } +TEST(TableTest, TestFactoriesApplyOptionsWithoutContextConfiguration) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::map schema_options = {{Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "id"}}; + for (const std::string& branch : {std::string("main"), std::string("dev")}) { + ASSERT_OK_AND_ASSIGN( + [[maybe_unused]] std::shared_ptr
table, + CreateManagedTable(dir->GetFileSystem(), dir->Str(), Identifier("db", "tbl", branch), + branch, schema_options)); + } + for (const std::string& branch : + {std::string("main"), std::string(" "), std::string(""), std::string("dev")}) { + SCOPED_TRACE(branch); + const std::map options = {{Options::BRANCH, branch}, + {Options::READ_BATCH_SIZE, "17"}}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, + ScanContextBuilder(dir->Str()).Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
loaded, + FileStoreTableFactory::Create(*scan_context, options)); + auto managed = std::dynamic_pointer_cast(loaded); + ASSERT_TRUE(managed); + ASSERT_EQ(managed->CurrentBranch(), BranchManager::NormalizeBranch(branch)); + auto effective_schema = std::dynamic_pointer_cast(loaded->LatestSchema()); + ASSERT_TRUE(effective_schema); + ASSERT_EQ(effective_schema->Options().at(Options::READ_BATCH_SIZE), "17"); + ASSERT_OK(TableScan::Create(std::move(scan_context), options).status()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, + ReadContextBuilder(dir->Str()).Finish()); + ASSERT_OK(TableRead::Create(std::move(read_context), options).status()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, + WriteContextBuilder(dir->Str(), "commit-user").Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write, + FileStoreWrite::Create(std::move(write_context), options)); + ASSERT_OK(write->Close()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + CommitContextBuilder(dir->Str(), "commit-user").Finish()); + ASSERT_OK(FileStoreCommit::Create(std::move(commit_context), options).status()); + } +} + +TEST(TableTest, TestFactoryOptionsCannotContradictTheCatalogBranch) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto catalog = std::make_shared(); + catalog->SetFileSystem(dir->GetFileSystem()); + const Identifier identifier("db", "tbl$branch_dev"); + const std::map options = {{Options::BRANCH, "other"}}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, + ScanContextBuilder(dir->Str()).WithCatalog(catalog, identifier).Finish()); + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context), options), "aimed at one branch"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, + ReadContextBuilder(dir->Str()).WithCatalog(catalog, identifier).Finish()); + ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context), options), "aimed at one branch"); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr write_context, + WriteContextBuilder(dir->Str(), "commit-user").WithCatalog(catalog, identifier).Finish()); + ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_context), options), + "aimed at one branch"); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr commit_context, + CommitContextBuilder(dir->Str(), "commit-user").WithCatalog(catalog, identifier).Finish()); + ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(commit_context), options), + "aimed at one branch"); +} + TEST(TableTest, TestNewCommitRefusesABranchACatalogCannotAddress) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); @@ -1305,117 +1322,6 @@ TEST(TableTest, TestNewScanReadWriteCommitRejectANullContext) { ASSERT_NOK_WITH_MSG(table->NewCommit(nullptr), "commit context is null pointer"); } -TEST(TableTest, TestOperatorsRefuseAnOptionThatContradictsTheTable) { - auto dir = UniqueTestDirectory::Create(); - ASSERT_TRUE(dir); - const Identifier identifier("db", "tbl"); - ASSERT_OK_AND_ASSIGN( - std::shared_ptr
table, - CreateManagedTable(dir->GetFileSystem(), dir->Str(), identifier, /*branch=*/"main", - {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); - - { - ScanContextBuilder scan_builder(dir->Str()); - scan_builder.AddOption(Options::BUCKET, "2"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, scan_builder.Finish()); - ASSERT_NOK_WITH_MSG(table->NewScan(std::move(ctx)), - "option 'bucket' is '2' in the context but '1' on the table"); - } - { - ReadContextBuilder read_builder(dir->Str()); - read_builder.AddOption(Options::BUCKET, "2"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, read_builder.Finish()); - ASSERT_NOK_WITH_MSG(table->NewRead(std::move(ctx)), - "option 'bucket' is '2' in the context but '1' on the table"); - } - { - WriteContextBuilder write_builder(dir->Str(), "commit-user"); - write_builder.AddOption(Options::BUCKET, "2"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, write_builder.Finish()); - ASSERT_NOK_WITH_MSG(table->NewWrite(std::move(ctx)), - "option 'bucket' is '2' in the context but '1' on the table"); - } - { - CommitContextBuilder commit_builder(dir->Str(), "commit-user"); - commit_builder.AddOption(Options::BUCKET, "2"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, commit_builder.Finish()); - ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(ctx)), - "option 'bucket' is '2' in the context but '1' on the table"); - } - - ASSERT_OK_AND_ASSIGN(std::shared_ptr
copied, table->Copy({{Options::BUCKET, "2"}})); - ScanContextBuilder agreeing_scan_builder(dir->Str()); - agreeing_scan_builder.AddOption(Options::BUCKET, "2"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, agreeing_scan_builder.Finish()); - ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr scan, - copied->NewScan(std::move(ctx))); -} - -TEST(TableTest, TestOperatorsRefuseAnOptionTheTableWasNotLoadedWith) { - auto dir = UniqueTestDirectory::Create(); - ASSERT_TRUE(dir); - const Identifier identifier("db", "tbl"); - ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, - CreateManagedTable(dir->GetFileSystem(), dir->Str(), identifier, - /*branch=*/"main", - {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); - WriteAndCommitTwoRows(table, dir->Str()); - - // The operators serve the configuration the table was loaded with and nothing else, so a - // context option that configuration does not carry - a `CoreOptions` key the schema never - // saved, or a key no one reads - is dropped in silence if it is not refused: refused on every - // side, whatever the key. - const std::string dropped_in_silence = "the table was not loaded with it"; - { - ScanContextBuilder scan_builder(dir->Str()); - scan_builder.AddOption(Options::SCAN_MODE, "from-snapshot") - .AddOption(Options::SCAN_SNAPSHOT_ID, "999"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_ctx, scan_builder.Finish()); - ASSERT_NOK_WITH_MSG(table->NewScan(std::move(scan_ctx)), dropped_in_silence); - } - { - ReadContextBuilder read_builder(dir->Str()); - read_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "999"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr read_ctx, read_builder.Finish()); - ASSERT_NOK_WITH_MSG(table->NewRead(std::move(read_ctx)), dropped_in_silence); - } - { - // The commit-lease switch a writer sets on a table whose schema does not carry it is the - // same case: refused rather than committed without the lease. - CommitContextBuilder commit_builder(dir->Str(), "commit-user"); - commit_builder.AddOption(Options::REST_COMMIT_ACQUIRE_LOCK, "true"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_ctx, commit_builder.Finish()); - ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(commit_ctx)), dropped_in_silence); - } - { - // A key no `CoreOptions` reads is refused the same way: whether it names a setting the - // table would drop or one the caller only mistyped, the table was not loaded with it. - ReadContextBuilder unknown_builder(dir->Str()); - unknown_builder.AddOption("some.option-nothing-reads", "value"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr unknown_ctx, unknown_builder.Finish()); - ASSERT_NOK_WITH_MSG(table->NewRead(std::move(unknown_ctx)), dropped_in_silence); - } - - // A `header.` key a one-step load folds into the table is the same case held against a table - // that was not loaded with it: the table was not, whatever the key names downstream. - ReadContextBuilder header_builder(dir->Str()); - header_builder.AddOption("header.X-Paimon-Read-Via", "table-view"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr header_ctx, header_builder.Finish()); - ASSERT_NOK_WITH_MSG(table->NewRead(std::move(header_ctx)), dropped_in_silence); - - // The way to serve an option is a table loaded with it: `Copy()` merges it into the schema, - // and the copy applies it - which then fails on the snapshot it names, proving the option - // reached the scan instead of being dropped. - ASSERT_OK_AND_ASSIGN( - std::shared_ptr
copied, - table->Copy({{Options::SCAN_MODE, "from-snapshot"}, {Options::SCAN_SNAPSHOT_ID, "999"}})); - ASSERT_OK_AND_ASSIGN(std::unique_ptr copied_scan_ctx, - ScanContextBuilder(dir->Str()).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr copied_scan, - copied->NewScan(std::move(copied_scan_ctx))); - ASSERT_NOK_WITH_MSG(copied_scan->CreatePlan(), "snapshot"); -} - TEST(TableTest, TestCreateFailsWhenTheSchemaOptionsDoNotParse) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); @@ -1491,9 +1397,7 @@ TEST(TableTest, TestCopyServesTheTableWithTheGivenOptionsOnTop) { ASSERT_OK(commit->Commit(messages)); ASSERT_OK(write->Close()); - // The `type` option is immutable: a value naming another type is refused, the way Java - // Paimon's copy refuses it, and one naming the type the table already serves - whatever its - // case - is dropped. + // A matching type is accepted case-insensitively and omitted from overrides. ASSERT_NOK_WITH_MSG(table->Copy({{Options::BUCKET, "2"}, {Options::TYPE, "format-table"}}), "Change 'type' is not supported yet."); ASSERT_OK_AND_ASSIGN(std::shared_ptr
copied, @@ -1520,6 +1424,252 @@ TEST(TableTest, TestCopyServesTheTableWithTheGivenOptionsOnTop) { ASSERT_FALSE(plan->Splits().empty()); } +TEST(TableTest, TestCopyTimeTravelUsesHistoricalSchema) { + for (const std::string& branch : {std::string("main"), std::string("dev")}) { + SCOPED_TRACE(branch); + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto fs = dir->GetFileSystem(); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
original, + CreateManagedTable(fs, dir->Str(), Identifier("db", "tbl"), branch, + {{Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}})); + WriteAndCommitTwoRows(original, dir->Str()); + auto managed = std::dynamic_pointer_cast(original); + ASSERT_OK_AND_ASSIGN(std::optional snapshot, + managed->GetSnapshotManager()->LatestSnapshot()); + ASSERT_TRUE(snapshot); + ASSERT_OK_AND_ASSIGN(std::string snapshot_json, snapshot->ToJsonString()); + TagManager tags(fs, dir->Str(), branch); + ASSERT_OK(fs->AtomicStore(tags.TagPath("old"), snapshot_json)); + + // Model a newly added column and its storage declarations in schema 1. + auto latest_options = + std::dynamic_pointer_cast(original->LatestSchema())->Options(); + latest_options[Options::BLOB_FIELD] = "payload"; + latest_options[Options::FALLBACK_BLOB_DESCRIPTOR_FIELD] = "payload"; + ASSERT_OK_AND_ASSIGN( + std::unique_ptr latest_schema, + TableSchema::Create(1, + arrow::schema({arrow::field("id", arrow::int32(), false), + BlobUtils::ToArrowField("payload", true)}), + {}, {}, latest_options)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
latest, + FileStoreTable::Create(fs, dir->Str(), + std::shared_ptr(std::move(latest_schema)), + Identifier("db", "tbl"), "", branch, {}, {})); + for (const auto& selector : std::vector>{ + {Options::SCAN_SNAPSHOT_ID, std::to_string(snapshot->Id())}, + {Options::SCAN_TAG_NAME, "old"}, + {Options::SCAN_TIMESTAMP_MILLIS, std::to_string(snapshot->TimeMillis())}}) { + SCOPED_TRACE(selector.first); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
historical, + latest->Copy({selector, {Options::READ_BATCH_SIZE, "1"}})); + auto schema = std::dynamic_pointer_cast(historical->LatestSchema()); + ASSERT_TRUE(schema); + EXPECT_EQ(schema->Id(), 0); + EXPECT_EQ(schema->FieldNames(), (std::vector{"id"})); + EXPECT_EQ(schema->Options().count(Options::BLOB_FIELD), 0); + EXPECT_EQ(schema->Options().count(Options::FALLBACK_BLOB_DESCRIPTOR_FIELD), 0); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
again, historical->Copy({})); + EXPECT_EQ(std::dynamic_pointer_cast(again->LatestSchema())->Id(), 0); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, + ScanContextBuilder(dir->Str()).Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, + historical->NewScan(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_FALSE(plan->Splits().empty()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, + ReadContextBuilder(dir->Str()).Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, + historical->NewRead(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, + read->CreateReader(plan->Splits())); + int64_t rows = 0; + while (true) { + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, reader->NextBatch()); + if (BatchReader::IsEofBatch(batch)) { + break; + } + ASSERT_OK_AND_ASSIGN(std::shared_ptr array, + ReadResultCollector::GetArray(std::move(batch))); + rows += array->length(); + } + EXPECT_EQ(rows, 2); + } + ASSERT_OK_AND_ASSIGN(std::shared_ptr
unavailable, + original->Copy({{Options::SCAN_SNAPSHOT_ID, "999"}})); + EXPECT_EQ(std::dynamic_pointer_cast(unavailable->LatestSchema())->Id(), 0); + ASSERT_NOK_WITH_MSG( + latest->Copy({{Options::SCAN_TAG_NAME, "old"}, {Options::BLOB_FIELD, "payload"}}), + "payload"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
explicitly_declared, + latest->Copy({{Options::BLOB_FIELD, "payload"}})); + ASSERT_NOK_WITH_MSG(explicitly_declared->Copy({{Options::SCAN_TAG_NAME, "old"}}), + "payload"); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr options_view, + SystemTableLoader::Load(OptionsSystemTable::kName, fs, dir->Str(), + std::dynamic_pointer_cast(latest->LatestSchema()), + {}, "tbl")); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
old_options, + options_view->Copy({{Options::SCAN_TAG_NAME, "old"}})); + auto old_view = std::dynamic_pointer_cast(old_options); + ASSERT_TRUE(old_view); + ASSERT_OK_AND_ASSIGN(std::vector option_rows, old_view->BuildRows()); + for (const auto& row : option_rows) { + EXPECT_NE(row.GetStringView(0), Options::BLOB_FIELD); + EXPECT_NE(row.GetStringView(0), Options::FALLBACK_BLOB_DESCRIPTOR_FIELD); + } + + // Finding a snapshot but failing to load its schema must not silently use the current one. + ASSERT_OK(fs->Delete(managed->GetSchemaManager()->SchemaDirectory() + "/schema-0", false)); + ASSERT_NOK_WITH_MSG(original->Copy({{Options::SCAN_TAG_NAME, "old"}}), "schema-0"); + } +} + +TEST(TableTest, TestCopyRejectsConflictingTimeTravelOptions) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
table, + CreateManagedTable(dir->GetFileSystem(), dir->Str(), Identifier("db", "t"))); + const std::vector> invalid = { + {{Options::SCAN_SNAPSHOT_ID, "1"}, {Options::SCAN_TAG_NAME, "tag"}}, + {{Options::SCAN_SNAPSHOT_ID, "1"}, {Options::SCAN_TIMESTAMP_MILLIS, "1"}}, + {{Options::SCAN_TAG_NAME, "tag"}, {Options::SCAN_TIMESTAMP_MILLIS, "1"}}, + {{Options::SCAN_MODE, "latest-full"}, {Options::SCAN_SNAPSHOT_ID, "1"}}, + {{Options::SCAN_MODE, "latest"}, {Options::SCAN_TIMESTAMP_MILLIS, "1"}}, + {{Options::SCAN_MODE, "from-snapshot"}, {Options::SCAN_TIMESTAMP_MILLIS, "1"}}, + {{Options::SCAN_MODE, "from-snapshot-full"}, {Options::SCAN_TAG_NAME, "tag"}}, + {{Options::SCAN_MODE, "from-timestamp"}, {Options::SCAN_SNAPSHOT_ID, "1"}}, + {{Options::SCAN_MODE, "from-snapshot"}}, + {{Options::SCAN_MODE, "from-snapshot-full"}}, + {{Options::SCAN_MODE, "from-timestamp"}}, + }; + for (const auto& options : invalid) { + ASSERT_NOK_WITH_MSG(table->Copy(options), "Invalid time travel options"); + } + // Unavailable snapshots defer their error to the scan. + ASSERT_OK_AND_ASSIGN(std::shared_ptr
historical, + table->Copy({{Options::SCAN_SNAPSHOT_ID, "1"}})); + ASSERT_NOK_WITH_MSG(historical->Copy({{Options::SCAN_TAG_NAME, "tag"}}), + "Invalid time travel options"); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::shared_ptr
another_snapshot, + historical->Copy({{Options::SCAN_SNAPSHOT_ID, "2"}})); +} + +TEST(TableTest, TestCopyProtectsDeletionVectorsMode) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto fs = dir->GetFileSystem(); + for (bool enabled : {false, true}) { + SCOPED_TRACE(enabled); + const std::string value = enabled ? "true" : "false"; + const std::string changed = enabled ? "false" : "true"; + ASSERT_OK_AND_ASSIGN( + std::unique_ptr schema, + TableSchema::Create( + 0, arrow::schema({arrow::field("id", arrow::int32(), false)}), {}, {"id"}, + {{Options::BUCKET, "1"}, {Options::DELETION_VECTORS_ENABLED, value}})); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
table, + FileStoreTable::Create(fs, dir->Str(), std::shared_ptr(std::move(schema)), + Identifier("db", "t"), "", "main", {}, {})); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::shared_ptr
unchanged, + table->Copy({{Options::DELETION_VECTORS_ENABLED, value}})); + ASSERT_NOK_WITH_MSG(table->Copy({{Options::DELETION_VECTORS_ENABLED, changed}}), + "Cannot change deletion vectors mode"); + // Check the original configuration, not the modifiable flag in this hint. + ASSERT_NOK_WITH_MSG(table->Copy({{Options::DELETION_VECTORS_ENABLED, changed}, + {"deletion-vectors.modifiable", "true"}}), + "Cannot change deletion vectors mode"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
modifiable, + table->Copy({{"deletion-vectors.modifiable", "true"}})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
toggled, + modifiable->Copy({{Options::DELETION_VECTORS_ENABLED, changed}})); + EXPECT_EQ(std::dynamic_pointer_cast(toggled->LatestSchema()) + ->Options() + .at(Options::DELETION_VECTORS_ENABLED), + changed); + } +} + +TEST(TableTest, TestSystemTableCopyUsesHistoricalSchemaForOperators) { + for (const std::string& branch : {std::string("main"), std::string("dev")}) { + for (const std::string& kind : + {std::string("ro"), std::string("audit_log"), std::string("binlog")}) { + SCOPED_TRACE(branch + "/" + kind); + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto fs = dir->GetFileSystem(); + std::map options = {{Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "id"}}; + const std::vector primary_keys = + kind == "ro" ? std::vector{} : std::vector{"id"}; + SchemaManager schemas(fs, dir->Str(), branch); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr schema, + schemas.CreateTable(arrow::schema({arrow::field("id", arrow::int32(), false), + arrow::field("v", arrow::int32())}), + {}, primary_keys, options)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
original, + FileStoreTable::Create(fs, dir->Str(), + std::shared_ptr(std::move(schema)), + Identifier("db", "tbl"), "", branch, {}, {})); + WriteAndCommitRow(original, dir->Str(), 1, 10); + options = std::dynamic_pointer_cast(original->LatestSchema())->Options(); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr latest, + TableSchema::Create(1, + arrow::schema({arrow::field("id", arrow::int32(), false), + arrow::field("new", arrow::utf8())}), + {}, primary_keys, options)); + ASSERT_OK_AND_ASSIGN(std::string latest_json, latest->ToJsonString()); + ASSERT_OK(fs->AtomicStore(schemas.SchemaDirectory() + "/schema-1", latest_json)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr view, + SystemTableLoader::Load( + kind, fs, dir->Str(), + std::shared_ptr(std::move(latest)), {}, "tbl")); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
historical, + view->Copy({{Options::SCAN_SNAPSHOT_ID, "1"}})); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, + ScanContextBuilder(dir->Str()).Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, + historical->NewScan(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_FALSE(plan->Splits().empty()); + ReadContextBuilder read_builder(dir->Str()); + read_builder.SetReadFieldNames({"id", "v"}); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, + historical->NewRead(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, + read->CreateReader(plan->Splits())); + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, reader->NextBatch()); + ASSERT_FALSE(BatchReader::IsEofBatch(batch)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr array, + ReadResultCollector::GetArray(std::move(batch))); + auto rows = std::dynamic_pointer_cast(array); + ASSERT_TRUE(rows); + ASSERT_EQ(rows->length(), 1); + auto values = rows->GetFieldByName("v"); + ASSERT_TRUE(values); + if (kind == "binlog") { + auto list = std::dynamic_pointer_cast(values); + ASSERT_TRUE(list); + values = list->values(); + } + auto ints = std::dynamic_pointer_cast(values); + ASSERT_TRUE(ints); + EXPECT_EQ(ints->Value(0), 10); + } + } +} + TEST(TableTest, TestCopyRefusesThePhysicalOptionsTheDataWasLaidOutWith) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); @@ -1533,13 +1683,19 @@ TEST(TableTest, TestCopyRefusesThePhysicalOptionsTheDataWasLaidOutWith) { "Change 'bucket-key' is not supported yet."); ASSERT_NOK_WITH_MSG(table->Copy({{Options::BUCKET_FUNCTION_TYPE, "hash"}}), "Change 'bucket-function.type' is not supported yet."); - ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::shared_ptr
restyled, - table->Copy({{Options::MERGE_ENGINE, "deduplicate"}})); + for (const std::string& key : + {std::string(Options::MERGE_ENGINE), std::string(Options::ROW_TRACKING_ENABLED), + std::string(Options::DATA_EVOLUTION_ENABLED)}) { + ASSERT_NOK_WITH_MSG(table->Copy({{key, "true"}}), "is not supported yet"); + } + ASSERT_NOK_WITH_MSG(table->Copy({{Options::BUCKET, "-1"}}), "Cannot change bucket to -1"); { + std::map scan_builder_options; ScanContextBuilder scan_builder(dir->Str()); - scan_builder.AddOption(Options::BUCKET_KEY, "v"); + scan_builder_options[Options::BUCKET_KEY] = "v"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, scan_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(ctx)), + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(ctx), scan_builder_options), "Change 'bucket-key' is not supported yet."); } @@ -1547,6 +1703,18 @@ TEST(TableTest, TestCopyRefusesThePhysicalOptionsTheDataWasLaidOutWith) { table->Copy({{Options::BUCKET_KEY, "id"}, {Options::BUCKET, "1"}})); ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::shared_ptr
rescaled, table->Copy({{Options::BUCKET, "2"}})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
normalized, + table->Copy({{"primary-key", " , "}, {"partition", " "}})); + auto normalized_schema = std::dynamic_pointer_cast(normalized->LatestSchema()); + ASSERT_TRUE(normalized_schema); + EXPECT_EQ(normalized_schema->Options().count("primary-key"), 0); + EXPECT_EQ(normalized_schema->Options().count("partition"), 0); + ASSERT_NOK_WITH_MSG(table->Copy({{"primary-key", "id"}}), "is not supported yet"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
bucket_unaware, + CreateManagedTable(dir->GetFileSystem(), dir->Str() + "/bucket_unaware", + Identifier("db", "bucket_unaware"))); + ASSERT_NOK_WITH_MSG(bucket_unaware->Copy({{Options::BUCKET, "1"}}), + "Cannot change bucket when it is -1"); ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::shared_ptr
ignoring_deletes, table->Copy({{Options::IGNORE_DELETE, "true"}})); { @@ -1577,7 +1745,7 @@ TEST(TableTest, TestALoadNamesTheDatabaseThePathLaysOut) { { ScanContextBuilder builder(table_dir); ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, FileStoreTableFactory::Create(*ctx)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, FileStoreTableFactory::Create(*ctx, {})); ASSERT_EQ(table->FullName(), "my_db.my_table"); } @@ -1591,7 +1759,7 @@ TEST(TableTest, TestALoadNamesTheDatabaseThePathLaysOut) { { ScanContextBuilder builder(loose_table_dir); ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, FileStoreTableFactory::Create(*ctx)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, FileStoreTableFactory::Create(*ctx, {})); ASSERT_EQ(table->FullName(), "loose_table"); } } @@ -1600,9 +1768,7 @@ TEST(TableTest, TestALoadResolvesTheContextFormatOverTheSchemaOne) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); auto schema = arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false)}); - // A schema whose stored format this process cannot resolve: resolving the schema's own - // options at load would fail on it, so the load only serves the table when the context - // overrides the format, and only when the override is merged before the resolution. + // Apply dynamic overrides before resolving the stored file format. ASSERT_OK_AND_ASSIGN( std::unique_ptr stored_schema, TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{}, @@ -1612,16 +1778,17 @@ TEST(TableTest, TestALoadResolvesTheContextFormatOverTheSchemaOne) { ScanContextBuilder builder(dir->Str()); builder.SetTableSchema(schema_json); ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, builder.Finish()); - ASSERT_NOK(FileStoreTableFactory::Create(*ctx)); + ASSERT_NOK(FileStoreTableFactory::Create(*ctx, {})); } { + std::map builder_options; ScanContextBuilder builder(dir->Str()); - builder.SetTableSchema(schema_json).AddOption(Options::FILE_FORMAT, "parquet"); + builder_options[Options::FILE_FORMAT] = "parquet"; + builder.SetTableSchema(schema_json); ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, FileStoreTableFactory::Create(*ctx)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + FileStoreTableFactory::Create(*ctx, builder_options)); ASSERT_NE(table, nullptr); - // The scan builds its path factory over the format the merged options resolve, so it - // only opens when the override, not the stored format, is what the table serves. ScanContextBuilder scan_builder(dir->Str()); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_ctx, scan_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(scan_ctx))); @@ -1646,24 +1813,24 @@ TEST(TableTest, TestCreateBuildsTheOperator) { ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_ctx, ScanContextBuilder(dir->Str()).WithCatalog(catalog, identifier).Finish()); ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr scan, - TableScan::Create(std::move(scan_ctx))); + TableScan::Create(std::move(scan_ctx), {})); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_ctx, ReadContextBuilder(dir->Str()).WithCatalog(catalog, identifier).Finish()); ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr read, - TableRead::Create(std::move(read_ctx))); + TableRead::Create(std::move(read_ctx), {})); ASSERT_OK_AND_ASSIGN( std::unique_ptr write_ctx, WriteContextBuilder(dir->Str(), "commit-user").WithCatalog(catalog, identifier).Finish()); ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr write, - FileStoreWrite::Create(std::move(write_ctx))); + FileStoreWrite::Create(std::move(write_ctx), {})); ASSERT_OK_AND_ASSIGN( std::unique_ptr commit_ctx, CommitContextBuilder(dir->Str(), "commit-user").WithCatalog(catalog, identifier).Finish()); ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_ctx))); + FileStoreCommit::Create(std::move(commit_ctx), {})); } TEST(TableTest, TestCreateWithSpecificTableSchemaSkipsTheCatalogSchema) { @@ -1677,8 +1844,7 @@ TEST(TableTest, TestCreateWithSpecificTableSchemaSkipsTheCatalogSchema) { catalog->SetFileSystem(dir->GetFileSystem()); catalog->SetTableSchema(checked_pointer_cast(table->LatestSchema())); - // A schema the caller gave replaces the catalog's on the main branch, so neither the write - // nor the commit asks the catalog for one. + // An explicit schema bypasses catalog schema loading on main. ASSERT_OK_AND_ASSIGN(std::string schema_json, table->LatestSchema()->GetJsonSchema()); WriteContextBuilder write_builder(dir->Str(), "commit-user"); @@ -1686,14 +1852,14 @@ TEST(TableTest, TestCreateWithSpecificTableSchemaSkipsTheCatalogSchema) { std::unique_ptr write_ctx, write_builder.WithCatalog(catalog, identifier).SetTableSchema(schema_json).Finish()); ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr write, - FileStoreWrite::Create(std::move(write_ctx))); + FileStoreWrite::Create(std::move(write_ctx), {})); CommitContextBuilder commit_builder(dir->Str(), "commit-user"); ASSERT_OK_AND_ASSIGN( std::unique_ptr commit_ctx, commit_builder.WithCatalog(catalog, identifier).SetTableSchema(schema_json).Finish()); ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_ctx))); + FileStoreCommit::Create(std::move(commit_ctx), {})); ASSERT_EQ(catalog->LoadTableSchemaCalls(), 0U); } @@ -1709,19 +1875,19 @@ TEST(TableTest, TestCreateWithCatalogReadsTheMainBranchSchemaFromIt) { catalog->SetFileSystem(dir->GetFileSystem()); catalog->SetTableSchema(checked_pointer_cast(table->LatestSchema())); - // All four loads take the schema the catalog holds on the main branch when the caller did - // not set one, so each of them asks the catalog exactly once. + // Without an explicit schema, each operator loads the catalog schema once. ScanContextBuilder scan_builder(dir->Str()); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_ctx, scan_builder.WithCatalog(catalog, identifier).Finish()); - ASSERT_OK_AND_ASSIGN(std::shared_ptr
scanned, FileStoreTableFactory::Create(*scan_ctx)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
scanned, + FileStoreTableFactory::Create(*scan_ctx, {})); ASSERT_EQ(scanned->FullName(), "db.tbl"); ReadContextBuilder read_builder(dir->Str()); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_ctx, read_builder.WithCatalog(catalog, identifier).Finish()); ASSERT_OK_AND_ASSIGN(std::shared_ptr
read_table, - FileStoreTableFactory::Create(*read_ctx)); + FileStoreTableFactory::Create(*read_ctx, {})); ASSERT_EQ(read_table->FullName(), "db.tbl"); ASSERT_EQ(catalog->LoadTableSchemaCalls(), 2U); diff --git a/src/paimon/core/utils/blob_view_lookup.cpp b/src/paimon/core/utils/blob_view_lookup.cpp index 30999fe62..7dec800b8 100644 --- a/src/paimon/core/utils/blob_view_lookup.cpp +++ b/src/paimon/core/utils/blob_view_lookup.cpp @@ -161,20 +161,21 @@ Result BlobViewLookup::LoadTableDescriptorChu .WithCatalog(catalog, identifier); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + TableScan::Create(std::move(scan_context), {})); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, table_scan->CreatePlan()); + std::map read_builder_options; ReadContextBuilder read_builder(table_path); std::vector read_field_ids = field_ids; read_field_ids.push_back(SpecialFieldIds::ROW_ID); + read_builder_options[Options::BLOB_AS_DESCRIPTOR] = "true"; read_builder.SetReadFieldIds(read_field_ids) - .AddOption(Options::BLOB_AS_DESCRIPTOR, "true") .EnablePrefetch(true) .WithMemoryPool(pool) .WithCatalog(catalog, identifier); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(std::move(read_context), read_builder_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, table_read->CreateReader(plan->Splits())); diff --git a/src/paimon/core/utils/branch_manager.cpp b/src/paimon/core/utils/branch_manager.cpp index 6a9a46b3d..2592fb979 100644 --- a/src/paimon/core/utils/branch_manager.cpp +++ b/src/paimon/core/utils/branch_manager.cpp @@ -36,7 +36,6 @@ namespace paimon { Result BranchManager::ResolveBranch(const std::optional& identifier, const std::map& options, - const std::optional& explicit_branch, const std::string& operation) { std::optional identifier_branch; if (identifier) { @@ -48,8 +47,7 @@ Result BranchManager::ResolveBranch(const std::optional : std::optional(option_iter->second); std::optional resolved; - for (const std::optional& named_branch : - {identifier_branch, option_branch, explicit_branch}) { + for (const std::optional& named_branch : {identifier_branch, option_branch}) { if (!named_branch) { continue; } diff --git a/src/paimon/core/utils/branch_manager.h b/src/paimon/core/utils/branch_manager.h index eebb45647..ee87017e9 100644 --- a/src/paimon/core/utils/branch_manager.h +++ b/src/paimon/core/utils/branch_manager.h @@ -74,17 +74,10 @@ class BranchManager { return branch == DEFAULT_MAIN_BRANCH; } - /// Returns the branch an operation is aimed at, which the table identifier, the `branch` - /// option and `explicit_branch` may each name. - /// - /// An absent name and a name which `NormalizeBranch` maps to `main` both mean the main - /// branch. Names which disagree are refused rather than silently chosen between, with - /// `operation` naming the operation in that error. Every name is put through - /// `CheckValidBranch` before it is compared, so neither the branch this returns nor a - /// branch it names in an error is one that would leave the table root. + /// Resolves and validates the identifier's branch and the branch option. + /// Rejects conflicting names and invalid branch paths; operation labels errors. static Result ResolveBranch(const std::optional& identifier, const std::map& options, - const std::optional& explicit_branch, const std::string& operation); /// Fails when the branch object name `Identifier` builds, as `tbl$branch_dev`, does not read diff --git a/src/paimon/core/utils/branch_manager_test.cpp b/src/paimon/core/utils/branch_manager_test.cpp index c73459241..caa8d96a7 100644 --- a/src/paimon/core/utils/branch_manager_test.cpp +++ b/src/paimon/core/utils/branch_manager_test.cpp @@ -55,49 +55,49 @@ TEST(BranchManagerTest, TestBranchPath) { TEST(BranchManagerTest, TestResolveBranch) { const std::map no_options; - ASSERT_OK_AND_ASSIGN(std::string unnamed, - BranchManager::ResolveBranch(/*identifier=*/std::nullopt, no_options, - /*explicit_branch=*/std::nullopt, "commit")); + ASSERT_OK_AND_ASSIGN( + std::string unnamed, + BranchManager::ResolveBranch(/*identifier=*/std::nullopt, no_options, "commit")); ASSERT_EQ(BranchManager::DEFAULT_MAIN_BRANCH, unnamed); - ASSERT_OK_AND_ASSIGN(std::string from_identifier, - BranchManager::ResolveBranch(Identifier("db", "tbl$branch_data"), - no_options, std::nullopt, "commit")); + ASSERT_OK_AND_ASSIGN( + std::string from_identifier, + BranchManager::ResolveBranch(Identifier("db", "tbl$branch_data"), no_options, "commit")); ASSERT_EQ("data", from_identifier); - ASSERT_OK_AND_ASSIGN(std::string from_option, - BranchManager::ResolveBranch(std::nullopt, {{Options::BRANCH, "data"}}, - std::nullopt, "commit")); + ASSERT_OK_AND_ASSIGN( + std::string from_option, + BranchManager::ResolveBranch(std::nullopt, {{Options::BRANCH, "data"}}, "commit")); ASSERT_EQ("data", from_option); - ASSERT_OK_AND_ASSIGN(std::string from_given, - BranchManager::ResolveBranch(std::nullopt, no_options, "data", "write")); + ASSERT_OK_AND_ASSIGN( + std::string from_given, + BranchManager::ResolveBranch(std::nullopt, {{Options::BRANCH, "data"}}, "write")); ASSERT_EQ("data", from_given); - ASSERT_OK_AND_ASSIGN(std::string empty, - BranchManager::ResolveBranch(std::nullopt, no_options, "", "write")); + ASSERT_OK_AND_ASSIGN(std::string empty, BranchManager::ResolveBranch( + std::nullopt, {{Options::BRANCH, ""}}, "write")); ASSERT_EQ(BranchManager::DEFAULT_MAIN_BRANCH, empty); - ASSERT_OK_AND_ASSIGN(std::string agreed, BranchManager::ResolveBranch( - Identifier("db", "tbl$branch_data"), - {{Options::BRANCH, "data"}}, "data", "write")); + ASSERT_OK_AND_ASSIGN(std::string agreed, + BranchManager::ResolveBranch(Identifier("db", "tbl$branch_data"), + {{Options::BRANCH, "data"}}, "write")); ASSERT_EQ("data", agreed); - ASSERT_NOK_WITH_MSG( - BranchManager::ResolveBranch(Identifier("db", "tbl$branch_data"), {{Options::BRANCH, "rt"}}, - std::nullopt, "commit"), - "a commit is aimed at one branch, but both 'data' and 'rt' were named"); - ASSERT_NOK_WITH_MSG( - BranchManager::ResolveBranch(std::nullopt, {{Options::BRANCH, "main"}}, "rt", "write"), - "a write is aimed at one branch, but both 'main' and 'rt' were named"); + ASSERT_NOK_WITH_MSG(BranchManager::ResolveBranch(Identifier("db", "tbl$branch_data"), + {{Options::BRANCH, "rt"}}, "commit"), + "a commit is aimed at one branch, but both 'data' and 'rt' were named"); + ASSERT_NOK_WITH_MSG(BranchManager::ResolveBranch(Identifier("db", "tbl$branch_main"), + {{Options::BRANCH, "rt"}}, "write"), + "a write is aimed at one branch, but both 'main' and 'rt' were named"); - ASSERT_NOK_WITH_MSG(BranchManager::ResolveBranch(Identifier("db", "tbl$a$b$c"), no_options, - std::nullopt, "commit"), - "Invalid object name"); + ASSERT_NOK_WITH_MSG( + BranchManager::ResolveBranch(Identifier("db", "tbl$a$b$c"), no_options, "commit"), + "Invalid object name"); ASSERT_NOK_WITH_MSG( BranchManager::ResolveBranch(Identifier("db", "tbl$branch_data"), - {{Options::BRANCH, "rt/../../outside"}}, std::nullopt, "read"), + {{Options::BRANCH, "rt/../../outside"}}, "read"), "branch name cannot contain path separators"); ASSERT_NOK_WITH_MSG( - BranchManager::ResolveBranch(std::nullopt, no_options, "line\nfeed", "write"), + BranchManager::ResolveBranch(std::nullopt, {{Options::BRANCH, "line\nfeed"}}, "write"), "branch name cannot contain control characters"); } diff --git a/src/paimon/core/utils/context_utils.h b/src/paimon/core/utils/context_utils.h index 428bc32d2..5a845e99d 100644 --- a/src/paimon/core/utils/context_utils.h +++ b/src/paimon/core/utils/context_utils.h @@ -19,25 +19,17 @@ #pragma once #include "fmt/format.h" - -#include -#include - -#include "paimon/common/utils/sensitive_config_utils.h" #include "paimon/status.h" namespace paimon { -/// The checks every kind of table's operator entry points share. +/// Shared operator-context checks. class ContextUtils { public: ContextUtils() = delete; ~ContextUtils() = delete; - /// The checks every operator entry point opens with: a context, a memory pool and an executor - /// must all be there for the operator to be built at all, so a null one is refused rather than - /// dereferenced. `kind` names the operator the first message speaks of, which is all that sets - /// the entry points' messages apart. + /// Requires a context, memory pool and executor; kind labels errors. template static Status CheckOperatorContext(const ContextPointer& context, const char* kind) { if (context == nullptr) { @@ -51,26 +43,6 @@ class ContextUtils { } return Status::OK(); } - - /// An operator created from a table serves that table's own configuration and adds nothing - /// from the context a caller hands it, so a context carrying any option at all is refused: - /// dynamic configuration belongs to `Table::Copy()`, and a branch is named through - /// `WithBranch()` or the identifier of `WithCatalog()`, never through an option here. The - /// one-step `Create()` entry points fold the options they are given into the table they load - /// and clear them, so their contexts arrive empty. `kind` names the operator the message - /// speaks of. - static Status CheckContextCarriesNoOptions(const std::string& kind, - const std::map& options) { - if (options.empty()) { - return Status::OK(); - } - const auto& [key, value] = *options.begin(); - return Status::Invalid(fmt::format( - "the {} context carries option '{}' as '{}', but an operator created from a table " - "serves that table's own configuration and adds nothing from the context: apply " - "dynamic configuration through Table::Copy()", - kind, key, SensitiveConfigUtils::RedactValue(key, value))); - } }; } // namespace paimon diff --git a/src/paimon/rest/rest_catalog.cpp b/src/paimon/rest/rest_catalog.cpp index af8c50623..b8101dddf 100644 --- a/src/paimon/rest/rest_catalog.cpp +++ b/src/paimon/rest/rest_catalog.cpp @@ -428,8 +428,7 @@ Result> RestCatalog::LoadSystemTable( } PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table_file_system, GetTableFileSystem(identifier)); - // The data table's logical name, branch included, so that a view of a table the REST store - // keeps under a UUID directory still answers with the name the catalog knows it by. + // Use the catalog name, including any branch, even when data lives under a UUID directory. PAIMON_ASSIGN_OR_RAISE( Identifier data_identifier, CatalogUtils::BranchIdentifier(identifier, branch.value_or(std::string()))); @@ -464,10 +463,7 @@ Result> RestCatalog::GetTable(const Identifier& identifie PAIMON_ASSIGN_OR_RAISE(Identifier load_identifier, ToLoadIdentifier(identifier)); return LoadSystemTable(identifier, load_identifier, NormalizeBranch(std::move(branch))); } - // Read UUID, path and schema together to avoid mixing table generations after recreation. - // The schema is read through the data-table path `LoadTableSchema` takes for one, so it - // arrives as the `TableSchema` the table below is built with, over the branch the identifier - // names - the same derivation that load makes. + // Load UUID, location and schema together to avoid mixing table generations. PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); branch = NormalizeBranch(std::move(branch)); PAIMON_ASSIGN_OR_RAISE(Identifier load_identifier, ToLoadIdentifier(identifier)); diff --git a/src/paimon/rest/rest_catalog.h b/src/paimon/rest/rest_catalog.h index 1663b6d2e..a489d2fdf 100644 --- a/src/paimon/rest/rest_catalog.h +++ b/src/paimon/rest/rest_catalog.h @@ -137,8 +137,7 @@ class RestCatalog : public Catalog, const Identifier& data_identifier, const std::optional& branch, std::string* table_path, std::string* table_id) const; - /// Loads the system table a `T$system` identifier names, on the branch the identifier - /// carries, from the same response that carries the data table's schema and path. + /// Loads a T$system view from the same response as its data table's schema and path. Result> LoadSystemTable( const Identifier& identifier, const Identifier& load_identifier, const std::optional& branch) const; diff --git a/src/paimon/rest/rest_catalog_test.cpp b/src/paimon/rest/rest_catalog_test.cpp index 78e1eb529..7b6bf381c 100644 --- a/src/paimon/rest/rest_catalog_test.cpp +++ b/src/paimon/rest/rest_catalog_test.cpp @@ -1717,7 +1717,7 @@ TEST_F(RestCatalogTest, FileStoreCommitIsBuiltFromTheCatalog) { CommitContextBuilder builder(dir->Str(), "commit-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.WithCatalog(shared_catalog, Identifier("db1", "t1")).Finish()); - ASSERT_OK(FileStoreCommit::Create(std::move(context))); + ASSERT_OK(FileStoreCommit::Create(std::move(context), {})); { std::lock_guard lock(state_->mutex); ASSERT_GT(state_->get_table_requests, requests_before); @@ -1950,7 +1950,7 @@ TEST_P(RestCatalogCommitRecoveryTest, CommitAcceptedThenLostRecoversAfterRestart CommitContextBuilder builder(dir->Str(), "commit-user"); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.WithCatalog(catalog, identifier).Finish()); - return FileStoreCommit::Create(std::move(context)); + return FileStoreCommit::Create(std::move(context), {}); }; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, create_commit()); @@ -2122,16 +2122,17 @@ class RestCatalogCommitLeaseTest : public RestCatalogTest { Result> CreateCommit( const std::shared_ptr& catalog, const std::optional& table_id, const std::map& options) { + std::map builder_options; CommitContextBuilder builder(dir_->Str(), "commit-user"); builder.WithCatalog(catalog, Identifier("db1", "t1")); if (table_id) { builder.WithTableId(table_id.value()); } for (const auto& [key, value] : options) { - builder.AddOption(key, value); + builder_options[key] = value; } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); - return FileStoreCommit::Create(std::move(context)); + return FileStoreCommit::Create(std::move(context), builder_options); } std::vector TableEvents() { diff --git a/src/paimon/testing/mock/mock_catalog.h b/src/paimon/testing/mock/mock_catalog.h index 8fa3b3fa8..4db1b0ccb 100644 --- a/src/paimon/testing/mock/mock_catalog.h +++ b/src/paimon/testing/mock/mock_catalog.h @@ -114,9 +114,6 @@ class MockVersionManagedCatalog : public Catalog, public VersionManagedCatalog { Result> GetTable(const Identifier& identifier) const override { ++get_table_calls_; - // A managed table's schema is a `TableSchema`, while the one this mock was set may be a - // format table's `DataSchema` - served by `LoadTableSchema` alone - or none at all; - // `GetTable`, the managed kind's door, lets `FileStoreTable::Create` refuse both. std::shared_ptr managed_schema = std::dynamic_pointer_cast(table_schema_); if (table_schema_ != nullptr && managed_schema == nullptr) { @@ -132,8 +129,6 @@ class MockVersionManagedCatalog : public Catalog, public VersionManagedCatalog { /*dynamic_options=*/{}); } - /// Sets the table's schema, which `LoadTableSchema` serves either kind of: a managed table's - /// `TableSchema` - the one a load hands the table is - or a format table's `DataSchema`. void SetTableSchema(const std::shared_ptr& table_schema) { table_schema_ = table_schema; } diff --git a/src/paimon/testing/utils/test_helper.h b/src/paimon/testing/utils/test_helper.h index 2bc1e9bbc..0b9c2dd91 100644 --- a/src/paimon/testing/utils/test_helper.h +++ b/src/paimon/testing/utils/test_helper.h @@ -94,19 +94,16 @@ class TestHelper { if (!temp_directory.empty()) { context_builder.WithTempDirectory(temp_directory); } - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, - context_builder.SetOptions(options) - .WithStreamingMode(is_streaming_mode) - .WithWriteId(12345) - .Finish()); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr write_context, + context_builder.WithStreamingMode(is_streaming_mode).WithWriteId(12345).Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(std::move(write_context), options)); CommitContextBuilder commit_context_builder(table_path, commit_user); - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr commit_context, - commit_context_builder.SetOptions(options).IgnoreEmptyCommit(false).Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, + commit_context_builder.IgnoreEmptyCommit(false).Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_context))); + FileStoreCommit::Create(std::move(commit_context), options)); return std::unique_ptr(new TestHelper(std::move(file_system), std::move(write), std::move(commit), commit_user, table_path, options)); @@ -157,15 +154,14 @@ class TestHelper { std::optional snapshot_id, bool is_streaming = true) { ScanContextBuilder scan_context_builder(table_path_); - scan_context_builder.WithStreamingMode(is_streaming) - .SetOptions(options_) - .AddOption(Options::SCAN_MODE, startup_mode.ToString()); + scan_context_builder.WithStreamingMode(is_streaming); + std::map options = options_; + options[Options::SCAN_MODE] = startup_mode.ToString(); if (snapshot_id) { - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, - std::to_string(snapshot_id.value())); + options[Options::SCAN_SNAPSHOT_ID] = std::to_string(snapshot_id.value()); } PAIMON_ASSIGN_OR_RAISE(auto scan_context, scan_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(scan_, TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE(scan_, TableScan::Create(std::move(scan_context), options)); return Scan(); } @@ -277,13 +273,13 @@ class TestHelper { const std::vector>& splits, std::unique_ptr<::ArrowSchema> read_schema) { ReadContextBuilder read_context_builder(table_path_); - read_context_builder.SetOptions(options_); if (read_schema != nullptr) { read_context_builder.SetReadSchema(std::move(read_schema)); } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_read, + TableRead::Create(std::move(read_context), options_)); PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(splits)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr collected, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -303,10 +299,10 @@ class TestHelper { const std::vector>& splits, const std::string& expected_result) { ReadContextBuilder read_context_builder(table_path_); - read_context_builder.SetOptions(options_); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_read, + TableRead::Create(std::move(read_context), options_)); PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(splits)); PAIMON_ASSIGN_OR_RAISE(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); diff --git a/test/inte/append_compaction_inte_test.cpp b/test/inte/append_compaction_inte_test.cpp index 1424533de..70ec813e7 100644 --- a/test/inte/append_compaction_inte_test.cpp +++ b/test/inte/append_compaction_inte_test.cpp @@ -412,10 +412,13 @@ TEST_P(AppendCompactionInteTest, TestAppendTableStreamWriteFullCompactionWithMap auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*read_schema, c_schema.get()).ok()); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(PathUtil::JoinPath(dir->Str(), "foo.db/bar")); - read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); + read_context_builder_options = options; + read_context_builder.SetReadSchema(std::move(c_schema)); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context), + read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -1035,10 +1038,13 @@ TEST_P(AppendCompactionInteTest, TestAppendTableCompactionDictionaryPassthrough) auto predicate = PredicateBuilder::Equal( /*field_index=*/1, /*field_name=*/"s", FieldType::STRING, Literal(FieldType::STRING, indexed_value.data(), indexed_value.size())); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(PathUtil::JoinPath(dir->Str(), "foo.db/bar")); - read_context_builder.SetOptions(options).SetPredicate(predicate); + read_context_builder_options = options; + read_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); ASSERT_OK_AND_ASSIGN(auto filtered, ReadResultCollector::CollectResult(batch_reader.get())); auto expected = std::make_shared( diff --git a/test/inte/blob_table_inte_test.cpp b/test/inte/blob_table_inte_test.cpp index 689ca9513..0850740d4 100644 --- a/test/inte/blob_table_inte_test.cpp +++ b/test/inte/blob_table_inte_test.cpp @@ -145,7 +145,7 @@ class BlobTableInteTest : public testing::Test, public ::testing::WithParamInter write_builder.WithWriteSchema(write_cols); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, write_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(auto file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(std::move(write_context), {})); for (const auto& write_array : write_arrays) { ArrowArray c_array; @@ -180,7 +180,7 @@ class BlobTableInteTest : public testing::Test, public ::testing::WithParamInter PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, commit_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_store_commit, - FileStoreCommit::Create(std::move(commit_context))); + FileStoreCommit::Create(std::move(commit_context), {})); return file_store_commit->Commit(commit_msgs); } @@ -280,7 +280,7 @@ class BlobTableInteTest : public testing::Test, public ::testing::WithParamInter scan_context_builder.SetGlobalIndexResult(global_index_result); } PAIMON_ASSIGN_OR_RAISE(auto scan_context, scan_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context), {})); PAIMON_ASSIGN_OR_RAISE(auto result_plan, table_scan->CreatePlan()); return result_plan; } @@ -291,15 +291,17 @@ class BlobTableInteTest : public testing::Test, public ::testing::WithParamInter const std::shared_ptr& plan, const std::shared_ptr& predicate = nullptr, const std::map& options = {}) const { auto splits = plan->Splits(); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); read_context_builder.SetReadFieldNames(read_schema).SetPredicate(predicate); read_context_builder.EnableLateMaterializing(false); if (!options.empty()) { - read_context_builder.SetOptions(options); + read_context_builder_options = options; } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context), + read_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(splits)); PAIMON_ASSIGN_OR_RAISE(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -4765,10 +4767,13 @@ TEST_P(BlobTableInteTest, TestReadArrayBlobTableFromJava) { const std::vector read_fields = {"id", "array_payloads"}; for (int64_t snapshot_id : {1, 2, 3}) { + std::map scan_builder_options; ScanContextBuilder scan_builder(table_path); - scan_builder.AddOption(Options::SCAN_SNAPSHOT_ID, std::to_string(snapshot_id)); + scan_builder_options[Options::SCAN_SNAPSHOT_ID] = std::to_string(snapshot_id); + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_builder_options)); ASSERT_OK_AND_ASSIGN(auto plan, table_scan->CreatePlan()); size_t array_layer_count = 0; @@ -4826,10 +4831,13 @@ TEST_P(BlobTableInteTest, TestReadMapBlobTableFromJava) { "large_decimal_payloads"}; for (int64_t snapshot_id : {1, 3}) { + std::map scan_builder_options; ScanContextBuilder scan_builder(table_path); - scan_builder.AddOption(Options::SCAN_SNAPSHOT_ID, std::to_string(snapshot_id)); + scan_builder_options[Options::SCAN_SNAPSHOT_ID] = std::to_string(snapshot_id); + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_builder_options)); ASSERT_OK_AND_ASSIGN(auto plan, table_scan->CreatePlan()); size_t string_layer_count = 0; diff --git a/test/inte/clean_inte_test.cpp b/test/inte/clean_inte_test.cpp index ec418d2c0..24dcd51a7 100644 --- a/test/inte/clean_inte_test.cpp +++ b/test/inte/clean_inte_test.cpp @@ -204,11 +204,14 @@ TEST_F(CleanInteTest, TestExpireSnapshotFailover) { auto dir = UniqueTestDirectory::Create(); std::string table_path = dir->Str(); ASSERT_TRUE(TestUtil::CopyDirectory(test_data_path, table_path)); + std::map commit_context_builder_options; CommitContextBuilder commit_context_builder(table_path, "commit_user_1"); - ASSERT_OK_AND_ASSIGN( - std::unique_ptr commit_context, - commit_context_builder.SetOptions(clean_options).IgnoreEmptyCommit(false).Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + commit_context_builder_options = clean_options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + + commit_context_builder.IgnoreEmptyCommit(false).Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context), + commit_context_builder_options)); ASSERT_OK_AND_ASSIGN(int32_t expired_items, commit->Expire()); ASSERT_EQ(expired_items, 4); } @@ -219,11 +222,14 @@ TEST_F(CleanInteTest, TestExpireSnapshotFailover) { ASSERT_TRUE(TestUtil::CopyDirectory(test_data_path, table_path)); ASSERT_OK(file_system_->Delete(PathUtil::JoinPath( table_path, "manifest/manifest-list-55a3b658-08e3-40aa-b692-29dc1e3ebbdc-1"))); + std::map commit_context_builder_options; CommitContextBuilder commit_context_builder(table_path, "commit_user_1"); - ASSERT_OK_AND_ASSIGN( - std::unique_ptr commit_context, - commit_context_builder.SetOptions(clean_options).IgnoreEmptyCommit(false).Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + commit_context_builder_options = clean_options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + + commit_context_builder.IgnoreEmptyCommit(false).Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context), + commit_context_builder_options)); ASSERT_OK_AND_ASSIGN(int32_t expired_items, commit->Expire()); ASSERT_EQ(expired_items, 4); } @@ -234,11 +240,14 @@ TEST_F(CleanInteTest, TestExpireSnapshotFailover) { ASSERT_TRUE(TestUtil::CopyDirectory(test_data_path, table_path)); ASSERT_OK(file_system_->Delete(PathUtil::JoinPath( table_path, "f1=10/bucket-1/data-7a912f84-04b7-4bbb-8dc6-53f4a292ea25-0.parquet"))); + std::map commit_context_builder_options; CommitContextBuilder commit_context_builder(table_path, "commit_user_1"); - ASSERT_OK_AND_ASSIGN( - std::unique_ptr commit_context, - commit_context_builder.SetOptions(clean_options).IgnoreEmptyCommit(false).Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + commit_context_builder_options = clean_options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + + commit_context_builder.IgnoreEmptyCommit(false).Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context), + commit_context_builder_options)); ASSERT_OK_AND_ASSIGN(int32_t expired_items, commit->Expire()); ASSERT_EQ(expired_items, 4); } @@ -248,11 +257,14 @@ TEST_F(CleanInteTest, TestExpireSnapshotFailover) { std::string table_path = dir->Str(); ASSERT_TRUE(TestUtil::CopyDirectory(test_data_path, table_path)); ASSERT_OK(file_system_->Delete(PathUtil::JoinPath(table_path, "snapshot/snapshot-3"))); + std::map commit_context_builder_options; CommitContextBuilder commit_context_builder(table_path, "commit_user_1"); - ASSERT_OK_AND_ASSIGN( - std::unique_ptr commit_context, - commit_context_builder.SetOptions(clean_options).IgnoreEmptyCommit(false).Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + commit_context_builder_options = clean_options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + + commit_context_builder.IgnoreEmptyCommit(false).Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context), + commit_context_builder_options)); ASSERT_OK_AND_ASSIGN(int32_t expired_items, commit->Expire()); ASSERT_EQ(expired_items, 1); } @@ -282,12 +294,15 @@ TEST_F(CleanInteTest, TestDropPartitionAndExpireSnapshot) { CreateTestTable(dir->Str(), /*db_name=*/"foo", /*table_name=*/"bar", &arrow_schema, /*partition_keys=*/{"f1"}, /*primary_keys=*/{}, options)); + std::map context_builder_options; WriteContextBuilder context_builder(table_path, "commit_user_1"); + context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.SetOptions(options).WithStreamingMode(true).Finish()); + + context_builder.WithStreamingMode(true).Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(std::move(write_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_0, MakeRecordBatch({{"Alice", 10, 1, 11.1}}, {{"f1", "10"}}, 0)); @@ -303,18 +318,20 @@ TEST_F(CleanInteTest, TestDropPartitionAndExpireSnapshot) { ASSERT_OK_AND_ASSIGN(std::vector> results, file_store_write->PrepareCommit(/*wait_compaction=*/false, 0)); ASSERT_EQ(3u, results.size()); + std::map commit_context_builder_options; CommitContextBuilder commit_context_builder(table_path, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - commit_context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .WithFileSystem(file_system_) - .AddOption(Options::SNAPSHOT_NUM_RETAINED_MAX, "2") - .AddOption(Options::SNAPSHOT_NUM_RETAINED_MIN, "1") - .AddOption(Options::SNAPSHOT_TIME_RETAINED, "1ms") - .AddOption(Options::MANIFEST_MERGE_MIN_COUNT, "2") - .AddOption(Options::SNAPSHOT_CLEAN_EMPTY_DIRECTORIES, "true") - .IgnoreEmptyCommit(false) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + commit_context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + commit_context_builder_options[Options::SNAPSHOT_NUM_RETAINED_MAX] = "2"; + commit_context_builder_options[Options::SNAPSHOT_NUM_RETAINED_MIN] = "1"; + commit_context_builder_options[Options::SNAPSHOT_TIME_RETAINED] = "1ms"; + commit_context_builder_options[Options::MANIFEST_MERGE_MIN_COUNT] = "2"; + commit_context_builder_options[Options::SNAPSHOT_CLEAN_EMPTY_DIRECTORIES] = "true"; + ASSERT_OK_AND_ASSIGN( + std::unique_ptr commit_context, + + commit_context_builder.WithFileSystem(file_system_).IgnoreEmptyCommit(false).Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context), + commit_context_builder_options)); ASSERT_OK(commit->Commit(results, 0, 10)); ASSERT_OK_AND_ASSIGN( @@ -430,12 +447,16 @@ TEST_F(CleanInteTest, TestDropPartitionAndExpireSnapshotWithIOException) { /*table_name=*/"bar", &arrow_schema, /*partition_keys=*/{"f1"}, /*primary_keys=*/{}, options)); + std::map context_builder_options; WriteContextBuilder context_builder(table_path, "commit_user_1"); + context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.SetOptions(options).WithStreamingMode(true).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(std::move(write_context))); + context_builder.WithStreamingMode(true).Finish()); + + ASSERT_OK_AND_ASSIGN( + std::unique_ptr file_store_write, + FileStoreWrite::Create(std::move(write_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_0, MakeRecordBatch({{"Alice", 10, 1, 11.1}}, {{"f1", "10"}}, 0)); @@ -451,19 +472,20 @@ TEST_F(CleanInteTest, TestDropPartitionAndExpireSnapshotWithIOException) { ASSERT_OK_AND_ASSIGN(std::vector> results, file_store_write->PrepareCommit(/*wait_compaction=*/false, 0)); ASSERT_EQ(3u, results.size()); + std::map commit_context_builder_options; CommitContextBuilder commit_context_builder(table_path, "commit_user_1"); + commit_context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + commit_context_builder_options[Options::SNAPSHOT_NUM_RETAINED_MAX] = "2"; + commit_context_builder_options[Options::SNAPSHOT_NUM_RETAINED_MIN] = "1"; + commit_context_builder_options[Options::SNAPSHOT_TIME_RETAINED] = "1ms"; + commit_context_builder_options[Options::MANIFEST_MERGE_MIN_COUNT] = "2"; + commit_context_builder_options[Options::SNAPSHOT_CLEAN_EMPTY_DIRECTORIES] = "true"; ASSERT_OK_AND_ASSIGN( std::unique_ptr commit_context, - commit_context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .WithFileSystem(file_system_) - .AddOption(Options::SNAPSHOT_NUM_RETAINED_MAX, "2") - .AddOption(Options::SNAPSHOT_NUM_RETAINED_MIN, "1") - .AddOption(Options::SNAPSHOT_TIME_RETAINED, "1ms") - .AddOption(Options::MANIFEST_MERGE_MIN_COUNT, "2") - .AddOption(Options::SNAPSHOT_CLEAN_EMPTY_DIRECTORIES, "true") - .IgnoreEmptyCommit(false) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + + commit_context_builder.WithFileSystem(file_system_).IgnoreEmptyCommit(false).Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context), + commit_context_builder_options)); ASSERT_OK(commit->Commit(results, 0)); ASSERT_OK_AND_ASSIGN( @@ -588,12 +610,15 @@ TEST_F(CleanInteTest, TestOrphanFilesClean) { /*partition_keys=*/{"f1", "f2"}, /*primary_keys=*/{}, options)); + std::map context_builder_options; WriteContextBuilder context_builder(table_path, commit_user); + context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.SetOptions(options).WithStreamingMode(true).Finish()); + + context_builder.WithStreamingMode(true).Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(std::move(write_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_0, MakeRecordBatch({{"Alice", 10, 1, 11.1}}, {{"f1", "10"}, {"f2", "1"}}, 0)); @@ -609,13 +634,15 @@ TEST_F(CleanInteTest, TestOrphanFilesClean) { ASSERT_OK_AND_ASSIGN(std::vector> results, file_store_write->PrepareCommit(/*wait_compaction=*/false, 0)); ASSERT_EQ(3u, results.size()); + std::map commit_context_builder_options; CommitContextBuilder commit_context_builder(table_path, commit_user); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - commit_context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .WithFileSystem(file_system_) - .IgnoreEmptyCommit(false) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + commit_context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + ASSERT_OK_AND_ASSIGN( + std::unique_ptr commit_context, + + commit_context_builder.WithFileSystem(file_system_).IgnoreEmptyCommit(false).Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context), + commit_context_builder_options)); ASSERT_OK(commit->Commit(results, 0)); ASSERT_OK_AND_ASSIGN( @@ -702,12 +729,15 @@ TEST_F(CleanInteTest, TestOrphanFilesCleanWithFileRetainCondition) { /*partition_keys=*/{"f1", "f2"}, /*primary_keys=*/{}, options)); + std::map context_builder_options; WriteContextBuilder context_builder(table_path, commit_user); + context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.SetOptions(options).WithStreamingMode(true).Finish()); + + context_builder.WithStreamingMode(true).Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(std::move(write_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_0, MakeRecordBatch({{"Alice", 10, 1, 11.1}}, {{"f1", "10"}, {"f2", "1"}}, 0)); @@ -723,13 +753,15 @@ TEST_F(CleanInteTest, TestOrphanFilesCleanWithFileRetainCondition) { ASSERT_OK_AND_ASSIGN(std::vector> results, file_store_write->PrepareCommit(/*wait_compaction=*/false, 0)); ASSERT_EQ(3u, results.size()); + std::map commit_context_builder_options; CommitContextBuilder commit_context_builder(table_path, commit_user); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - commit_context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .WithFileSystem(file_system_) - .IgnoreEmptyCommit(false) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + commit_context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + ASSERT_OK_AND_ASSIGN( + std::unique_ptr commit_context, + + commit_context_builder.WithFileSystem(file_system_).IgnoreEmptyCommit(false).Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context), + commit_context_builder_options)); ASSERT_OK(commit->Commit(results, 0)); ASSERT_OK_AND_ASSIGN( @@ -812,12 +844,15 @@ TEST_F(CleanInteTest, TestOrphanFilesCleanWithIOException) { /*partition_keys=*/{"f1"}, /*primary_keys=*/{}, options)); + std::map context_builder_options; WriteContextBuilder context_builder(table_path, commit_user); + context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.SetOptions(options).WithStreamingMode(true).Finish()); + + context_builder.WithStreamingMode(true).Finish()); std::string root_path = write_context->GetRootPath(); ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(std::move(write_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_0, MakeRecordBatch({{"Alice", 10, 1, 11.1}}, {{"f1", "10"}}, 0)); @@ -833,14 +868,16 @@ TEST_F(CleanInteTest, TestOrphanFilesCleanWithIOException) { ASSERT_OK_AND_ASSIGN(std::vector> results, file_store_write->PrepareCommit(/*wait_compaction=*/false, 0)); ASSERT_EQ(3u, results.size()); + std::map commit_context_builder_options; CommitContextBuilder commit_context_builder(root_path, commit_user); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - commit_context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .WithFileSystem(file_system_) - .AddOption(Options::BUCKET, "2") - .IgnoreEmptyCommit(false) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + commit_context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + commit_context_builder_options[Options::BUCKET] = "2"; + ASSERT_OK_AND_ASSIGN( + std::unique_ptr commit_context, + + commit_context_builder.WithFileSystem(file_system_).IgnoreEmptyCommit(false).Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context), + commit_context_builder_options)); ASSERT_OK(commit->Commit(results, 0)); ASSERT_OK_AND_ASSIGN( diff --git a/test/inte/data_evolution_table_test.cpp b/test/inte/data_evolution_table_test.cpp index 395de88d0..585ee2621 100644 --- a/test/inte/data_evolution_table_test.cpp +++ b/test/inte/data_evolution_table_test.cpp @@ -113,7 +113,7 @@ class DataEvolutionTableTest : public ::testing::Test, write_builder.WithWriteSchema(write_cols); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, write_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(auto file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(std::move(write_context), {})); for (const auto& write_array : write_arrays) { ArrowArray c_array; PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*write_array, &c_array)); @@ -163,7 +163,7 @@ class DataEvolutionTableTest : public ::testing::Test, PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, commit_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_store_commit, - FileStoreCommit::Create(std::move(commit_context))); + FileStoreCommit::Create(std::move(commit_context), {})); return file_store_commit->Commit(commit_msgs); } @@ -300,6 +300,7 @@ class DataEvolutionTableTest : public ::testing::Test, const std::string& table_path, const std::vector& read_schema, int32_t limit, const std::shared_ptr& predicate = nullptr, const std::vector& row_ranges = {}, bool enable_predicate_filter = false) const { + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); scan_context_builder.SetLimit(limit); scan_context_builder.SetPredicate(predicate); @@ -307,10 +308,12 @@ class DataEvolutionTableTest : public ::testing::Test, scan_context_builder.SetGlobalIndexResult( BitmapGlobalIndexResult::FromRanges(row_ranges)); } - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, - FinishScanContext(scan_context_builder)); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr result_plan, table_scan->CreatePlan()); LimitScanResult result; @@ -322,7 +325,7 @@ class DataEvolutionTableTest : public ::testing::Test, PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(std::move(read_context), {})); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch_reader, table_read->CreateReader(result.splits)); PAIMON_ASSIGN_OR_RAISE(result.rows, @@ -335,11 +338,14 @@ class DataEvolutionTableTest : public ::testing::Test, /// the row count derived from them. Reading the splits back is ScanAndReadWithLimit's job, /// which verifies the returned rows after their reader has been destroyed. Result>> PlanSplits(const std::string& table_path) const { + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, - FinishScanContext(scan_context_builder)); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, table_scan->CreatePlan()); return plan->Splits(); } @@ -412,7 +418,7 @@ class DataEvolutionTableTest : public ::testing::Test, PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, commit_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_store_commit, - FileStoreCommit::Create(std::move(commit_context))); + FileStoreCommit::Create(std::move(commit_context), {})); file_store_commit->RowIdCheckConflict(row_id_check_from_snapshot); return file_store_commit->Commit(commit_msgs); } @@ -470,11 +476,14 @@ class DataEvolutionTableTest : public ::testing::Test, ASSERT_OK_AND_ASSIGN(std::string anchor, PlannedAnchorFileName(table_path)); ASSERT_OK(CommitDeletionVectors(table_path, base_msgs[0], {{anchor, deleted}})); } + std::map scan_builder_options; ScanContextBuilder scan_builder(table_path); scan_builder.SetGlobalIndexResult(std::make_shared( RoaringBitmap64::From(candidates), std::vector(scores))); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_builder)); - ASSERT_OK_AND_ASSIGN(auto scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_builder, scan_builder_options)); + ASSERT_OK_AND_ASSIGN(auto scan, + TableScan::Create(std::move(scan_context), scan_builder_options)); ASSERT_OK_AND_ASSIGN(auto plan, scan->CreatePlan()); ASSERT_EQ(plan->Splits().size(), 1); auto scored_split = std::dynamic_pointer_cast(plan->Splits()[0]); @@ -510,7 +519,7 @@ class DataEvolutionTableTest : public ::testing::Test, .SetPredicate(bound_predicate) .EnablePredicateFilter(true); ASSERT_OK_AND_ASSIGN(auto read_context, read_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto read, TableRead::Create(std::move(read_context), {})); // Reuse the same TableRead with and without scores: the internal row-id // projection must not affect a subsequent split's output schema. for (bool with_scores : {true, false}) { @@ -560,14 +569,17 @@ class DataEvolutionTableTest : public ::testing::Test, bool check_scan_plan_when_empty_result = true, bool apply_predicate_to_scan = true) const { // scan + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); scan_context_builder.SetPredicate(apply_predicate_to_scan ? predicate : nullptr); if (!row_ranges.empty()) { auto global_index_result = BitmapGlobalIndexResult::FromRanges(row_ranges); scan_context_builder.SetGlobalIndexResult(global_index_result); } - PAIMON_ASSIGN_OR_RAISE(auto scan_context, FinishScanContext(scan_context_builder)); - PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE(auto scan_context, FinishScanContext(scan_context_builder, + scan_context_builder_options)); + PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context), + scan_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto result_plan, table_scan->CreatePlan()); if (!expected_array && check_scan_plan_when_empty_result) { if (!result_plan->Splits().empty()) { @@ -581,7 +593,7 @@ class DataEvolutionTableTest : public ::testing::Test, read_context_builder.SetReadFieldNames(read_schema).SetPredicate(predicate); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context), {})); PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(splits)); PAIMON_ASSIGN_OR_RAISE(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -626,14 +638,17 @@ class DataEvolutionTableTest : public ::testing::Test, const std::vector>& expected_first_row_ids, const std::vector& expected_row_counts) { ASSERT_EQ(expected_first_row_ids.size(), expected_row_counts.size()); + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); scan_context_builder.SetPredicate(predicate); if (!row_ranges.empty()) { auto global_index_result = BitmapGlobalIndexResult::FromRanges(row_ranges); scan_context_builder.SetGlobalIndexResult(global_index_result); } - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context), + scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); const auto& result_splits = result_plan->Splits(); if (expected_first_row_ids.empty()) { @@ -658,14 +673,15 @@ class DataEvolutionTableTest : public ::testing::Test, ASSERT_EQ(result_row_counts, expected_row_counts); } - Result> FinishScanContext(ScanContextBuilder& builder) const { + Result> FinishScanContext( + ScanContextBuilder& builder, std::map& options) const { if (EnableSnapshotLiveManifestCache()) { if (!snapshot_live_manifest_cache_) { snapshot_live_manifest_cache_ = std::make_shared(/*max_weight=*/64 * 1024 * 1024); } - builder.AddOption(Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, "3") - .WithCache(snapshot_live_manifest_cache_); + options[Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS] = "3"; + builder.WithCache(snapshot_live_manifest_cache_); } return builder.Finish(); } @@ -1090,16 +1106,19 @@ TEST_P(DataEvolutionTableTest, TestMultipleSharedShreddingMapsPartialOverwrite) auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*read_schema, c_schema.get()).ok()); + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context), + scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ReadContextBuilder read_context_builder(table_path); read_context_builder.SetReadSchema(std::move(c_schema)); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context), {})); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -2015,7 +2034,7 @@ TEST_P(DataEvolutionTableTest, TestReadJavaIndependentlyRolledVectorStore) { read_builder.SetReadFieldNames({"embedding"}); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr read, - TableRead::Create(std::move(read_context))); + TableRead::Create(std::move(read_context), {})); const std::vector, std::shared_ptr>> gap_cases = {{gap_split, gap_expected}, {std::make_shared(gap_split, selected_ranges), @@ -2081,7 +2100,7 @@ TEST_P(DataEvolutionTableTest, TestReadJavaIndependentlyRolledVectorStore) { ASSERT_OK_AND_ASSIGN(std::unique_ptr sequence_context, sequence_read_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr sequence_read, - TableRead::Create(std::move(sequence_context))); + TableRead::Create(std::move(sequence_context), {})); ASSERT_OK_AND_ASSIGN(std::unique_ptr sequence_reader, sequence_read->CreateReader(split)); ASSERT_OK_AND_ASSIGN(std::shared_ptr sequence_result, @@ -2192,14 +2211,17 @@ TEST_P(DedicatedVectorTableTest, TestDedicatedVectorStore) { WriteContextBuilder partial_builder(table_path, "partial"); partial_builder.WithWriteSchema({"embedding"}); ASSERT_OK_AND_ASSIGN(std::unique_ptr partial_context, partial_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(partial_context)).status(), + ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(partial_context), {}).status(), "partial writes"); + std::map invalid_builder_options; WriteContextBuilder invalid_builder(table_path, "invalid"); - invalid_builder.AddOption(Options::DATA_EVOLUTION_ENABLED, "false"); + invalid_builder_options[Options::DATA_EVOLUTION_ENABLED] = "false"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr invalid_context, invalid_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(invalid_context)).status(), - "requires row-tracking.enabled and data-evolution.enabled"); + ASSERT_NOK_WITH_MSG( + FileStoreWrite::Create(std::move(invalid_context), invalid_builder_options).status(), + "requires row-tracking.enabled and data-evolution.enabled"); } TEST_P(DedicatedVectorTableTest, TestIndependentVectorRolling) { @@ -3826,7 +3848,7 @@ TEST_P(DataEvolutionDvTableTest, TestReadWithDeletionVectorsOnEveryRowRangeGroup ASSERT_OK_AND_ASSIGN(std::unique_ptr count_context, count_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr count_table_read, - TableRead::Create(std::move(count_context))); + TableRead::Create(std::move(count_context), {})); ASSERT_OK_AND_ASSIGN(std::unique_ptr count_reader, count_table_read->CreateCountReader(planned_splits)); ASSERT_OK_AND_ASSIGN(int64_t counted_rows, count_reader->CountRows()); diff --git a/test/inte/global_index_test.cpp b/test/inte/global_index_test.cpp index c5b421629..0de3ccb01 100644 --- a/test/inte/global_index_test.cpp +++ b/test/inte/global_index_test.cpp @@ -129,7 +129,7 @@ class GlobalIndexTest : public ::testing::Test, public ::testing::WithParamInter write_builder.WithFileSystem(fs_); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, write_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(auto file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(std::move(write_context), {})); ArrowArray c_array; EXPECT_TRUE(arrow::ExportArray(*write_array, &c_array).ok()); auto record_batch = std::make_unique( @@ -157,7 +157,7 @@ class GlobalIndexTest : public ::testing::Test, public ::testing::WithParamInter PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, commit_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_store_commit, - FileStoreCommit::Create(std::move(commit_context))); + FileStoreCommit::Create(std::move(commit_context), {})); return file_store_commit->Commit(commit_msgs); } @@ -168,7 +168,7 @@ class GlobalIndexTest : public ::testing::Test, public ::testing::WithParamInter scan_context_builder.SetPartitionFilter(partition_filters); scan_context_builder.WithFileSystem(fs_); PAIMON_ASSIGN_OR_RAISE(auto scan_context, scan_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context), {})); PAIMON_ASSIGN_OR_RAISE(auto result_plan, table_scan->CreatePlan()); EXPECT_EQ(result_plan->Splits().size(), 1); return std::dynamic_pointer_cast(result_plan->Splits()[0]); @@ -396,16 +396,18 @@ class GlobalIndexTest : public ::testing::Test, public ::testing::WithParamInter const std::map& options = {}, const std::shared_ptr& index_result = nullptr, const std::shared_ptr& cache = nullptr) const { + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); + scan_context_builder_options = options; scan_context_builder.SetPredicate(predicate) - .SetOptions(options) .SetGlobalIndexResult(index_result) .WithFileSystem(fs_); if (cache) { scan_context_builder.WithCache(cache); } PAIMON_ASSIGN_OR_RAISE(auto scan_context, scan_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context), + scan_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto result_plan, table_scan->CreatePlan()); return result_plan; } @@ -444,7 +446,7 @@ class GlobalIndexTest : public ::testing::Test, public ::testing::WithParamInter .EnableLateMaterializing(false); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context), {})); PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(splits)); PAIMON_ASSIGN_OR_RAISE(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); diff --git a/test/inte/nested_column_pruning_inte_test.cpp b/test/inte/nested_column_pruning_inte_test.cpp index b9b1b99ca..b20aab5f1 100644 --- a/test/inte/nested_column_pruning_inte_test.cpp +++ b/test/inte/nested_column_pruning_inte_test.cpp @@ -92,13 +92,16 @@ class NestedColumnPruningInteTest : public ::testing::Test, const std::shared_ptr& expected_schema, const std::string& expected_json, const std::shared_ptr& predicate = nullptr) const { + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_MODE, StartupMode::LatestFull().ToString()); + scan_context_builder_options[Options::SCAN_MODE] = StartupMode::LatestFull().ToString(); + if (predicate) { scan_context_builder.SetPredicate(predicate); } ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context), + scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_FALSE(result_plan->Splits().empty()); @@ -110,7 +113,7 @@ class NestedColumnPruningInteTest : public ::testing::Test, read_context_builder.SetPredicate(predicate); } ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context), {})); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -350,10 +353,12 @@ TEST_P(NestedColumnPruningInteTest, QueryStructSubFieldsAllNonExistent) { auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path_); - read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); + read_context_builder_options = options; + read_context_builder.SetReadSchema(std::move(c_schema)); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), + ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context), read_context_builder_options), "does not support schema evolution inside struct"); } @@ -412,10 +417,12 @@ TEST_P(NestedColumnPruningInteTest, QueryStructSubFieldsWithNonExistentField) { auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path_); - read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); + read_context_builder_options = options; + read_context_builder.SetReadSchema(std::move(c_schema)); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), + ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context), read_context_builder_options), "does not support schema evolution inside struct"); } @@ -491,10 +498,13 @@ TEST_P(NestedColumnPruningInteTest, QueryStructSubFieldsWithTypeMismatchShouldFa helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); ASSERT_FALSE(data_splits.empty()); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path_); - read_context_builder.SetOptions(options); + read_context_builder_options = options; + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_NOK_WITH_MSG(table_read->CreateReader(data_splits), "PruneDataType nested field type mismatch for 'a': read string vs " @@ -581,10 +591,13 @@ TEST_P(NestedColumnPruningInteTest, auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path_); - read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); + read_context_builder_options = options; + read_context_builder.SetReadSchema(std::move(c_schema)); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto data_splits, helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); ASSERT_FALSE(data_splits.empty()); @@ -766,10 +779,13 @@ TEST_P(NestedColumnPruningInteTest, PruneNestedStructWithSpecialFields) { auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path_); - read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); + read_context_builder_options = options; + read_context_builder.SetReadSchema(std::move(c_schema)); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -1178,10 +1194,15 @@ TEST_P(NestedColumnPruningInteTest, NestedStructMapSelectedKeysWithPredicate) { ASSERT_OK_AND_ASSIGN( auto helper, TestHelper::Create(test_dir_, table_schema, /*partition_keys=*/{}, /*primary_keys=*/{}, options, /*is_streaming_mode=*/false)); + std::map write_context_builder_options; WriteContextBuilder write_context_builder(table_path_, "commit_user_1"); + write_context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - write_context_builder.SetOptions(options).Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + + write_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto file_store_write, + FileStoreWrite::Create(std::move(write_context), write_context_builder_options)); auto write_one_row = [&](const std::string& data) -> Status { PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch, @@ -1201,10 +1222,14 @@ TEST_P(NestedColumnPruningInteTest, NestedStructMapSelectedKeysWithPredicate) { /*commit_identifier=*/0)); ASSERT_OK(file_store_write->Close()); + std::map commit_context_builder_options; CommitContextBuilder commit_context_builder(table_path_, "commit_user_1"); + commit_context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - commit_context_builder.SetOptions(options).Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + + commit_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context), + commit_context_builder_options)); ASSERT_OK(commit->Commit(commit_msgs, /*commit_identifier=*/0)); // Read selected MAP keys together with nested STRUCT sub-fields. @@ -1309,10 +1334,13 @@ TEST_P(NestedColumnPruningInteTest, MapSelectedKeysWithOrcDictionaryEncodedMap) auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path_); - read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); + read_context_builder_options = options; + read_context_builder.SetReadSchema(std::move(c_schema)); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -1464,10 +1492,13 @@ TEST_P(NestedColumnPruningInteTest, PruneListStructSubFields) { auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path_); - read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); + read_context_builder_options = options; + read_context_builder.SetReadSchema(std::move(c_schema)); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); auto create_reader_result = table_read->CreateReader(data_splits); ASSERT_NOK_WITH_MSG(create_reader_result, "partial projection inside list"); } diff --git a/test/inte/paimon_read_compat_inte_test.cpp b/test/inte/paimon_read_compat_inte_test.cpp index 5761720d4..23c1ecca9 100644 --- a/test/inte/paimon_read_compat_inte_test.cpp +++ b/test/inte/paimon_read_compat_inte_test.cpp @@ -71,20 +71,26 @@ Result> ReadTable( std::string table_path = GetDataDir() + "/" + file_format + "/" + kDatabaseName + ".db/" + table_name; + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.SetOptions(options); + scan_context_builder_options = options; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, table_scan->CreatePlan()); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - read_context_builder.SetOptions(options).SetReadFieldNames(field_names); + read_context_builder_options = options; + read_context_builder.SetReadFieldNames(field_names); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); return ReadResultCollector::CollectResult(std::move(batch_reader)); diff --git a/test/inte/pk_compaction_inte_test.cpp b/test/inte/pk_compaction_inte_test.cpp index 153a7926c..28987f6f1 100644 --- a/test/inte/pk_compaction_inte_test.cpp +++ b/test/inte/pk_compaction_inte_test.cpp @@ -351,15 +351,17 @@ class PkCompactionInteTest : public ::testing::Test { int32_t bucket, const std::shared_ptr& write_array, int64_t commit_identifier, bool is_streaming = true, const std::vector& row_kinds = {}, bool write_only = true) const { + std::map write_builder_options; WriteContextBuilder write_builder(table_path, "commit_user_1"); write_builder.WithStreamingMode(is_streaming) .WithTempDirectory(PathUtil::JoinPath(dir_->Str(), "tmp")); if (write_only) { - write_builder.AddOption(Options::WRITE_ONLY, "true"); + write_builder_options[Options::WRITE_ONLY] = "true"; } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, write_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto file_store_write, - FileStoreWrite::Create(std::move(write_context))); + PAIMON_ASSIGN_OR_RAISE( + auto file_store_write, + FileStoreWrite::Create(std::move(write_context), write_builder_options)); ArrowArray c_array; PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*write_array, &c_array)); auto record_batch = std::make_unique(partition, bucket, row_kinds, &c_array); @@ -376,7 +378,7 @@ class PkCompactionInteTest : public ::testing::Test { PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, commit_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_store_commit, - FileStoreCommit::Create(std::move(commit_context))); + FileStoreCommit::Create(std::move(commit_context), {})); return file_store_commit->Commit(commit_msgs); } @@ -397,13 +399,16 @@ class PkCompactionInteTest : public ::testing::Test { const std::string& table_path, const std::map& partition, int32_t bucket, bool full_compaction, int64_t commit_identifier, const std::map& options = {}) { + std::map write_builder_options; WriteContextBuilder write_builder(table_path, "commit_user_1"); - write_builder.SetOptions(options); + write_builder_options = options; + write_builder.WithStreamingMode(true).WithTempDirectory( PathUtil::JoinPath(dir_->Str(), "tmp")); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, write_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto file_store_write, - FileStoreWrite::Create(std::move(write_context))); + PAIMON_ASSIGN_OR_RAISE( + auto file_store_write, + FileStoreWrite::Create(std::move(write_context), write_builder_options)); PAIMON_RETURN_NOT_OK(file_store_write->Compact(partition, bucket, full_compaction)); PAIMON_ASSIGN_OR_RAISE( std::vector> commit_messages, @@ -422,16 +427,19 @@ class PkCompactionInteTest : public ::testing::Test { std::map options = {{Options::FILE_SYSTEM, "local"}}; ASSERT_OK_AND_ASSIGN(auto helper, TestHelper::Create(table_path, options, /*is_streaming_mode=*/false)); + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.WithStreamingMode(false).SetOptions(options).AddOption( - Options::SCAN_MODE, StartupMode::LatestFull().ToString()); + scan_context_builder_options = options; + scan_context_builder_options[Options::SCAN_MODE] = StartupMode::LatestFull().ToString(); + scan_context_builder.WithStreamingMode(false); if (predicate) { scan_context_builder.SetPredicate(predicate); } ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr result_plan, table_scan->CreatePlan()); std::vector> data_splits = result_plan->Splits(); @@ -456,15 +464,18 @@ class PkCompactionInteTest : public ::testing::Test { auto iter = expected_data_per_partition_bucket.find(key); ASSERT_TRUE(iter != expected_data_per_partition_bucket.end()) << "Unexpected partition=" << key.first << " bucket=" << key.second; + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - read_context_builder.SetOptions(options); + read_context_builder_options = options; + if (predicate) { read_context_builder.SetPredicate(predicate); } ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(std::shared_ptr read_result, @@ -490,20 +501,26 @@ class PkCompactionInteTest : public ::testing::Test { template void ScanAllRows(const std::string& table_path, Fn consume) { std::map options = {{Options::FILE_SYSTEM, "local"}}; + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.WithStreamingMode(false).SetOptions(options).AddOption( - Options::SCAN_MODE, StartupMode::LatestFull().ToString()); + scan_context_builder_options = options; + scan_context_builder_options[Options::SCAN_MODE] = StartupMode::LatestFull().ToString(); + scan_context_builder.WithStreamingMode(false); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr result_plan, table_scan->CreatePlan()); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - read_context_builder.SetOptions(options); + read_context_builder_options = options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(result_plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, @@ -617,13 +634,14 @@ TEST_F(PkCompactionInteTest, TestMetadataOnlyLevelUpgradeKeepsValueStats) { ASSERT_FALSE(before->value_stats == SimpleStats::EmptyStats()); ASSERT_EQ(before->value_stats, after->value_stats); + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.WithStreamingMode(false) - .AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::SCAN_MODE, StartupMode::LatestFull().ToString()); + scan_context_builder_options[Options::FILE_SYSTEM] = "local"; + scan_context_builder_options[Options::SCAN_MODE] = StartupMode::LatestFull().ToString(); + scan_context_builder.WithStreamingMode(false); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + TableScan::Create(std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, table_scan->CreatePlan()); ASSERT_EQ(1u, plan->Splits().size()); auto split = std::dynamic_pointer_cast(plan->Splits()[0]); @@ -999,7 +1017,7 @@ TEST_F(PkCompactionInteTest, DeduplicateWithBitmap64HighPositions) { // Reload committed metadata and DV bytes; no test-created DV or lookup result is injected. ScanContextBuilder scan_builder(table_path); ASSERT_OK_AND_ASSIGN(auto scan_context, scan_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan, TableScan::Create(std::move(scan_context), {})); ASSERT_OK_AND_ASSIGN(auto plan, scan->CreatePlan()); int32_t dv_count = 0; for (const auto& split : plan->Splits()) { @@ -1746,10 +1764,13 @@ TEST_F(PkCompactionInteTest, CompactWithSchemaEvolution) { // Step 5: Read and verify using DataSplits from CompactAfter in write CommitMessages. { + std::map context_builder_options; ReadContextBuilder context_builder(table_path); - context_builder.AddOption(Options::FILE_FORMAT, "parquet"); + context_builder_options[Options::FILE_FORMAT] = "parquet"; + ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); // Use the latest snapshot id for both splits. // The original table has 6 snapshots. Each write with force-compact produces @@ -2016,7 +2037,7 @@ TEST_F(PkCompactionInteTest, ContinuousWriteWithBackgroundCompact) { PathUtil::JoinPath(dir_->Str(), "tmp")); ASSERT_OK_AND_ASSIGN(auto write_context, write_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(std::move(write_context), {})); auto write_batch = [&](const std::string& json_data) -> Status { auto array = @@ -2220,12 +2241,13 @@ TEST_F(PkCompactionInteTest, WriteAndCompactWithBranch) { auto data_type = arrow::struct_(fields); // Step 2: Write data to the branch-rt and compact. + std::map write_builder_options; WriteContextBuilder write_builder(table_path, "commit_user_1"); - write_builder.WithStreamingMode(true) - .WithTempDirectory(PathUtil::JoinPath(dir_->Str(), "tmp")) - .WithBranch("rt"); + write_builder_options[Options::BRANCH] = "rt"; + write_builder.WithStreamingMode(true).WithTempDirectory(PathUtil::JoinPath(dir_->Str(), "tmp")); ASSERT_OK_AND_ASSIGN(auto write_context, write_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN(auto file_store_write, + FileStoreWrite::Create(std::move(write_context), write_builder_options)); // dt=20240726, bucket-0 only has ["20240726", "cherry", 3] one row auto write_array = arrow::ipc::internal::json::ArrayFromJSON(data_type, R"([ @@ -2276,11 +2298,15 @@ TEST_F(PkCompactionInteTest, WriteAndCompactWithBranch) { ASSERT_EQ(compact_after[0]->row_count, 2); // Step 4: Commit to branch-rt. + std::map commit_builder_options; CommitContextBuilder commit_builder(table_path, "commit_user_1"); - commit_builder.AddOption(Options::FILE_SYSTEM, "local").AddOption(Options::BRANCH, "rt"); + commit_builder_options[Options::FILE_SYSTEM] = "local"; + commit_builder_options[Options::BRANCH] = "rt"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_commit, - FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr file_store_commit, + FileStoreCommit::Create(std::move(commit_context), commit_builder_options)); ASSERT_OK(file_store_commit->Commit(commit_msgs)); auto fs = std::make_shared(); @@ -2299,13 +2325,14 @@ TEST_F(PkCompactionInteTest, WriteAndCompactWithBranch) { // Step 5: Scan and read branch-rt. std::map branch_options = {{Options::FILE_SYSTEM, "local"}, {Options::BRANCH, "rt"}}; + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.WithStreamingMode(false) - .SetOptions(branch_options) - .AddOption(Options::SCAN_MODE, StartupMode::LatestFull().ToString()); + scan_context_builder_options = branch_options; + scan_context_builder_options[Options::SCAN_MODE] = StartupMode::LatestFull().ToString(); + scan_context_builder.WithStreamingMode(false); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + TableScan::Create(std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, table_scan->CreatePlan()); std::vector> splits = plan->Splits(); @@ -2316,11 +2343,13 @@ TEST_F(PkCompactionInteTest, WriteAndCompactWithBranch) { ASSERT_EQ(split_impl->SnapshotId(), 3); } + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - read_context_builder.SetOptions(branch_options); + read_context_builder_options = branch_options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(std::shared_ptr result_array, diff --git a/test/inte/primary_key_sorted_index_inte_test.cpp b/test/inte/primary_key_sorted_index_inte_test.cpp index f2811a699..bd5729ce8 100644 --- a/test/inte/primary_key_sorted_index_inte_test.cpp +++ b/test/inte/primary_key_sorted_index_inte_test.cpp @@ -199,24 +199,25 @@ class PrimaryKeySortedIndexInteTest : public ::testing::Test, const std::optional& snapshot_id, bool index_enabled, const std::vector>& partition_filters, const std::string& branch, const std::shared_ptr& file_system) const { + std::map builder_options; ScanContextBuilder builder(TablePath(partitioned)); - builder.SetPredicate(predicate).AddOption(Options::GLOBAL_INDEX_ENABLED, - index_enabled ? "true" : "false"); + builder_options[Options::GLOBAL_INDEX_ENABLED] = index_enabled ? "true" : "false"; + builder.SetPredicate(predicate); if (snapshot_id != std::nullopt) { - builder.AddOption(Options::SCAN_SNAPSHOT_ID, fmt::format("{}", snapshot_id.value())); + builder_options[Options::SCAN_SNAPSHOT_ID] = fmt::format("{}", snapshot_id.value()); } if (!partition_filters.empty()) { builder.SetPartitionFilter(partition_filters); } if (!branch.empty()) { - builder.AddOption(Options::BRANCH, branch); + builder_options[Options::BRANCH] = branch; } if (file_system != nullptr) { builder.WithFileSystem(file_system); } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_scan, - TableScan::Create(std::move(context))); + TableScan::Create(std::move(context), builder_options)); return table_scan->CreatePlan(); } @@ -224,6 +225,7 @@ class PrimaryKeySortedIndexInteTest : public ::testing::Test, const std::vector>& splits, const std::map& options, const std::shared_ptr& file_system) const { + std::map builder_options; ReadContextBuilder builder(TablePath(partitioned)); if (partitioned) { builder.SetReadFieldNames({"pt", "id", "score", "tag"}); @@ -232,14 +234,14 @@ class PrimaryKeySortedIndexInteTest : public ::testing::Test, } builder.SetPredicate(predicate).EnablePredicateFilter(true); for (const auto& [key, value] : options) { - builder.AddOption(key, value); + builder_options[key] = value; } if (file_system != nullptr) { builder.WithFileSystem(file_system); } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, - TableRead::Create(std::move(context))); + TableRead::Create(std::move(context), builder_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch_reader, table_read->CreateReader(splits)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr result, @@ -547,7 +549,7 @@ TEST_P(PrimaryKeySortedIndexInteTest, ScoredSplitFailsBeforeForceKeepDeleteFallb .EnablePredicateFilter(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(context))); + TableRead::Create(std::move(context), {})); auto* key_value_table_read = dynamic_cast(table_read.get()); ASSERT_TRUE(key_value_table_read != nullptr); key_value_table_read->ForceKeepDelete(true); diff --git a/test/inte/read_inte_test.cpp b/test/inte/read_inte_test.cpp index 5aa2f22c3..9adc162b1 100644 --- a/test/inte/read_inte_test.cpp +++ b/test/inte/read_inte_test.cpp @@ -280,22 +280,28 @@ Result> ReadSystemTable( PAIMON_ASSIGN_OR_RAISE( std::shared_ptr table, SystemTableLoader::LoadFromPath(core_options.GetFileSystem(), system_table_path, options)); + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(system_table_path); - scan_context_builder.SetOptions(options).WithStreamingMode(streaming_mode); + scan_context_builder_options = options; + scan_context_builder.WithStreamingMode(streaming_mode); if (predicate) { scan_context_builder.SetPredicate(predicate); } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_context_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured_table_1, + table->Copy(scan_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_scan, - table->NewScan(std::move(scan_context))); + configured_table_1->NewScan(std::move(scan_context))); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, table_scan->CreatePlan()); if (read_next_plan) { PAIMON_ASSIGN_OR_RAISE(plan, table_scan->CreatePlan()); } + std::map read_context_builder_options; ReadContextBuilder read_context_builder(system_table_path); - read_context_builder.SetOptions(options); + read_context_builder_options = options; + if (predicate) { read_context_builder.SetPredicate(predicate); } @@ -304,8 +310,10 @@ Result> ReadSystemTable( } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured_table_2, + table->Copy(read_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, - table->NewRead(std::move(read_context))); + configured_table_2->NewRead(std::move(read_context))); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr result, @@ -377,18 +385,21 @@ TEST_P(ReadInteTest, TestAppendSimple) { DataField(2, arrow::field("f2", arrow::int32())), DataField(3, arrow::field("f3", arrow::float64()))}; + std::map context_builder_options; ReadContextBuilder context_builder(path); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", "false") - .AddOption("orc.read.enable-metrics", "true"); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + + context_builder_options["test.enable-adaptive-prefetch-strategy"] = "false"; + context_builder_options["orc.read.enable-metrics"] = "true"; + context_builder.EnablePrefetch(param.enable_prefetch); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); if (specific_table_schema) { context_builder.SetTableSchema(specific_table_schema.value()); } ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); std::vector file_list; if (param.file_format == "orc") { @@ -460,18 +471,21 @@ TEST_P(ReadInteTest, TestReadWithLimits) { auto param = GetParam(); std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/append_09.db/append_09"; + std::map context_builder_options; ReadContextBuilder context_builder(path); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption(Options::READ_BATCH_SIZE, "1"); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options[Options::READ_BATCH_SIZE] = "1"; + + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder_options["orc.read.enable-metrics"] = "true"; context_builder.EnablePrefetch(param.enable_prefetch) .SetReadAheadCacheEnabled(param.read_ahead_cache_enabled) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy) - .AddOption("orc.read.enable-metrics", "true") .SetPrefetchBatchCount(10); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); std::vector file_list; if (param.file_format == "orc") { @@ -523,14 +537,17 @@ TEST_P(ReadInteTest, TestReadAheadCacheMetrics) { auto param = GetParam(); std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/append_09.db/append_09"; + std::map context_builder_options; ReadContextBuilder context_builder(path); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + + context_builder_options["test.enable-adaptive-prefetch-strategy"] = "false"; context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", "false") .SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); std::vector file_list; if (param.file_format == "orc") { @@ -597,16 +614,19 @@ TEST_P(ReadInteTest, TestReadOnlyPartitionField) { DataField(0, arrow::field("dt", arrow::utf8())), }; + std::map context_builder_options; ReadContextBuilder context_builder(path); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder.SetReadFieldNames({"dt"}); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); std::vector> data_splits; data_splits.reserve(3); @@ -657,17 +677,25 @@ TEST(SystemTableReadInteTest, TestReadOptionsSystemTable) { std::string system_table_path = PathUtil::JoinPath(dir->Str(), "warehouse/db1.db/tbl1$options"); ASSERT_OK_AND_ASSIGN(auto table, catalog->GetTable(Identifier("db1", "tbl1$options"))); + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(system_table_path); - scan_context_builder.SetOptions(options); + scan_context_builder_options = options; + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, table->NewScan(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_3, + table->Copy(scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, configured_table_3->NewScan(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto plan, table_scan->CreatePlan()); ASSERT_EQ(plan->Splits().size(), 1); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(system_table_path); - read_context_builder.SetOptions(options); + read_context_builder_options = options; + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, table->NewRead(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_4, + table->Copy(read_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_read, configured_table_4->NewRead(std::move(read_context))); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(auto result, ReadResultCollector::CollectResult(std::move(batch_reader))); ASSERT_TRUE(result); @@ -692,22 +720,33 @@ TEST(SystemTableReadInteTest, TestReadBranchOptionsSystemTable) { ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options)); ASSERT_OK_AND_ASSIGN(auto table, SystemTableLoader::LoadFromPath(core_options.GetFileSystem(), system_table_path, options)); + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(system_table_path); - scan_context_builder.SetOptions(options); + scan_context_builder_options = options; + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, table->NewScan(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_5, + table->Copy(scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, configured_table_5->NewScan(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto plan, table_scan->CreatePlan()); ASSERT_EQ(plan->Splits().size(), 1); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(system_table_path); - read_context_builder.SetOptions(options); + read_context_builder_options = options; + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, table->NewRead(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_6, + table->Copy(read_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_read, configured_table_6->NewRead(std::move(read_context))); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(auto result, ReadResultCollector::CollectResult(std::move(batch_reader))); - std::map expected = { - {"bucket", "2"}, {"file.format", "parquet"}, {"manifest.format", "avro"}}; + std::map expected = {{"branch", "rt"}, + {"bucket", "2"}, + {"file-system", "local"}, + {"file.format", "parquet"}, + {"manifest.format", "avro"}}; ASSERT_EQ(CollectStringMap(result), expected) << result->ToString(); } @@ -1016,23 +1055,31 @@ TEST(SystemTableReadInteTest, TestReadOptimizedPrimaryKeyProjectionAndPredicateP ASSERT_OK_AND_ASSIGN( std::shared_ptr table, SystemTableLoader::LoadFromPath(core_options.GetFileSystem(), table_path + "$ro", options)); + std::map ro_scan_context_builder_options; ScanContextBuilder ro_scan_context_builder(table_path + "$ro"); - ro_scan_context_builder.SetOptions(options).SetPredicate(predicate); + ro_scan_context_builder_options = options; + ro_scan_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(std::unique_ptr ro_scan_context, ro_scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_7, + table->Copy(ro_scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr ro_table_scan, - table->NewScan(std::move(ro_scan_context))); + configured_table_7->NewScan(std::move(ro_scan_context))); ASSERT_OK_AND_ASSIGN(std::shared_ptr ro_plan, ro_table_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(int64_t ro_file_count, CountDataFiles(ro_plan->Splits())); ASSERT_EQ(ro_file_count, 1); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path + "$ro"); // Do not enable row-level predicate filtering: the result must come from the file reader's // row-group/page predicate pushdown. - read_context_builder.SetOptions(options).SetPredicate(predicate).SetReadFieldNames({"k", "v"}); + read_context_builder_options = options; + read_context_builder.SetPredicate(predicate).SetReadFieldNames({"k", "v"}); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_8, + table->Copy(read_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - table->NewRead(std::move(read_context))); + configured_table_8->NewRead(std::move(read_context))); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(ro_plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, @@ -1080,11 +1127,15 @@ TEST(SystemTableReadInteTest, TestReadOptimizedSystemTableNestedProjection) { ASSERT_OK_AND_ASSIGN( std::shared_ptr table, SystemTableLoader::LoadFromPath(core_options.GetFileSystem(), table_path + "$ro", options)); + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path + "$ro"); - scan_context_builder.SetOptions(options); + scan_context_builder_options = options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_9, + table->Copy(scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - table->NewScan(std::move(scan_context))); + configured_table_9->NewScan(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, table_scan->CreatePlan()); auto projected_schema = arrow::schema({ @@ -1093,11 +1144,15 @@ TEST(SystemTableReadInteTest, TestReadOptimizedSystemTableNestedProjection) { }); auto c_projected_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_projected_schema.get()).ok()); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path + "$ro"); - read_context_builder.SetOptions(options).SetReadSchema(std::move(c_projected_schema)); + read_context_builder_options = options; + read_context_builder.SetReadSchema(std::move(c_projected_schema)); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_10, + table->Copy(read_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - table->NewRead(std::move(read_context))); + configured_table_10->NewRead(std::move(read_context))); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, @@ -1294,7 +1349,7 @@ TEST(SystemTableReadInteTest, TestReadFilesSystemTableForPartitionedPartialWrite ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(std::move(write_context), {})); ASSERT_OK_AND_ASSIGN( std::unique_ptr batch, TestHelper::MakeRecordBatch(arrow::struct_({fields[2]}), R"([[10], [20]])", @@ -1308,7 +1363,7 @@ TEST(SystemTableReadInteTest, TestReadFilesSystemTableForPartitionedPartialWrite ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_context))); + FileStoreCommit::Create(std::move(commit_context), {})); ASSERT_OK(commit->Commit(commit_messages)); ASSERT_OK_AND_ASSIGN(auto files_result, ReadSystemTable(table_path + "$files", options)); @@ -1618,7 +1673,7 @@ TEST(SystemTableReadInteTest, TestReadAuditLogSystemTableWithSequenceNumber) { std::map options = { {Options::FILE_SYSTEM, "local"}, {Options::FILE_FORMAT, "parquet"}, - + {Options::TABLE_READ_SEQUENCE_NUMBER_ENABLED, "true"}, {Options::BUCKET, "1"}, }; auto dir = UniqueTestDirectory::Create(); @@ -1635,7 +1690,8 @@ TEST(SystemTableReadInteTest, TestReadAuditLogSystemTableWithSequenceNumber) { /*expected_commit_messages=*/std::nullopt)); auto read_options = options; - read_options[Options::TABLE_READ_SEQUENCE_NUMBER_ENABLED] = "true"; + // Java allows the persisted table property, but rejects enabling it through a Copy hint. + read_options.erase(Options::TABLE_READ_SEQUENCE_NUMBER_ENABLED); ASSERT_OK_AND_ASSIGN( auto result, ReadSystemTable(PathUtil::JoinPath(dir->Str(), "foo.db/bar$audit_log"), read_options)); @@ -1928,17 +1984,19 @@ TEST_P(ReadInteTest, TestAppendReadWithMultipleBuckets) { auto param = GetParam(); std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/append_09.db/append_09"; + std::map context_builder_options; ReadContextBuilder context_builder(path); context_builder.SetReadFieldNames({"f3", "f0", "f1"}); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2") - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy) - .EnablePrefetch(param.enable_prefetch); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); std::vector file_list_0; std::vector file_list_1; @@ -2009,19 +2067,20 @@ TEST_P(ReadInteTest, TestAppendReadWithPredicate) { std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/append_09.db/append_09"; + std::map context_builder_options; ReadContextBuilder context_builder(path); context_builder.SetReadFieldNames({"f3", "f0", "f1"}); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .SetPredicate(predicate) - .EnablePredicateFilter(true) - .EnablePrefetch(param.enable_prefetch) - .AddOption("read.batch-size", "2") - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy) - .AddOption("orc.read.enable-metrics", "true"); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder_options["orc.read.enable-metrics"] = "true"; + context_builder.SetPredicate(predicate).EnablePredicateFilter(true).EnablePrefetch( + param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); std::vector file_list_0; std::vector file_list_1; @@ -2111,18 +2170,21 @@ TEST_P(ReadInteTest, TestAppendReadWithComplexTypePredicate) { std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/append_complex_data.db/append_complex_data"; + std::map context_builder_options; ReadContextBuilder context_builder(path); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); context_builder.SetReadFieldNames({"f6", "f2", "f4", "f3", "f5"}); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2"); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + context_builder.SetPredicate(predicate); context_builder.EnablePredicateFilter(true); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); std::vector file_list_0; std::vector file_list_1; @@ -2185,19 +2247,20 @@ TEST_P(ReadInteTest, TestAppendReadWithPredicateOnlyPushdown) { std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/append_09.db/append_09"; + std::map context_builder_options; ReadContextBuilder context_builder(path); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); context_builder.SetReadFieldNames({"f3", "f0", "f1"}); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2") - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy) - .SetPredicate(predicate) - .EnableLateMaterializing(false) - .EnablePrefetch(param.enable_prefetch); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.SetPredicate(predicate).EnableLateMaterializing(false).EnablePrefetch( + param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); std::vector file_list_0; std::vector file_list_1; @@ -2269,18 +2332,19 @@ TEST_P(ReadInteTest, TestAppendReadWithLateMaterializing) { std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/append_09.db/append_09"; + std::map context_builder_options; ReadContextBuilder context_builder(path); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); context_builder.SetReadFieldNames({"f3", "f0", "f1"}); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2") - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy) - .SetPredicate(predicate) - .EnableLateMaterializing(true) - .EnablePrefetch(param.enable_prefetch); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.SetPredicate(predicate).EnableLateMaterializing(true).EnablePrefetch( + param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); std::vector file_list_0; std::vector file_list_1; @@ -2347,19 +2411,20 @@ TEST_P(ReadInteTest, TestAppendReadWithPredicateAllFiltered) { std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/append_09.db/append_09"; + std::map context_builder_options; ReadContextBuilder context_builder(path); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); context_builder.SetReadFieldNames({"f3", "f0", "f1"}); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2") - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy) - .SetPredicate(predicate) - .EnablePredicateFilter(true) - .EnablePrefetch(param.enable_prefetch); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.SetPredicate(predicate).EnablePredicateFilter(true).EnablePrefetch( + param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); std::vector file_list_0; std::vector file_list_1; @@ -2433,18 +2498,20 @@ TEST_P(ReadInteTest, TestAppendReadIOException) { for (size_t i = 0; i < 200; i++) { ScopeGuard guard([&io_hook]() { io_hook->Clear(); }); io_hook->Reset(i, IOHook::Mode::RETURN_ERROR); + std::map context_builder_options; ReadContextBuilder context_builder(paimon::test::GetDataDir() + "/" + param.file_format + "/append_09.db/append_09/"); context_builder.SetReadFieldNames({"f3", "f0", "f1"}); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2") - .EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - Result> table_read = TableRead::Create(std::move(read_context)); + Result> table_read = + TableRead::Create(std::move(read_context), context_builder_options); CHECK_HOOK_STATUS(table_read.status(), i); Result> batch_reader = table_read.value()->CreateReader(data_splits); @@ -2479,16 +2546,19 @@ TEST_P(ReadInteTest, TestPkTableWithDeletionVectorSimple) { data_file = "data-ed5d7184-cf42-4f4a-bf83-1e3080d9012d-0.parquet"; dv_file = "index/index-62372b7b-d1cf-4dde-a162-c9d9d35006b4-1"; } + std::map context_builder_options; ReadContextBuilder context_builder(path); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2"); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); DataSplitsDv input_data_splits = { {path + "f1=10/bucket-1", @@ -2528,16 +2598,18 @@ TEST_P(ReadInteTest, TestPkTableWithDeletionVector) { auto param = GetParam(); std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/pk_09.db/pk_09"; + std::map context_builder_options; ReadContextBuilder context_builder(path); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2") - .EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); std::vector file_list; std::string deletion_file; @@ -2595,19 +2667,22 @@ TEST_P(ReadInteTest, TestPkTableWithSnapshot6) { auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/3, /*field_name=*/"f3", FieldType::DOUBLE, Literal(15.0)); std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/pk_09.db/pk_09"; + std::map context_builder_options; ReadContextBuilder context_builder(path); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2"); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + context_builder.SetPredicate(predicate); context_builder.EnablePredicateFilter(true); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); std::vector file_list_0; std::vector file_list_1; @@ -2681,17 +2756,20 @@ TEST_P(ReadInteTest, TestPkTableWithSnapshot8) { auto param = GetParam(); std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/pk_09.db/pk_09"; + std::map context_builder_options; ReadContextBuilder context_builder(path); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); context_builder.SetReadFieldNames({"f0", "f3", "f1"}); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2"); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); std::vector file_list_0; std::vector file_list_1; @@ -2760,20 +2838,23 @@ TEST_P(ReadInteTest, TestAppendReadWithSchemaEvolution) { DataField(5, arrow::field("a", arrow::int32())), DataField(8, arrow::field("e", arrow::int32()))}; + std::map context_builder_options; ReadContextBuilder context_builder(path); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2"); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); if (specific_table_schema) { context_builder.SetTableSchema(specific_table_schema.value()); } ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); std::vector file_list_0; std::vector file_list_1; @@ -2867,15 +2948,18 @@ TEST_P(ReadInteTest, TestAppendReadWithSchemaEvolutionWithReaderBuildParallelism /*schema ids*/ {1, 0}}}; auto data_splits = CreateDataSplits(input_data_splits, /*snapshot_id=*/2); + std::map context_builder_options; ReadContextBuilder context_builder(path); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2"); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); ASSERT_OK_AND_ASSIGN(auto result_array, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -2932,20 +3016,23 @@ TEST_P(ReadInteTest, TestAppendReadWithSchemaEvolutionWithPredicateFilter) { auto param = GetParam(); std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/append_table_with_alter_table.db/append_table_with_alter_table/"; + std::map context_builder_options; ReadContextBuilder context_builder(path); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); context_builder.SetReadFieldNames({"a", "k", "key1", "d", "key0", "c"}); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2"); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + context_builder.SetPredicate(predicate); context_builder.EnablePredicateFilter(true); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); std::vector file_list_0; std::vector file_list_1; @@ -3012,18 +3099,21 @@ TEST_P(ReadInteTest, TestAppendReadWithSchemaEvolutionWithPredicateOnlyPushDown) std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/append_table_with_alter_table.db/" "append_table_with_alter_table/"; + std::map context_builder_options; ReadContextBuilder context_builder(path); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); context_builder.SetReadFieldNames({"a", "k", "key1", "d", "key0", "c"}); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2"); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + context_builder.SetPredicate(predicate); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); std::vector file_list_0; std::vector file_list_1; @@ -3086,17 +3176,20 @@ TEST_P(ReadInteTest, TestPkReadSnapshot5WithSchemaEvolution) { std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/pk_table_with_alter_table.db/pk_table_with_alter_table/"; + std::map context_builder_options; ReadContextBuilder context_builder(path); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); context_builder.SetReadFieldNames({"key1", "k", "key_2", "c", "d", "a", "key0", "e"}); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2"); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); std::vector file_list_0; std::vector file_list_1; @@ -3172,17 +3265,20 @@ TEST_P(ReadInteTest, TestPkReadSnapshot6WithSchemaEvolution) { auto param = GetParam(); std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/pk_table_with_alter_table.db/pk_table_with_alter_table/"; + std::map context_builder_options; ReadContextBuilder context_builder(path); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); context_builder.SetReadFieldNames({"key1", "k", "key_2", "c", "d", "a", "key0", "e"}); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2"); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); std::vector file_list_0; std::vector file_list_1; @@ -3258,18 +3354,20 @@ TEST_P(ReadInteTest, TestPkReadSnapshot6WithSchemaEvolutionWithPredicateOnlyPush FieldType::INT, Literal(510)); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({equal, less_than})); + std::map context_builder_options; ReadContextBuilder context_builder(path); context_builder.SetReadFieldNames({{"key1", "k", "key_2", "c", "d", "a", "key0", "e"}}); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2"); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); context_builder.SetPredicate(predicate); - context_builder.EnablePrefetch(param.enable_prefetch) - .EnableLateMaterializing(false) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch).EnableLateMaterializing(false); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); std::vector file_list_0; std::vector file_list_1; @@ -3344,18 +3442,20 @@ TEST_P(ReadInteTest, TestPkReadSnapshot6WithSchemaEvolutionWithLateMaterializing FieldType::INT, Literal(510)); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({equal, less_than})); + std::map context_builder_options; ReadContextBuilder context_builder(path); context_builder.SetReadFieldNames({{"key1", "k", "key_2", "c", "d", "a", "key0", "e"}}); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2"); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); context_builder.SetPredicate(predicate); - context_builder.EnableLateMaterializing(true) - .EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnableLateMaterializing(true).EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); std::vector file_list_0; std::vector file_list_1; @@ -3428,19 +3528,22 @@ TEST_P(ReadInteTest, TestPkReadSnapshot6WithSchemaEvolutionWithPredicateFilter) FieldType::INT, Literal(510)); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({equal, less_than})); + std::map context_builder_options; ReadContextBuilder context_builder(path); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); context_builder.SetReadFieldNames({"key1", "k", "key_2", "c", "d", "a", "key0", "e"}); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2"); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + context_builder.SetPredicate(predicate); context_builder.EnablePredicateFilter(true); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); std::vector file_list_0; std::vector file_list_1; @@ -3521,17 +3624,20 @@ TEST_P(ReadInteTest, TestAppendReadWithSchemaEvolutionWithBuildInFieldId) { "data-11b12094-192f-4ad8-92a8-ae8cba5e25ef-0.parquet"}; } + std::map context_builder_options; ReadContextBuilder context_builder(path); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); context_builder.SetReadFieldNames({"key0", "key1", "k", "c", "d", "a", "e"}); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2"); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); DataSplitsSchema input_data_splits = { {path + "key0=0/key1=1/bucket-0", BinaryRowGenerator::GenerateRow({0, 1}, pool_.get()), @@ -3585,16 +3691,19 @@ TEST_P(ReadInteTest, TestAppendReadNestedType) { DataField(7, arrow::field("f6", arrow::decimal128(2, 2)))}; std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/append_complex_build_in_fieldid.db/append_complex_build_in_fieldid/"; + std::map context_builder_options; ReadContextBuilder context_builder(path); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2"); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); std::vector file_list; if (param.file_format == "orc") { @@ -3641,17 +3750,20 @@ TEST_P(ReadInteTest, TestAppendReadWithSchemaEvolutionWithCast) { std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/append_table_alter_table_with_cast.db/" "append_table_alter_table_with_cast/"; + std::map context_builder_options; ReadContextBuilder context_builder(path); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); context_builder.SetReadFieldNames({"f4", "key0", "key1", "f3", "f1", "f2", "f0", "f6"}); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2"); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); std::vector file_list_0; std::vector file_list_1; @@ -3727,15 +3839,15 @@ TEST_P(ReadInteTest, TestAppendReadWithSchemaEvolutionWithCastWithPredicatePushD ReadContextBuilder context_builder(path); context_builder.SetReadFieldNames({"f4", "key0", "key1", "f3", "f1", "f2", "f0", "f6"}); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2"); + const std::map read_options = { + {Options::FILE_FORMAT, param.file_format}, + {"read.batch-size", "2"}, + {"test.enable-adaptive-prefetch-strategy", param.enable_adaptive_prefetch_strategy}}; context_builder.SetPredicate(predicate); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context), read_options)); std::vector file_list_0; std::vector file_list_1; @@ -3806,15 +3918,16 @@ TEST_P(ReadInteTest, TestReadWithPKFallBackBranch) { ReadContextBuilder context_builder(path); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder.EnablePrefetch(param.enable_prefetch); + const std::map read_options = { + {Options::FILE_FORMAT, param.file_format}, + {"test.enable-adaptive-prefetch-strategy", param.enable_adaptive_prefetch_strategy}}; if (specific_table_schema) { context_builder.SetTableSchema(specific_table_schema.value()); } ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); ASSERT_OK_AND_ASSIGN(auto result_array, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -3864,7 +3977,7 @@ TEST_P(ReadInteTest, TestReadWithAppendFallBackBranch) { context_builder.EnablePrefetch(param.enable_prefetch); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context), {})); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); ASSERT_OK_AND_ASSIGN(auto result_array, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -3904,13 +4017,15 @@ TEST_P(ReadInteTest, TestFallBackBranchStreamRead) { std::vector read_fields = {DataField(0, arrow::field("dt", arrow::utf8())), DataField(1, arrow::field("name", arrow::utf8())), DataField(2, arrow::field("amount", arrow::int32()))}; + std::map context_builder_options; ReadContextBuilder context_builder(path); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_split)); ASSERT_OK_AND_ASSIGN(auto result_array, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -3950,17 +4065,19 @@ TEST_P(ReadInteTest, TestReadWithPKRtBranch) { DataField(2, arrow::field("amount", arrow::int32())), }; + std::map context_builder_options; ReadContextBuilder context_builder(path); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy) - .WithBranch("rt"); + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder_options[Options::BRANCH] = "rt"; + context_builder.EnablePrefetch(param.enable_prefetch); if (specific_table_schema) { context_builder.SetTableSchema(specific_table_schema.value()); } ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); ASSERT_OK_AND_ASSIGN(auto result_array, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -4007,17 +4124,19 @@ TEST_P(ReadInteTest, TestReadWithAppendPtBranch) { DataField(2, arrow::field("value2", arrow::int32())), }; + std::map context_builder_options; ReadContextBuilder context_builder(path); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy) - .WithBranch("test"); + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder_options[Options::BRANCH] = "test"; + context_builder.EnablePrefetch(param.enable_prefetch); if (specific_table_schema) { context_builder.SetTableSchema(specific_table_schema.value()); } ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); ASSERT_OK_AND_ASSIGN(auto result_array, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -4138,15 +4257,17 @@ TEST_P(ReadInteTest, TestSpecificFs) { auto countable_fs = std::make_shared(std::make_shared(), &io_count); + std::map context_builder_options; ReadContextBuilder context_builder(path); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", "false") - .AddOption("orc.read.enable-metrics", "true") - .WithFileSystem(countable_fs); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + + context_builder_options["test.enable-adaptive-prefetch-strategy"] = "false"; + context_builder_options["orc.read.enable-metrics"] = "true"; + context_builder.EnablePrefetch(param.enable_prefetch).WithFileSystem(countable_fs); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); std::vector file_list; if (param.file_format == "orc") { @@ -4207,20 +4328,28 @@ Result> ReadGlobalSystemTable( std::string sys_path = PathUtil::JoinPath(PathUtil::JoinPath(warehouse, "sys"), table_name); + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(sys_path); - scan_context_builder.SetOptions(options); + scan_context_builder_options = options; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_context_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured_table_11, + system_table->Copy(scan_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_scan, - system_table->NewScan(std::move(scan_context))); + configured_table_11->NewScan(std::move(scan_context))); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, table_scan->CreatePlan()); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(sys_path); - read_context_builder.SetOptions(options); + read_context_builder_options = options; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured_table_12, + system_table->Copy(read_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, - system_table->NewRead(std::move(read_context))); + configured_table_12->NewRead(std::move(read_context))); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr result, diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp index 7c4110257..7aad2410f 100644 --- a/test/inte/realtime_write_inte_test.cpp +++ b/test/inte/realtime_write_inte_test.cpp @@ -551,13 +551,15 @@ class RealtimeWriteInteTest : public ::testing::Test { Result> CreateRealtimeWriter( const std::shared_ptr& realtime_context, const std::string& temp_directory) const { + std::map builder_options; WriteContextBuilder builder(table_path_, commit_user_); - builder.SetOptions(options_).WithStreamingMode(true).WithRealtimeContext(realtime_context); + builder_options = options_; + builder.WithStreamingMode(true).WithRealtimeContext(realtime_context); if (!temp_directory.empty()) { builder.WithTempDirectory(temp_directory); } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); - return FileStoreWrite::Create(std::move(context)); + return FileStoreWrite::Create(std::move(context), builder_options); } Result> CreateRealtimeWriter() const { @@ -731,22 +733,28 @@ class RealtimeWriteInteTest : public ::testing::Test { Result Commit(const std::vector& realtime_commits, int64_t commit_identifier) const { + std::map builder_options; CommitContextBuilder builder(table_path_, commit_user_); + builder_options = options_; PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, - builder.SetOptions(options_).Finish()); + + builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit, - FileStoreCommit::Create(std::move(context))); + FileStoreCommit::Create(std::move(context), builder_options)); return commit->CommitWithProgress(realtime_commits, commit_identifier, /*watermark=*/std::nullopt); } Result DropPartition(const std::map& partition, int64_t commit_identifier) const { + std::map builder_options; CommitContextBuilder builder(table_path_, kDropPartitionCommitUser); + builder_options = options_; PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, - builder.SetOptions(options_).Finish()); + + builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit, - FileStoreCommit::Create(std::move(context))); + FileStoreCommit::Create(std::move(context), builder_options)); PAIMON_RETURN_NOT_OK(commit->DropPartition({partition}, commit_identifier)); PAIMON_ASSIGN_OR_RAISE(CoreOptions options, CoreOptions::FromMap(options_)); SnapshotManager snapshot_manager(options.GetFileSystem(), table_path_); @@ -759,11 +767,14 @@ class RealtimeWriteInteTest : public ::testing::Test { } Result TruncateTable(int64_t commit_identifier) const { + std::map builder_options; CommitContextBuilder builder(table_path_, kTruncateCommitUser); + builder_options = options_; PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, - builder.SetOptions(options_).Finish()); + + builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit, - FileStoreCommit::Create(std::move(context))); + FileStoreCommit::Create(std::move(context), builder_options)); PAIMON_RETURN_NOT_OK(commit->TruncateTable(commit_identifier)); PAIMON_ASSIGN_OR_RAISE(CoreOptions options, CoreOptions::FromMap(options_)); SnapshotManager snapshot_manager(options.GetFileSystem(), table_path_); @@ -776,11 +787,14 @@ class RealtimeWriteInteTest : public ::testing::Test { } Result RollbackToAsLatest(int64_t target_snapshot_id) const { + std::map builder_options; CommitContextBuilder builder(table_path_, kRollbackCommitUser); + builder_options = options_; PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, - builder.SetOptions(options_).Finish()); + + builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit, - FileStoreCommit::Create(std::move(context))); + FileStoreCommit::Create(std::move(context), builder_options)); PAIMON_ASSIGN_OR_RAISE(bool rolled_back, commit->RollbackToAsLatest(target_snapshot_id)); if (!rolled_back) { return Status::Invalid("failed to commit rollback snapshot"); @@ -798,12 +812,15 @@ class RealtimeWriteInteTest : public ::testing::Test { Result CompactAndCommit(const std::map& partition, int32_t bucket, int64_t commit_identifier, bool full_compaction = true) const { + std::map write_builder_options; WriteContextBuilder write_builder(table_path_, commit_user_); - write_builder.SetOptions(options_).WithStreamingMode(true).WithTempDirectory( + write_builder_options = options_; + write_builder.WithStreamingMode(true).WithTempDirectory( PathUtil::JoinPath(dir_->Str(), "compact-tmp")); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, write_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr compaction_writer, - FileStoreWrite::Create(std::move(write_context))); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr compaction_writer, + FileStoreWrite::Create(std::move(write_context), write_builder_options)); PAIMON_RETURN_NOT_OK(compaction_writer->Compact(partition, bucket, full_compaction)); PAIMON_ASSIGN_OR_RAISE( std::vector> compaction_messages, @@ -812,11 +829,15 @@ class RealtimeWriteInteTest : public ::testing::Test { return Status::Invalid("compaction did not produce a commit message"); } + std::map commit_builder_options; CommitContextBuilder commit_builder(table_path_, commit_user_); + commit_builder_options = options_; PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, - commit_builder.SetOptions(options_).Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr compaction_commit, - FileStoreCommit::Create(std::move(commit_context))); + + commit_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr compaction_commit, + FileStoreCommit::Create(std::move(commit_context), commit_builder_options)); PAIMON_RETURN_NOT_OK(compaction_commit->Commit(compaction_messages, commit_identifier)); PAIMON_RETURN_NOT_OK(compaction_writer->Close()); @@ -831,40 +852,47 @@ class RealtimeWriteInteTest : public ::testing::Test { } Result ExpireSnapshots() const { + std::map builder_options; CommitContextBuilder builder(table_path_, commit_user_); + builder_options = options_; PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, - builder.SetOptions(options_).Finish()); + + builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit, - FileStoreCommit::Create(std::move(context))); + FileStoreCommit::Create(std::move(context), builder_options)); return commit->Expire(); } Result> CreatePlan( const std::shared_ptr& realtime_context, const std::shared_ptr& predicate) const { + std::map scan_builder_options; ScanContextBuilder scan_builder(table_path_); if (realtime_context) { scan_builder.WithRealtimeContext(realtime_context); } scan_builder.SetPredicate(predicate).WithMemoryPool(pool_); + scan_builder_options = options_; PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, - scan_builder.SetOptions(options_).Finish()); + + scan_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, - TableScan::Create(std::move(scan_context))); + TableScan::Create(std::move(scan_context), scan_builder_options)); return scan->CreatePlan(); } Result> CreateQueryReader( const std::shared_ptr& plan, const std::shared_ptr& realtime_context) const { + std::map read_builder_options; ReadContextBuilder read_builder(table_path_); - read_builder.SetOptions(options_) - .SetReadFieldNames({"id", "payload", "pt"}) + read_builder_options = options_; + read_builder.SetReadFieldNames({"id", "payload", "pt"}) .WithRealtimeContext(realtime_context) .WithMemoryPool(pool_); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(std::move(read_context), read_builder_options)); return table_read->CreateReader(plan->Splits()); } @@ -879,16 +907,17 @@ class RealtimeWriteInteTest : public ::testing::Test { const std::shared_ptr& plan, const std::shared_ptr& realtime_context, const std::vector& read_fields, const std::shared_ptr& predicate, bool enable_predicate_filter) const { + std::map read_builder_options; ReadContextBuilder read_builder(table_path_); - read_builder.SetOptions(options_) - .SetReadFieldNames(read_fields) + read_builder_options = options_; + read_builder.SetReadFieldNames(read_fields) .SetPredicate(predicate) .EnablePredicateFilter(enable_predicate_filter) .WithRealtimeContext(realtime_context) .WithMemoryPool(pool_); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(std::move(read_context), read_builder_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, table_read->CreateReader(plan->Splits())); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr result, @@ -902,14 +931,15 @@ class RealtimeWriteInteTest : public ::testing::Test { const std::string& expected_json) const { std::unique_ptr c_read_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*read_schema, c_read_schema.get()).ok()); + std::map read_builder_options; ReadContextBuilder read_builder(table_path_); - read_builder.SetOptions(options_) - .SetReadSchema(std::move(c_read_schema)) + read_builder_options = options_; + read_builder.SetReadSchema(std::move(c_read_schema)) .WithRealtimeContext(realtime_context) .WithMemoryPool(pool_); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(std::move(read_context), read_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, @@ -988,13 +1018,13 @@ class RealtimeWriteInteTest : public ::testing::Test { Result CountRows(const std::shared_ptr& plan, const std::shared_ptr& realtime_context) const { + std::map read_builder_options; ReadContextBuilder read_builder(table_path_); - read_builder.SetOptions(options_) - .WithRealtimeContext(realtime_context) - .WithMemoryPool(pool_); + read_builder_options = options_; + read_builder.WithRealtimeContext(realtime_context).WithMemoryPool(pool_); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(std::move(read_context), read_builder_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr count_reader, table_read->CreateCountReader(plan->Splits())); return count_reader->CountRows(); @@ -1251,31 +1281,39 @@ TEST_F(RealtimeWriteInteTest, TestRealtimeOperationsRequireEnabledOption) { ASSERT_OK_AND_ASSIGN(std::shared_ptr realtime_context, RealtimeContext::Create()); + std::map write_builder_options; WriteContextBuilder write_builder(table_path_, commit_user_); - write_builder.SetOptions(disabled_options) - .WithStreamingMode(true) - .WithRealtimeContext(realtime_context); + write_builder_options = disabled_options; + write_builder.WithStreamingMode(true).WithRealtimeContext(realtime_context); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_context)), + ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_context), write_builder_options), "real-time write requires realtime.enabled=true"); + std::map scan_builder_options; ScanContextBuilder scan_builder(table_path_); - scan_builder.SetOptions(disabled_options).WithRealtimeContext(realtime_context); + scan_builder_options = disabled_options; + scan_builder.WithRealtimeContext(realtime_context); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context), scan_builder_options), "real-time scan requires realtime.enabled=true"); + std::map read_builder_options; ReadContextBuilder read_builder(table_path_); - read_builder.SetOptions(disabled_options).WithRealtimeContext(realtime_context); + read_builder_options = disabled_options; + read_builder.WithRealtimeContext(realtime_context); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), + ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context), read_builder_options), "real-time read requires realtime.enabled=true"); + std::map commit_builder_options; CommitContextBuilder commit_builder(table_path_, commit_user_); + commit_builder_options = disabled_options; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - commit_builder.SetOptions(disabled_options).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_context))); + + commit_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr commit, + FileStoreCommit::Create(std::move(commit_context), commit_builder_options)); ASSERT_NOK_WITH_MSG(commit->CommitWithProgress(/*realtime_commits=*/{}, /*commit_identifier=*/0, /*watermark=*/std::nullopt), @@ -1496,11 +1534,15 @@ TEST_F(RealtimeWriteInteTest, TestRealtimeDeduplicateCommitRetryMergesLatestDv) auto concurrent_message = std::make_shared( initial_message->Partition(), /*bucket=*/0, /*total_buckets=*/1, DataIncrement({}, {}, {}, {concurrent_dv}, {}), CompactIncrement({}, {}, {}, {}, {})); + std::map concurrent_builder_options; CommitContextBuilder concurrent_builder(table_path_, "concurrent-dv-writer"); + concurrent_builder_options = options_; ASSERT_OK_AND_ASSIGN(std::unique_ptr concurrent_context, - concurrent_builder.SetOptions(options_).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr concurrent_commit, - FileStoreCommit::Create(std::move(concurrent_context))); + + concurrent_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr concurrent_commit, + FileStoreCommit::Create(std::move(concurrent_context), concurrent_builder_options)); bool interleaved = false; auto fs = std::make_shared( PathUtil::JoinPath(table_path_, "snapshot/snapshot-2"), [&]() -> Status { @@ -1508,11 +1550,14 @@ TEST_F(RealtimeWriteInteTest, TestRealtimeDeduplicateCommitRetryMergesLatestDv) return concurrent_commit->Commit({concurrent_message}, /*commit_identifier=*/0, /*watermark=*/std::nullopt); }); + std::map builder_options; CommitContextBuilder builder(table_path_, commit_user_); + builder_options = options_; ASSERT_OK_AND_ASSIGN(std::unique_ptr context, - builder.SetOptions(options_).WithFileSystem(fs).Finish()); + + builder.WithFileSystem(fs).Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(context))); + FileStoreCommit::Create(std::move(context), builder_options)); ASSERT_OK_AND_ASSIGN( int64_t snapshot_id, commit->CommitWithProgress(progress, /*commit_identifier=*/1, /*watermark=*/std::nullopt)); @@ -1574,11 +1619,14 @@ TEST_F(RealtimeWriteInteTest, DISABLED_TestRealtimeDeduplicateCommitRetryRemapsC interleaved = true; return Status::OK(); }); + std::map builder_options; CommitContextBuilder builder(table_path_, commit_user_); + builder_options = options_; ASSERT_OK_AND_ASSIGN(std::unique_ptr context, - builder.SetOptions(options_).WithFileSystem(fs).Finish()); + + builder.WithFileSystem(fs).Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(context))); + FileStoreCommit::Create(std::move(context), builder_options)); ASSERT_OK_AND_ASSIGN( int64_t snapshot_id, commit->CommitWithProgress(progress, /*commit_identifier=*/2, /*watermark=*/std::nullopt)); @@ -2460,18 +2508,24 @@ TEST_F(RealtimeWriteInteTest, TestRealtimeCommitRetriesSnapshotConflict) { const std::string first_snapshot_path = PathUtil::JoinPath(table_path_, "snapshot/snapshot-1"); auto conflict_file_system = std::make_shared(first_snapshot_path); + std::map bucket0_builder_options; CommitContextBuilder bucket0_builder(table_path_, "bucket0_realtime_commit_user"); + bucket0_builder_options = options_; + ASSERT_OK_AND_ASSIGN(std::unique_ptr bucket0_context, + + bucket0_builder.WithFileSystem(conflict_file_system).Finish()); ASSERT_OK_AND_ASSIGN( - std::unique_ptr bucket0_context, - bucket0_builder.SetOptions(options_).WithFileSystem(conflict_file_system).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr bucket0_commit, - FileStoreCommit::Create(std::move(bucket0_context))); + std::unique_ptr bucket0_commit, + FileStoreCommit::Create(std::move(bucket0_context), bucket0_builder_options)); + std::map bucket1_builder_options; CommitContextBuilder bucket1_builder(table_path_, "bucket1_realtime_commit_user"); + bucket1_builder_options = options_; + ASSERT_OK_AND_ASSIGN(std::unique_ptr bucket1_context, + + bucket1_builder.WithFileSystem(conflict_file_system).Finish()); ASSERT_OK_AND_ASSIGN( - std::unique_ptr bucket1_context, - bucket1_builder.SetOptions(options_).WithFileSystem(conflict_file_system).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr bucket1_commit, - FileStoreCommit::Create(std::move(bucket1_context))); + std::unique_ptr bucket1_commit, + FileStoreCommit::Create(std::move(bucket1_context), bucket1_builder_options)); std::optional> bucket0_result; std::optional> bucket1_result; @@ -2539,13 +2593,16 @@ TEST_F(RealtimeWriteInteTest, TestRealtimeCommitRetriesConflictWithCompaction) { ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot_id)); } + std::map compact_write_builder_options; WriteContextBuilder compact_write_builder(table_path_, "compaction_write_user"); - compact_write_builder.SetOptions(options_).WithStreamingMode(true).WithTempDirectory( + compact_write_builder_options = options_; + compact_write_builder.WithStreamingMode(true).WithTempDirectory( PathUtil::JoinPath(dir_->Str(), "conflict-compact-tmp")); ASSERT_OK_AND_ASSIGN(std::unique_ptr compact_write_context, compact_write_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr compact_writer, - FileStoreWrite::Create(std::move(compact_write_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr compact_writer, + FileStoreWrite::Create(std::move(compact_write_context), compact_write_builder_options)); ASSERT_OK(compact_writer->Compact(/*partition=*/{}, /*bucket=*/0, /*full_compaction=*/true)); ASSERT_OK_AND_ASSIGN( @@ -2569,18 +2626,24 @@ TEST_F(RealtimeWriteInteTest, TestRealtimeCommitRetriesConflictWithCompaction) { PathUtil::JoinPath(table_path_, "snapshot/snapshot-3"); auto conflict_file_system = std::make_shared(conflict_snapshot_path); + std::map append_commit_builder_options; CommitContextBuilder append_commit_builder(table_path_, commit_user_); + append_commit_builder_options = options_; + ASSERT_OK_AND_ASSIGN(std::unique_ptr append_commit_context, + + append_commit_builder.WithFileSystem(conflict_file_system).Finish()); ASSERT_OK_AND_ASSIGN( - std::unique_ptr append_commit_context, - append_commit_builder.SetOptions(options_).WithFileSystem(conflict_file_system).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr append_commit, - FileStoreCommit::Create(std::move(append_commit_context))); + std::unique_ptr append_commit, + FileStoreCommit::Create(std::move(append_commit_context), append_commit_builder_options)); + std::map compact_commit_builder_options; CommitContextBuilder compact_commit_builder(table_path_, "compaction_commit_user"); + compact_commit_builder_options = options_; + ASSERT_OK_AND_ASSIGN(std::unique_ptr compact_commit_context, + + compact_commit_builder.WithFileSystem(conflict_file_system).Finish()); ASSERT_OK_AND_ASSIGN( - std::unique_ptr compact_commit_context, - compact_commit_builder.SetOptions(options_).WithFileSystem(conflict_file_system).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr compact_commit, - FileStoreCommit::Create(std::move(compact_commit_context))); + std::unique_ptr compact_commit, + FileStoreCommit::Create(std::move(compact_commit_context), compact_commit_builder_options)); std::optional> append_result; std::optional compact_status; @@ -2844,14 +2907,15 @@ TEST_F(RealtimeWriteInteTest, TestPkRead) { ASSERT_OK_AND_ASSIGN(std::shared_ptr lifetime_plan, CreatePlan(realtime_context, /*predicate=*/nullptr)); + std::map read_builder_options; ReadContextBuilder read_builder(table_path_); - read_builder.SetOptions(options_) - .SetReadFieldNames({"id", "payload", "pt"}) + read_builder_options = options_; + read_builder.SetReadFieldNames({"id", "payload", "pt"}) .WithRealtimeContext(realtime_context) .WithMemoryPool(pool_); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(std::move(read_context), read_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, table_read->CreateReader(lifetime_plan->Splits())); ASSERT_FALSE(query_view->expired()); @@ -2923,10 +2987,14 @@ TEST_F(RealtimeWriteInteTest, TestPkRealtimeReadOptimizedScanUnsupported) { ASSERT_OK_AND_ASSIGN(std::shared_ptr table, SystemTableLoader::LoadFromPath(core_options.GetFileSystem(), table_path_ + "$ro", options_)); + std::map scan_builder_options; ScanContextBuilder scan_builder(table_path_ + "$ro"); - scan_builder.SetOptions(options_).WithRealtimeContext(realtime_context).WithMemoryPool(pool_); + scan_builder_options = options_; + scan_builder.WithRealtimeContext(realtime_context).WithMemoryPool(pool_); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); - Result> scan = table->NewScan(std::move(scan_context)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_1, + table->Copy(scan_builder_options)); + Result> scan = configured_table_1->NewScan(std::move(scan_context)); ASSERT_TRUE(scan.status().IsNotImplemented()) << scan.status().ToString(); ASSERT_NE(std::string::npos, scan.status().ToString().find( "PK real-time union read does not support read-optimized")); @@ -2956,14 +3024,15 @@ TEST_F(RealtimeWriteInteTest, TestPkDeleteInsertAndPinnedReadsAcrossRefresh) { CreatePlan(realtime_context, /*predicate=*/nullptr)); ASSERT_OK_AND_ASSIGN(std::shared_ptr reader_plan, CreatePlan(realtime_context, /*predicate=*/nullptr)); + std::map read_builder_options; ReadContextBuilder read_builder(table_path_); - read_builder.SetOptions(options_) - .SetReadFieldNames({"id", "payload", "pt"}) + read_builder_options = options_; + read_builder.SetReadFieldNames({"id", "payload", "pt"}) .WithRealtimeContext(realtime_context) .WithMemoryPool(pool_); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(std::move(read_context), read_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr pinned_reader, table_read->CreateReader(reader_plan->Splits())); @@ -3154,14 +3223,15 @@ TEST_F(RealtimeWriteInteTest, TestPkNestedProjectionAcrossDiskAndMemory) { }); auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); + std::map read_builder_options; ReadContextBuilder read_builder(table_path_); - read_builder.SetOptions(options_) - .SetReadSchema(std::move(c_schema)) + read_builder_options = options_; + read_builder.SetReadSchema(std::move(c_schema)) .WithRealtimeContext(realtime_context) .WithMemoryPool(pool_); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(std::move(read_context), read_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, @@ -3449,22 +3519,28 @@ TEST_F(RealtimeWriteInteTest, TestPkPartitionBucketRecovery) { TEST_F(RealtimeWriteInteTest, TestPkRecovery) { CreatePkTable(); + std::map seed_builder_options; WriteContextBuilder seed_builder(table_path_, commit_user_); - seed_builder.SetOptions(options_).WithStreamingMode(true); + seed_builder_options = options_; + seed_builder.WithStreamingMode(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr seed_context, seed_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr seed_writer, - FileStoreWrite::Create(std::move(seed_context))); + FileStoreWrite::Create(std::move(seed_context), seed_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr seed_batch, MakeLogicalBatch({Row{99, "seed", "p0"}}, /*bucket=*/0)); ASSERT_OK(seed_writer->Write(std::move(seed_batch))); ASSERT_OK_AND_ASSIGN(std::vector> seed_messages, seed_writer->PrepareCommit(/*wait_compaction=*/false, /*commit_identifier=*/0)); + std::map seed_commit_builder_options; CommitContextBuilder seed_commit_builder(table_path_, commit_user_); + seed_commit_builder_options = options_; ASSERT_OK_AND_ASSIGN(std::unique_ptr seed_commit_context, - seed_commit_builder.SetOptions(options_).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr seed_commit, - FileStoreCommit::Create(std::move(seed_commit_context))); + + seed_commit_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr seed_commit, + FileStoreCommit::Create(std::move(seed_commit_context), seed_commit_builder_options)); ASSERT_OK(seed_commit->Commit(seed_messages, /*commit_identifier=*/0)); ASSERT_OK(seed_writer->Close()); const std::vector mutations = { @@ -3551,11 +3627,14 @@ TEST_F(RealtimeWriteInteTest, TestPkCompaction) { ASSERT_OK_AND_ASSIGN(uint64_t memory_usage, GetRealtimeMemoryUsage(realtime_context)); ASSERT_EQ(0, memory_usage); } + std::map compact_builder_options; WriteContextBuilder compact_builder(table_path_, commit_user_); - compact_builder.SetOptions(options_).WithStreamingMode(true); + compact_builder_options = options_; + compact_builder.WithStreamingMode(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr compact_context, compact_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr compact_writer, - FileStoreWrite::Create(std::move(compact_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr compact_writer, + FileStoreWrite::Create(std::move(compact_context), compact_builder_options)); ASSERT_OK(compact_writer->Compact(/*partition=*/{}, /*bucket=*/0, /*full_compaction=*/true)); ASSERT_OK_AND_ASSIGN( @@ -3583,11 +3662,15 @@ TEST_F(RealtimeWriteInteTest, TestPkCompaction) { std::max(compacted_live_max_sequence_number, file->max_sequence_number); } ASSERT_LT(compacted_live_max_sequence_number, kHistoricalMaxSequenceNumber); + std::map commit_builder_options; CommitContextBuilder commit_builder(table_path_, commit_user_); + commit_builder_options = options_; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - commit_builder.SetOptions(options_).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_context))); + + commit_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr commit, + FileStoreCommit::Create(std::move(commit_context), commit_builder_options)); ASSERT_OK(commit->Commit(compact_messages, /*commit_identifier=*/4)); ASSERT_OK(compact_writer->Close()); @@ -3907,11 +3990,14 @@ TEST_F(RealtimeWriteInteTest, TestCommitWithProgressRejectsCoveredRangesFromAnot writer->PrepareCommitWithProgress(/*commit_identifier=*/0)); ASSERT_OK(Commit(commits, /*commit_identifier=*/0)); + std::map builder_options; CommitContextBuilder builder(table_path_, "another_realtime_commit_user"); + builder_options = options_; ASSERT_OK_AND_ASSIGN(std::unique_ptr context, - builder.SetOptions(options_).Finish()); + + builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(context))); + FileStoreCommit::Create(std::move(context), builder_options)); ASSERT_NOK_WITH_MSG(commit->CommitWithProgress(commits, /*commit_identifier=*/0, /*watermark=*/std::nullopt), "another commit user or identifier"); @@ -4727,14 +4813,15 @@ TEST_F(RealtimeWriteInteTest, TestSuccessfulReaderCreationConsumesRealtimeSplitT ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, CreatePlan(realtime_context, /*predicate=*/nullptr)); + std::map read_builder_options; ReadContextBuilder read_builder(table_path_); - read_builder.SetOptions(options_) - .SetReadFieldNames({"id", "payload", "pt"}) + read_builder_options = options_; + read_builder.SetReadFieldNames({"id", "payload", "pt"}) .WithRealtimeContext(realtime_context) .WithMemoryPool(pool_); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(std::move(read_context), read_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr first_reader, table_read->CreateReader(plan->Splits())); first_reader->Close(); @@ -4759,16 +4846,17 @@ TEST_F(RealtimeWriteInteTest, TestFailedReaderCreationPreservesRealtimeSplitTick CreatePlan(realtime_context, /*predicate=*/nullptr)); ASSERT_EQ(1, plan->Splits().size()); + std::map read_builder_options; ReadContextBuilder read_builder(table_path_); - read_builder.SetOptions(options_) - .SetReadFieldNames({"id", "payload", "pt"}) + read_builder_options = options_; + read_builder.SetReadFieldNames({"id", "payload", "pt"}) .SetPredicate(std::make_shared()) .EnablePredicateFilter(true) .WithRealtimeContext(realtime_context) .WithMemoryPool(pool_); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(std::move(read_context), read_builder_options)); ASSERT_NOK_WITH_MSG(table_read->CreateReader(plan->Splits()), "cannot cast predicate unsupported"); @@ -5622,14 +5710,15 @@ TEST_F(RealtimeWriteInteTest, TestReaderPinsMemoryAcrossRefresh) { std::dynamic_pointer_cast(plan->Splits()[0]); ASSERT_NE(nullptr, realtime_split); + std::map read_builder_options; ReadContextBuilder read_builder(table_path_); - read_builder.SetOptions(options_) - .SetReadFieldNames({"id", "payload", "pt"}) + read_builder_options = options_; + read_builder.SetReadFieldNames({"id", "payload", "pt"}) .WithRealtimeContext(realtime_context) .WithMemoryPool(pool_); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(std::move(read_context), read_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr realtime_context_impl, diff --git a/test/inte/scan_and_read_inte_test.cpp b/test/inte/scan_and_read_inte_test.cpp index b7cc73b96..2e23606e3 100644 --- a/test/inte/scan_and_read_inte_test.cpp +++ b/test/inte/scan_and_read_inte_test.cpp @@ -144,21 +144,23 @@ class ScanAndReadInteTest : public testing::Test, } } - void AddReadOptionsForPrefetch(ReadContextBuilder* read_context_builder) { - read_context_builder->AddOption("test.enable-adaptive-prefetch-strategy", "false"); + void AddReadOptionsForPrefetch(ReadContextBuilder* read_context_builder, + std::map& options) { + options["test.enable-adaptive-prefetch-strategy"] = "false"; if (EnablePrefetch()) { read_context_builder->EnablePrefetch(true).SetPrefetchBatchCount(3); } } - Result> FinishScanContext(ScanContextBuilder& builder) { + Result> FinishScanContext( + ScanContextBuilder& builder, std::map& options) { if (EnableSnapshotLiveManifestCache()) { if (!snapshot_live_manifest_cache_) { snapshot_live_manifest_cache_ = std::make_shared(/*max_weight=*/64 * 1024 * 1024); } - builder.AddOption(Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, "3") - .WithCache(snapshot_live_manifest_cache_); + options[Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS] = "3"; + builder.WithCache(snapshot_live_manifest_cache_); } return builder.Finish(); } @@ -240,12 +242,15 @@ TEST_P(ScanAndReadInteTest, TestWithAppendSnapshotIOException) { ScopeGuard guard([&io_hook]() { io_hook->Clear(); }); io_hook->Reset(i, IOHook::Mode::RETURN_ERROR); // scan + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - Result> scan_context = FinishScanContext(scan_context_builder); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + + Result> scan_context = + FinishScanContext(scan_context_builder, scan_context_builder_options); CHECK_HOOK_STATUS(scan_context.status(), i); Result> table_scan = - TableScan::Create(std::move(scan_context).value()); + TableScan::Create(std::move(scan_context).value(), scan_context_builder_options); CHECK_HOOK_STATUS(table_scan.status(), i); Result> result_plan = table_scan.value()->CreatePlan(); CHECK_HOOK_STATUS(result_plan.status(), i); @@ -254,12 +259,14 @@ TEST_P(ScanAndReadInteTest, TestWithAppendSnapshotIOException) { auto splits = result_plan.value()->Splits(); ASSERT_EQ(3, splits.size()); // read + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - Result> table_read = TableRead::Create(std::move(read_context)); + Result> table_read = + TableRead::Create(std::move(read_context), read_context_builder_options); CHECK_HOOK_STATUS(table_read.status(), i); Result> batch_reader = table_read.value()->CreateReader(splits); @@ -297,12 +304,15 @@ TEST_P(ScanAndReadInteTest, TestWithPkSnapshotIOException) { ScopeGuard guard([&io_hook]() { io_hook->Clear(); }); io_hook->Reset(i, IOHook::Mode::RETURN_ERROR); // scan + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "6"); - Result> scan_context = FinishScanContext(scan_context_builder); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "6"; + + Result> scan_context = + FinishScanContext(scan_context_builder, scan_context_builder_options); CHECK_HOOK_STATUS(scan_context.status(), i); Result> table_scan = - TableScan::Create(std::move(scan_context).value()); + TableScan::Create(std::move(scan_context).value(), scan_context_builder_options); CHECK_HOOK_STATUS(table_scan.status(), i); Result> result_plan = table_scan.value()->CreatePlan(); CHECK_HOOK_STATUS(result_plan.status(), i); @@ -310,12 +320,14 @@ TEST_P(ScanAndReadInteTest, TestWithPkSnapshotIOException) { auto splits = result_plan.value()->Splits(); ASSERT_EQ(3, splits.size()); // read + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - Result> table_read = TableRead::Create(std::move(read_context)); + Result> table_read = + TableRead::Create(std::move(read_context), read_context_builder_options); CHECK_HOOK_STATUS(table_read.status(), i); Result> batch_reader = table_read.value()->CreateReader(splits); @@ -350,21 +362,27 @@ TEST_P(ScanAndReadInteTest, TestWithAppendSnapshot1) { std::string table_path = GetDataDir() + "/" + file_format + "/append_09.db/append_09"; // scan + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); auto splits = result_plan->Splits(); ASSERT_EQ(3, splits.size()); // read + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -392,10 +410,14 @@ TEST_P(ScanAndReadInteTest, TestWithAppendSnapshot3) { std::string table_path = GetDataDir() + "/" + file_format + "/append_09.db/append_09"; // scan + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "3"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "3"; + + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 3); @@ -403,10 +425,12 @@ TEST_P(ScanAndReadInteTest, TestWithAppendSnapshot3) { ASSERT_EQ(3, splits.size()); // read + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -505,21 +529,20 @@ TEST(SelectiveManifestDecodeInteTest, TestPartitionedPointLookup) { bool entry_cache) -> Result { std::shared_ptr pool = GetMemoryPool(); scan_pools.push_back(pool); + std::map builder_options; ScanContextBuilder builder(table_path); - builder.WithMemoryPool(pool) - .WithExecutor(executor) - .WithCache(cache) - .AddOption(Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, entry_cache ? "3" : "0") - .AddOption(Options::SCAN_MANIFEST_ENTRY_LAZY_DECODE_ENABLED, - lazy_decode ? "true" : "false") - .AddOption(Options::READ_BATCH_SIZE, "128"); + builder_options[Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS] = entry_cache ? "3" : "0"; + builder_options[Options::SCAN_MANIFEST_ENTRY_LAZY_DECODE_ENABLED] = + lazy_decode ? "true" : "false"; + builder_options[Options::READ_BATCH_SIZE] = "128"; + builder.WithMemoryPool(pool).WithExecutor(executor).WithCache(cache); if (entry_cache) { // No explicit bucket or partition filter: the predicate must select the bucket. builder.SetPredicate(predicate); } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_scan, - TableScan::Create(std::move(context))); + TableScan::Create(std::move(context), builder_options)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, table_scan->CreatePlan()); PAIMON_ASSIGN_OR_RAISE(uint64_t enabled, table_scan->GetMetrics()->GetCounter( ScanMetrics::LAST_SNAPSHOT_CACHE_ENABLED)); @@ -541,7 +564,7 @@ TEST(SelectiveManifestDecodeInteTest, TestPartitionedPointLookup) { builder.SetPredicate(predicate).EnablePredicateFilter(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr read, - TableRead::Create(std::move(context))); + TableRead::Create(std::move(context), {})); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, read->CreateReader(result.plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr rows, @@ -690,15 +713,15 @@ TEST(SelectiveManifestDecodeInteTest, TestSchemaEvolutionPreservesBucketPruning) } auto cache = std::make_shared(capacities); for (int32_t attempt = 0; attempt < 2; ++attempt) { + std::map builder_options; ScanContextBuilder builder(table_path); - builder.SetPredicate(predicate) - .WithCache(cache) - .AddOption(Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, "3") - .AddOption(Options::SCAN_MANIFEST_ENTRY_LAZY_DECODE_ENABLED, - lazy_decode ? "true" : "false"); + builder_options[Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS] = "3"; + builder_options[Options::SCAN_MANIFEST_ENTRY_LAZY_DECODE_ENABLED] = + lazy_decode ? "true" : "false"; + builder.SetPredicate(predicate).WithCache(cache); ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, - TableScan::Create(std::move(context))); + TableScan::Create(std::move(context), builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); ASSERT_EQ(plan->Splits().size(), 1); auto split = std::dynamic_pointer_cast(plan->Splits()[0]); @@ -712,7 +735,7 @@ TEST(SelectiveManifestDecodeInteTest, TestSchemaEvolutionPreservesBucketPruning) ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr read, - TableRead::Create(std::move(read_context))); + TableRead::Create(std::move(read_context), {})); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr rows, @@ -779,10 +802,11 @@ TEST_P(ScanAndReadInteTest, TestWithAppendBucketKeyPointLookup) { // All buckets have overlapping stats for this key. An explicit filter bypasses // inference, proving that ordinary statistics cannot account for the pruning. for (int32_t bucket = 0; bucket < kNumBuckets; ++bucket) { + std::map builder_options; ScanContextBuilder builder(table_path); builder.SetPredicate(predicate).SetBucketFilter(bucket); - ASSERT_OK_AND_ASSIGN(auto context, FinishScanContext(builder)); - ASSERT_OK_AND_ASSIGN(auto scan, TableScan::Create(std::move(context))); + ASSERT_OK_AND_ASSIGN(auto context, FinishScanContext(builder, builder_options)); + ASSERT_OK_AND_ASSIGN(auto scan, TableScan::Create(std::move(context), builder_options)); ASSERT_OK_AND_ASSIGN(auto plan, scan->CreatePlan()); ASSERT_FALSE(plan->Splits().empty()); @@ -804,23 +828,29 @@ TEST_P(ScanAndReadInteTest, TestWithAppendBucketKeyPointLookup) { auto lookup_predicate = PredicateBuilder::Equal( 0, "rowkey", field_type, Literal(field_type, lookup_key.data(), lookup_key.size())); + std::map scan_builder_options; ScanContextBuilder scan_builder(table_path); - scan_builder.SetPredicate(lookup_predicate) - .AddOption(Options::BUCKET, current_bucket_count); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_builder)); - ASSERT_OK_AND_ASSIGN(auto scan, TableScan::Create(std::move(scan_context))); + scan_builder_options[Options::BUCKET] = current_bucket_count; + scan_builder.SetPredicate(lookup_predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_builder, scan_builder_options)); + ASSERT_OK_AND_ASSIGN( + auto scan, TableScan::Create(std::move(scan_context), scan_builder_options)); ASSERT_OK_AND_ASSIGN(auto plan, scan->CreatePlan()); if (EnableSnapshotLiveManifestCache()) { ASSERT_OK_AND_ASSIGN( uint64_t enabled, scan->GetMetrics()->GetCounter(ScanMetrics::LAST_SNAPSHOT_CACHE_ENABLED)); ASSERT_EQ(enabled, 1); + std::map cached_builder_options; ScanContextBuilder cached_builder(table_path); - cached_builder.SetPredicate(lookup_predicate) - .AddOption(Options::BUCKET, current_bucket_count); - ASSERT_OK_AND_ASSIGN(auto cached_context, FinishScanContext(cached_builder)); - ASSERT_OK_AND_ASSIGN(auto cached_scan, - TableScan::Create(std::move(cached_context))); + cached_builder_options[Options::BUCKET] = current_bucket_count; + cached_builder.SetPredicate(lookup_predicate); + ASSERT_OK_AND_ASSIGN(auto cached_context, + FinishScanContext(cached_builder, cached_builder_options)); + ASSERT_OK_AND_ASSIGN( + auto cached_scan, + TableScan::Create(std::move(cached_context), cached_builder_options)); ASSERT_OK_AND_ASSIGN(plan, cached_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(uint64_t hit, cached_scan->GetMetrics()->GetCounter( ScanMetrics::LAST_SNAPSHOT_CACHE_HIT)); @@ -833,11 +863,13 @@ TEST_P(ScanAndReadInteTest, TestWithAppendBucketKeyPointLookup) { ASSERT_TRUE(data_split); ASSERT_EQ(data_split->Bucket(), bucket_ids[lookup_index]); } + std::map read_builder_options; ReadContextBuilder read_builder(table_path); - AddReadOptionsForPrefetch(&read_builder); + AddReadOptionsForPrefetch(&read_builder, read_builder_options); read_builder.SetPredicate(lookup_predicate).EnablePredicateFilter(true); ASSERT_OK_AND_ASSIGN(auto read_context, read_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN( + auto read, TableRead::Create(std::move(read_context), read_builder_options)); ASSERT_OK_AND_ASSIGN(auto reader, read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(auto result, ReadResultCollector::CollectResult(std::move(reader))); @@ -861,10 +893,14 @@ TEST_P(ScanAndReadInteTest, TestWithAppendSnapshot5) { std::string table_path = GetDataDir() + "/" + file_format + "/append_09.db/append_09"; // scan + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "5"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "5"; + + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 5); @@ -872,10 +908,12 @@ TEST_P(ScanAndReadInteTest, TestWithAppendSnapshot5) { ASSERT_EQ(3, splits.size()); // read + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -911,18 +949,24 @@ TEST_P(ScanAndReadInteTest, TestWithAppendSnapshot5WithReaderBuildParallelism) { auto file_format = FileFormat(); std::string table_path = GetDataDir() + "/" + file_format + "/append_09.db/append_09"; + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "5"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "5"; + + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); auto splits = result_plan->Splits(); ASSERT_EQ(3, splits.size()); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -953,15 +997,21 @@ TEST_P(ScanAndReadInteTest, TestWithAppendSnapshotWithStreamWithDefaultMode) { std::string table_path = GetDataDir() + "/" + file_format + "/append_09.db/append_09"; // scan + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1").WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + scan_context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); + + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); std::vector> expected_snapshot_ids = {std::nullopt, 1, 2, 3, 4}; auto expected_snapshot1 = std::make_shared( @@ -1002,10 +1052,14 @@ TEST_P(ScanAndReadInteTest, TestJavaPaimon1WithAppendSnapshot1) { std::string table_path = GetDataDir() + "/" + file_format + "/append_10.db/append_10"; // scan + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -1013,10 +1067,12 @@ TEST_P(ScanAndReadInteTest, TestJavaPaimon1WithAppendSnapshot1) { ASSERT_EQ(3, splits.size()); // read + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -1041,10 +1097,14 @@ TEST_P(ScanAndReadInteTest, TestJavaPaimon1WithAppendSnapshotOfNestedType) { "/append_complex_build_in_fieldid.db/" "append_complex_build_in_fieldid/"; // scan + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -1069,10 +1129,12 @@ TEST_P(ScanAndReadInteTest, TestJavaPaimon1WithAppendSnapshotOfNestedType) { std::shared_ptr arrow_data_type = DataField::ConvertDataFieldsToArrowStructType(read_fields); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -1096,14 +1158,18 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot6) { "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; // normal batch scan case for pk+dv, all data in level 0 is filtered out + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "6"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "6"; + + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); ReadContextBuilder read_context_builder(table_path); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context), {})); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 6); @@ -1137,15 +1203,21 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot1) { auto file_format = FileFormat(); std::string table_path = GetDataDir() + "/" + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -1160,16 +1232,22 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot6WithPartitionAndBu std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "6"); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "6"; + scan_context_builder.SetBucketFilter(1).SetPartitionFilter({{{"f1", "10"}}}); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 6); @@ -1196,8 +1274,9 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot6WithPredicate) { "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; // predicate: f0 != "Alice" (key predicate) and f3 > 18 (value predicate) and all data in level // 0 + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "6"); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "6"; std::string literal_str = "Alice"; auto not_equal = PredicateBuilder::NotEqual( @@ -1207,14 +1286,18 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot6WithPredicate) { FieldType::DOUBLE, Literal(18.0)); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({not_equal, greater_than})); scan_context_builder.SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); read_context_builder.SetPredicate(predicate).EnableLateMaterializing(false); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 6); @@ -1237,8 +1320,9 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot6WithLateMaterializ auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "6"); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "6"; std::string literal_str = "Alice"; auto not_equal = PredicateBuilder::NotEqual( @@ -1248,14 +1332,18 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot6WithLateMaterializ FieldType::DOUBLE, Literal(18.0)); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({not_equal, greater_than})); scan_context_builder.SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); read_context_builder.SetPredicate(predicate).EnableLateMaterializing(true); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 6); @@ -1278,15 +1366,19 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot4WithPredicate) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "4"); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "4"; + scan_context_builder.SetBucketFilter(0).SetPartitionFilter({{{"f1", "10"}}}); // predicate f3 > 20.0 will be applied, as in kv mode value filter is enabled auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/3, /*field_name=*/"f3", FieldType::DOUBLE, Literal(20.0)); scan_context_builder.SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 4); @@ -1300,15 +1392,20 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot6WithLimit) { // normal batch scan case for pk+dv, all data in level 0 is filtered out, set row limits to 6, // data in partition 20 is truncated + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); scan_context_builder.SetLimit(6); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 6); @@ -1336,17 +1433,22 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvStreamFromSnapshot4) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "4") - .AddOption(Options::SCAN_MODE, "from-snapshot-full") - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "4"; + scan_context_builder_options[Options::SCAN_MODE] = "from-snapshot-full"; + scan_context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); + + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); // from an compact snapshot std::vector> expected_snapshot_ids = {4, 5, 6}; @@ -1388,17 +1490,22 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvStreamFromSnapshot5) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "5") - .AddOption(Options::SCAN_MODE, "from-snapshot-full") - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "5"; + scan_context_builder_options[Options::SCAN_MODE] = "from-snapshot-full"; + scan_context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); + + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); // from an append snapshot, first plan is snapshot5 with merge, second plan is snapshot6 // (isStreaming=true) @@ -1434,15 +1541,21 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvStreamFromSnapshot6) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_MODE, "latest-full").WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - + scan_context_builder_options[Options::SCAN_MODE] = "latest-full"; + scan_context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); + + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); // from an append snapshot, first plan is snapshot6 with merge, level 0 data in base manifest // list is contained @@ -1471,17 +1584,22 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvStreamFromSnapshot1) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1") - .AddOption(Options::SCAN_MODE, "from-snapshot-full") - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + scan_context_builder_options[Options::SCAN_MODE] = "from-snapshot-full"; + scan_context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); + + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); // from the first snapshot std::vector> expected_snapshot_ids = {1, 3, 5, 6}; @@ -1530,17 +1648,22 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvStreamFromSnapshot2) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "2") - .AddOption(Options::SCAN_MODE, "from-snapshot") - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "2"; + scan_context_builder_options[Options::SCAN_MODE] = "from-snapshot"; + scan_context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); + + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); // use from-snapshot mode, first plan is empty, second plan is snapshot 3 with delta files // (snapshot 2 is compact snapshot, so skipped) @@ -1575,18 +1698,22 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithNestedType) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_nested_type.db/pk_table_nested_type/"; + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN( auto read_context, read_context_builder.SetReadFieldNames({"shopId", "dt", "hr", "col0", "col1", "col2"}) .Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context), {})); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 2); @@ -1629,14 +1756,19 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanLatestSnapshot) { "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; // normal batch scan case for pk+mor, use latest snapshot if not specified + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 5); @@ -1676,15 +1808,21 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot2) { // normal batch scan case for pk+mor, read snapshot 2 (append snapshot), all data is in level 0 // with merge read + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "2"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "2"; + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); + + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 2); @@ -1720,16 +1858,22 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot5WithPartitionAndB "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; // all data not in partition 10, bucket 1 is filtered out + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "5"); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "5"; + scan_context_builder.SetBucketFilter(1).SetPartitionFilter({{{"f1", "10"}}}); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 5); @@ -1758,8 +1902,9 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot5WithPredicate) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "5"); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "5"; // predicate: f0 != "Alice" and f0 < "Lucy" and f3 <= 30.0 // as f3 <= 30.0 is a value filter, it does not take effect @@ -1780,14 +1925,18 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot5WithPredicate) { PredicateBuilder::And({not_equal, less_than, less_or_equal})); scan_context_builder.SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); read_context_builder.SetPredicate(predicate).EnableLateMaterializing(false); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 5); @@ -1821,8 +1970,9 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot5WithLateMateriali std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "5"); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "5"; std::string literal_str = "Alice"; auto not_equal = PredicateBuilder::NotEqual( @@ -1837,14 +1987,18 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot5WithLateMateriali ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({not_equal, less_than, less_or_equal})); scan_context_builder.SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); read_context_builder.SetPredicate(predicate).EnableLateMaterializing(true); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 5); @@ -1869,16 +2023,20 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot3WithPredicate) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "3"); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "3"; + scan_context_builder.SetBucketFilter(1).SetPartitionFilter({{{"f1", "10"}}}); // predicate f3 > 20.0 will be applied as all data files in bucket 1 in compact snapshot 3 are // all filtered auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/3, /*field_name=*/"f3", FieldType::DOUBLE, Literal(20.0)); scan_context_builder.SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 3); @@ -1890,20 +2048,27 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvWithInvalidAggregateBatchScanSnapsho std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "3") - .AddOption(Options::MERGE_ENGINE, "aggregation") - .AddOption("fields.f3.aggregate-function", "rbm32"); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "3"; + scan_context_builder_options[Options::MERGE_ENGINE] = "aggregation"; + scan_context_builder_options["fields.f3.aggregate-function"] = "rbm32"; + scan_context_builder.SetBucketFilter(1).SetPartitionFilter({{{"f1", "10"}}}); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); - read_context_builder.AddOption(Options::MERGE_ENGINE, "aggregation") - .AddOption("fields.f3.aggregate-function", "rbm32"); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); + read_context_builder_options[Options::MERGE_ENGINE] = "aggregation"; + read_context_builder_options["fields.f3.aggregate-function"] = "rbm32"; + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 3); @@ -1930,20 +2095,27 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorWithInvalidAggregateBatchScanSnapsh std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "3") - .AddOption(Options::MERGE_ENGINE, "aggregation") - .AddOption("fields.f3.aggregate-function", "rbm32"); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "3"; + scan_context_builder_options[Options::MERGE_ENGINE] = "aggregation"; + scan_context_builder_options["fields.f3.aggregate-function"] = "rbm32"; + scan_context_builder.SetBucketFilter(1).SetPartitionFilter({{{"f1", "10"}}}); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); - read_context_builder.AddOption(Options::MERGE_ENGINE, "aggregation") - .AddOption("fields.f3.aggregate-function", "rbm32"); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); + read_context_builder_options[Options::MERGE_ENGINE] = "aggregation"; + read_context_builder_options["fields.f3.aggregate-function"] = "rbm32"; + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 3); @@ -1957,24 +2129,31 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithAggregateBatchScanSnapshot3WithPredica std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "3") - .AddOption(Options::MERGE_ENGINE, "aggregation") - .AddOption("fields.f3.aggregate-function", "sum"); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "3"; + scan_context_builder_options[Options::MERGE_ENGINE] = "aggregation"; + scan_context_builder_options["fields.f3.aggregate-function"] = "sum"; + scan_context_builder.SetBucketFilter(1).SetPartitionFilter({{{"f1", "10"}}}); // predicate f3 > 20.0 will not be applied, as merge engine is aggregation auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/3, /*field_name=*/"f3", FieldType::DOUBLE, Literal(20.0)); scan_context_builder.SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); - read_context_builder.AddOption(Options::MERGE_ENGINE, "aggregation") - .AddOption("fields.f3.aggregate-function", "sum"); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); + read_context_builder_options[Options::MERGE_ENGINE] = "aggregation"; + read_context_builder_options["fields.f3.aggregate-function"] = "sum"; + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 3); @@ -2001,24 +2180,31 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithPartialUpdateBatchScanSnapshot3WithPre std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "3") - .AddOption(Options::MERGE_ENGINE, "partial-update") - .AddOption(Options::IGNORE_DELETE, "true"); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "3"; + scan_context_builder_options[Options::MERGE_ENGINE] = "partial-update"; + scan_context_builder_options[Options::IGNORE_DELETE] = "true"; + scan_context_builder.SetBucketFilter(1).SetPartitionFilter({{{"f1", "10"}}}); // predicate f3 > 20.0 will not be applied, as merge engine is partial-update auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/3, /*field_name=*/"f3", FieldType::DOUBLE, Literal(20.0)); scan_context_builder.SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); - read_context_builder.AddOption(Options::MERGE_ENGINE, "partial-update") - .AddOption(Options::IGNORE_DELETE, "true"); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); + read_context_builder_options[Options::MERGE_ENGINE] = "partial-update"; + read_context_builder_options[Options::IGNORE_DELETE] = "true"; + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 3); @@ -2047,15 +2233,20 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot5WithLimit) { // in pk mor mode, limit does not take effect, as we do not know the number of records after // merging + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); scan_context_builder.SetLimit(6); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 5); @@ -2087,17 +2278,22 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorStreamFromSnapshot4) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "4") - .AddOption(Options::SCAN_MODE, "from-snapshot-full") - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "4"; + scan_context_builder_options[Options::SCAN_MODE] = "from-snapshot-full"; + scan_context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); + + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); // first plan is snapshot4 (isStreaming=false), second plan is snapshot5 std::vector> expected_snapshot_ids = {4, 5}; @@ -2132,17 +2328,22 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorStreamFromSnapshot1) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1") - .AddOption(Options::SCAN_MODE, "from-snapshot-full") - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + scan_context_builder_options[Options::SCAN_MODE] = "from-snapshot-full"; + scan_context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); + + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); std::vector> expected_snapshot_ids = {1, 2, 4, 5}; auto expected_snapshot1_batch = std::make_shared( @@ -2193,17 +2394,22 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorStreamFromSnapshot2) { // use from-snapshot mode, first plan is empty, snapshot 2 in second plan only includes delta // files + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "2") - .AddOption(Options::SCAN_MODE, "from-snapshot") - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "2"; + scan_context_builder_options[Options::SCAN_MODE] = "from-snapshot"; + scan_context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); + + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); std::vector> expected_snapshot_ids = {std::nullopt, 2, 4, 5}; auto expected_snapshot2_stream = std::make_shared( @@ -2238,21 +2444,26 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorStreamFromSnapshot5WithPredicate) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "5") - .AddOption(Options::SCAN_MODE, "from-snapshot") - .WithStreamingMode(true); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "5"; + scan_context_builder_options[Options::SCAN_MODE] = "from-snapshot"; + scan_context_builder.WithStreamingMode(true); auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/3, /*field_name=*/"f3", FieldType::DOUBLE, Literal(50.0)); scan_context_builder.SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); read_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); // first plan is snapshot 5 with empty plan, second plan is snapshot 5 with delta files // (predicates take no effects in delta mode) @@ -2274,24 +2485,26 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorStreamPredicateFilter) { std::string table_path = paimon::test::GetDataDir() + FileFormat() + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "5") - .AddOption(Options::SCAN_MODE, "from-snapshot") - .WithStreamingMode(true); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "5"; + scan_context_builder_options[Options::SCAN_MODE] = "from-snapshot"; + scan_context_builder.WithStreamingMode(true); std::shared_ptr predicate = PredicateBuilder::GreaterThan( /*field_index=*/3, /*field_name=*/"f3", FieldType::DOUBLE, Literal(31.05)); scan_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, - FinishScanContext(scan_context_builder)); + FinishScanContext(scan_context_builder, scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + TableScan::Create(std::move(scan_context), scan_context_builder_options)); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); read_context_builder.SetPredicate(predicate).EnablePredicateFilter(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(std::move(read_context), read_context_builder_options)); std::vector> expected_snapshot_ids = {std::nullopt, 5}; auto expected_snapshot5_stream = std::make_shared( @@ -2313,15 +2526,21 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithFirstRowBatchScanSnapshot5) { "/pk_table_scan_and_read_first_row.db/pk_table_scan_and_read_first_row/"; // normal batch scan case for pk+first row, all data in level 0 is filtered out + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "5"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "5"; + + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 5); @@ -2353,17 +2572,22 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithFirstRowStreamFromSnapshot3) { paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_first_row.db/pk_table_scan_and_read_first_row/"; + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "3") - .AddOption(Options::SCAN_MODE, "from-snapshot-full") - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "3"; + scan_context_builder_options[Options::SCAN_MODE] = "from-snapshot-full"; + scan_context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); + + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); // from a compact snapshot std::vector> expected_snapshot_ids = {3, 4, 5}; @@ -2406,15 +2630,21 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithFirstRowStreamFromSnapshot5) { paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_first_row.db/pk_table_scan_and_read_first_row/"; + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_MODE, "latest-full").WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - + scan_context_builder_options[Options::SCAN_MODE] = "latest-full"; + scan_context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); + + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); // from an append snapshot, first plan is snapshot5 with merge, all level 0 data is merged std::vector> expected_snapshot_ids = {5}; @@ -2445,15 +2675,20 @@ TEST_P(ScanAndReadInteTest, TestWithPKWith09VersionDvBatchScanLatestSnapshot) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_09.db/pk_09/"; // normal batch scan case for pk+dv (09 version) + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); scan_context_builder.SetLimit(2); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 8); @@ -2486,15 +2721,21 @@ TEST_P(ScanAndReadInteTest, TestWithEmptyPartitionValue) { [&](const std::string& table_path, const std::vector>& partition_filters, const std::shared_ptr& expected) { + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); scan_context_builder.SetPartitionFilter(partition_filters); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN( + auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context), + scan_context_builder_options)); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context), + read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -2555,15 +2796,21 @@ TEST_P(ScanAndReadInteTest, TestWithMultipleEmptyPartitionValue) { auto check_result = [&](const std::vector>& partition_filters, const std::shared_ptr& expected) { + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); scan_context_builder.SetPartitionFilter(partition_filters); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN( + auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context), + scan_context_builder_options)); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context), + read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -2604,10 +2851,14 @@ TEST_P(ScanAndReadInteTest, TestMemoryUse) { std::string table_path = paimon::test::GetDataDir() + file_format + "/append_09.db/append_09/"; // scan + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -2616,12 +2867,14 @@ TEST_P(ScanAndReadInteTest, TestMemoryUse) { // read std::shared_ptr read_pool = GetMemoryPool(); { + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); read_context_builder.WithMemoryPool(read_pool); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context), + read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -2705,26 +2958,35 @@ TEST_P(ScanAndReadInteTest, TestPkScanWithPostponeBucket) { { // batch scan + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); scan_context_builder.WithStreamingMode(false); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context), + scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 2); ASSERT_TRUE(result_plan->Splits().empty()); } { // stream scan: from snapshot 1 + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1").WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + scan_context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context), + scan_context_builder_options)); + + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context), + read_context_builder_options)); std::vector> expected_snapshot_ids = {std::nullopt, 1, 2}; auto expected_snapshot1 = std::make_shared( @@ -2749,16 +3011,22 @@ TEST_P(ScanAndReadInteTest, TestPkScanWithPostponeBucket) { } { // stream scan: from snapshot 2 + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "2").WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "2"; + scan_context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context), + scan_context_builder_options)); + + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context), + read_context_builder_options)); std::vector> expected_snapshot_ids = {std::nullopt, 2}; auto expected_snapshot2 = std::make_shared( @@ -2775,17 +3043,20 @@ TEST_P(ScanAndReadInteTest, TestPkScanWithPostponeBucket) { } { // stream scan: from snapshot full + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1") - .AddOption(Options::SCAN_MODE, "from-snapshot-full") - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + scan_context_builder_options[Options::SCAN_MODE] = "from-snapshot-full"; + scan_context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context), + scan_context_builder_options)); ReadContextBuilder read_context_builder(table_path); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context), {})); std::vector> expected_snapshot_ids = {1, 2}; auto expected_snapshot1 = std::make_shared( @@ -2817,22 +3088,27 @@ TEST_P(ScanAndReadInteTest, TestScanWithPredicateAndReadWithUnorderedFieldForPar } std::string table_path = paimon::test::GetDataDir() + "parquet/parquet_append_table.db/parquet_append_table"; + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); auto predicate = PredicateBuilder::LessThan( /*field_index=*/3, /*field_name=*/"f4", FieldType::INT, Literal(300006)); scan_context_builder.SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 2); ASSERT_EQ(result_plan->Splits().size(), 1); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); read_context_builder.SetReadFieldNames({"f10", "f8", "f4", "f13"}); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -2868,11 +3144,15 @@ TEST_P(ScanAndReadInteTest, TestPkSchemaEvolutionScanWithRenamedPkPredicate) { auto predicate = PredicateBuilder::GreaterThan( /*field_index=*/2, /*field_name=*/"key_2", FieldType::INT, Literal(500)); + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "6"); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "6"; + scan_context_builder.SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 6); ASSERT_EQ(result_plan->Splits().size(), 1); @@ -2885,11 +3165,13 @@ TEST_P(ScanAndReadInteTest, TestPkSchemaEvolutionScanWithRenamedPkPredicate) { } ASSERT_EQ(data_file_count, 1); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); read_context_builder.SetReadFieldNames({"key1", "k", "key_2", "c", "d", "a", "key0", "e"}); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -2921,20 +3203,26 @@ TEST_P(ScanAndReadInteTest, TestAppendTableWithMultipleFileFormat) { paimon::test::GetDataDir() + "/append_table_with_multiple_file_format.db/append_table_with_multiple_file_format"; // scan + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "2"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "2"; + + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 2); auto splits = result_plan->Splits(); ASSERT_EQ(1, splits.size()); // read + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -2960,19 +3248,24 @@ TEST_P(ScanAndReadInteTest, TestPkDvTableIndexInDataAndNoExternalPath) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_dv_index_in_data_no_external.db/pk_dv_index_in_data_no_external"; // scan + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 4); auto splits = result_plan->Splits(); ASSERT_EQ(3, splits.size()); // read + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -3000,19 +3293,24 @@ TEST_P(ScanAndReadInteTest, TestPkDvTableIndexNotInDataAndNoExternalPath) { paimon::test::GetDataDir() + file_format + "/pk_dv_index_not_in_data_no_external.db/pk_dv_index_not_in_data_no_external"; // scan + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 4); auto splits = result_plan->Splits(); ASSERT_EQ(3, splits.size()); // read + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -3042,9 +3340,12 @@ TEST_P(ScanAndReadInteTest, TestPkDvTableIndexNotInDataAndWithExternalPath) { std::string external_path = paimon::test::GetDataDir() + file_format + "/pk_dv_index_not_in_data_with_external.db/external"; // scan + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 4); @@ -3053,10 +3354,12 @@ TEST_P(ScanAndReadInteTest, TestPkDvTableIndexNotInDataAndWithExternalPath) { AdjustSplitWithExternalPath("FILE:/tmp/external", external_path, /*adjust_index=*/false, &splits); // read + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -3087,22 +3390,28 @@ TEST_P(ScanAndReadInteTest, TestScanAndReadWithDisableIndex) { Literal(FieldType::STRING, "Alice0", 6)); // scan + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN( - auto scan_context, - scan_context_builder.AddOption("file-index.read.enabled", "false").Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + scan_context_builder_options["file-index.read.enabled"] = "false"; + ASSERT_OK_AND_ASSIGN(auto scan_context, + + scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); auto splits = result_plan->Splits(); ASSERT_EQ(1, splits.size()); // read + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - read_context_builder.AddOption("file-index.read.enabled", "false"); - AddReadOptionsForPrefetch(&read_context_builder); + read_context_builder_options["file-index.read.enabled"] = "false"; + + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -3162,24 +3471,29 @@ TEST(ScanAndReadFileIndexInteTest, TestPkDvEmbeddedBitmapFiltersScanAndRead) { // file. Disabling file-index reads verifies that baseline explicitly. const std::shared_ptr absent_predicate = PredicateBuilder::Equal( /*field_index=*/1, /*field_name=*/"indexed_value", FieldType::INT, Literal(15)); + std::map stats_only_scan_builder_options; ScanContextBuilder stats_only_scan_builder(table_path); - stats_only_scan_builder.SetOptions(options) - .AddOption(Options::FILE_INDEX_READ_ENABLED, "false") - .SetPredicate(absent_predicate); + stats_only_scan_builder_options = options; + stats_only_scan_builder_options[Options::FILE_INDEX_READ_ENABLED] = "false"; + stats_only_scan_builder.SetPredicate(absent_predicate); ASSERT_OK_AND_ASSIGN(std::unique_ptr stats_only_scan_context, stats_only_scan_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr stats_only_scan, - TableScan::Create(std::move(stats_only_scan_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr stats_only_scan, + TableScan::Create(std::move(stats_only_scan_context), stats_only_scan_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr stats_only_plan, stats_only_scan->CreatePlan()); ASSERT_FALSE(stats_only_plan->Splits().empty()); // The embedded bitmap knows that 15 is absent and eliminates the file during scan planning. + std::map indexed_scan_builder_options; ScanContextBuilder indexed_scan_builder(table_path); - indexed_scan_builder.SetOptions(options).SetPredicate(absent_predicate); + indexed_scan_builder_options = options; + indexed_scan_builder.SetPredicate(absent_predicate); ASSERT_OK_AND_ASSIGN(std::unique_ptr indexed_scan_context, indexed_scan_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr indexed_scan, - TableScan::Create(std::move(indexed_scan_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr indexed_scan, + TableScan::Create(std::move(indexed_scan_context), indexed_scan_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr indexed_plan, indexed_scan->CreatePlan()); ASSERT_TRUE(indexed_plan->Splits().empty()); @@ -3188,22 +3502,25 @@ TEST(ScanAndReadFileIndexInteTest, TestPkDvEmbeddedBitmapFiltersScanAndRead) { // the other rows. const std::shared_ptr present_predicate = PredicateBuilder::Equal( /*field_index=*/1, /*field_name=*/"indexed_value", FieldType::INT, Literal(20)); + std::map retained_scan_builder_options; ScanContextBuilder retained_scan_builder(table_path); - retained_scan_builder.SetOptions(options).SetPredicate(present_predicate); + retained_scan_builder_options = options; + retained_scan_builder.SetPredicate(present_predicate); ASSERT_OK_AND_ASSIGN(std::unique_ptr retained_scan_context, retained_scan_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr retained_scan, - TableScan::Create(std::move(retained_scan_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr retained_scan, + TableScan::Create(std::move(retained_scan_context), retained_scan_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr retained_plan, retained_scan->CreatePlan()); ASSERT_FALSE(retained_plan->Splits().empty()); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - read_context_builder.SetOptions(options) - .SetPredicate(present_predicate) - .EnablePredicateFilter(false); + read_context_builder_options = options; + read_context_builder.SetPredicate(present_predicate).EnablePredicateFilter(false); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(retained_plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, @@ -3263,24 +3580,29 @@ TEST(ScanAndReadFileIndexInteTest, TestPkDvReconstructsFileIndexPredicateAfterSc /*field_index=*/1, /*field_name=*/"renamed_value", FieldType::INT, Literal(15)); // The evolved min/max range is still [10, 30], so statistics alone retain the old file. + std::map stats_only_scan_builder_options; ScanContextBuilder stats_only_scan_builder(table_path); - stats_only_scan_builder.SetOptions(options) - .AddOption(Options::FILE_INDEX_READ_ENABLED, "false") - .SetPredicate(predicate); + stats_only_scan_builder_options = options; + stats_only_scan_builder_options[Options::FILE_INDEX_READ_ENABLED] = "false"; + stats_only_scan_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(std::unique_ptr stats_only_scan_context, stats_only_scan_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr stats_only_scan, - TableScan::Create(std::move(stats_only_scan_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr stats_only_scan, + TableScan::Create(std::move(stats_only_scan_context), stats_only_scan_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr stats_only_plan, stats_only_scan->CreatePlan()); ASSERT_FALSE(stats_only_plan->Splits().empty()); // Reconstructing the predicate to schema-0 lets its embedded bitmap prove that 15 is absent. + std::map indexed_scan_builder_options; ScanContextBuilder indexed_scan_builder(table_path); - indexed_scan_builder.SetOptions(options).SetPredicate(predicate); + indexed_scan_builder_options = options; + indexed_scan_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(std::unique_ptr indexed_scan_context, indexed_scan_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr indexed_scan, - TableScan::Create(std::move(indexed_scan_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr indexed_scan, + TableScan::Create(std::move(indexed_scan_context), indexed_scan_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr indexed_plan, indexed_scan->CreatePlan()); ASSERT_TRUE(indexed_plan->Splits().empty()); } @@ -3293,9 +3615,12 @@ TEST_P(ScanAndReadInteTest, TestPkDvTableIndexInDataAndWithExternalPath) { std::string external_path = paimon::test::GetDataDir() + file_format + "/pk_dv_index_in_data_with_external.db/external"; // scan + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 4); @@ -3304,10 +3629,12 @@ TEST_P(ScanAndReadInteTest, TestPkDvTableIndexInDataAndWithExternalPath) { AdjustSplitWithExternalPath("FILE:/tmp/external", external_path, /*adjust_index=*/true, &splits); // read + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -3334,9 +3661,12 @@ TEST_P(ScanAndReadInteTest, TestTimestampType) { std::string table_path = paimon::test::GetDataDir() + file_format + "/append_with_multiple_ts_precision_and_timezone.db" "/append_with_multiple_ts_precision_and_timezone/"; + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -3344,10 +3674,12 @@ TEST_P(ScanAndReadInteTest, TestTimestampType) { ASSERT_EQ(1, splits.size()); // read + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -3382,9 +3714,12 @@ TEST_P(ScanAndReadInteTest, TestCastTimestampType) { "/append_with_cast_timestamp.db" "/append_with_cast_timestamp/"; // scan + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -3392,10 +3727,12 @@ TEST_P(ScanAndReadInteTest, TestCastTimestampType) { ASSERT_EQ(1, splits.size()); // read + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -3433,7 +3770,7 @@ TEST_F(ScanAndReadInteTest, TestParquetDictionaryBinaryStoredArrowSchema) { ScanContextBuilder scan_context_builder(table_path); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + TableScan::Create(std::move(scan_context), {})); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, table_scan->CreatePlan()); ASSERT_EQ(1, plan->SnapshotId().value()); ASSERT_EQ(1, plan->Splits().size()); @@ -3441,7 +3778,7 @@ TEST_F(ScanAndReadInteTest, TestParquetDictionaryBinaryStoredArrowSchema) { ReadContextBuilder read_context_builder(table_path); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(std::move(read_context), {})); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, @@ -3509,6 +3846,7 @@ TEST_F(ScanAndReadInteTest, TestLanceJavaCompatibility) { const std::shared_ptr& expected_result, bool exact_filter = false) { ScanContextBuilder scan_builder(table_path); + std::map read_builder_options; ReadContextBuilder read_builder(table_path); if (predicate) { scan_builder.SetPredicate(predicate); @@ -3521,14 +3859,16 @@ TEST_F(ScanAndReadInteTest, TestLanceJavaCompatibility) { read_builder.SetReadFieldNames(projection); } read_builder.EnablePredicateFilter(exact_filter); - read_builder.AddOption("read.batch-size", "1"); + read_builder_options["read.batch-size"] = "1"; + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan, TableScan::Create(std::move(scan_context), {})); ASSERT_OK_AND_ASSIGN(auto plan, scan->CreatePlan()); ASSERT_EQ(plan->SnapshotId(), std::optional(2)); ASSERT_FALSE(plan->Splits().empty()); ASSERT_OK_AND_ASSIGN(auto read_context, read_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto read, + TableRead::Create(std::move(read_context), read_builder_options)); ASSERT_OK_AND_ASSIGN(auto reader, read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(std::move(reader))); ASSERT_TRUE(actual); @@ -3597,21 +3937,27 @@ TEST_F(ScanAndReadInteTest, TestLancePythonCompatibility) { auto check = [&](int64_t snapshot_id, const std::vector& projection, const std::shared_ptr& expected_result) { SCOPED_TRACE("snapshot " + std::to_string(snapshot_id)); + std::map scan_builder_options; ScanContextBuilder scan_builder(table_path); - scan_builder.AddOption(Options::SCAN_SNAPSHOT_ID, std::to_string(snapshot_id)); + scan_builder_options[Options::SCAN_SNAPSHOT_ID] = std::to_string(snapshot_id); + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan, + TableScan::Create(std::move(scan_context), scan_builder_options)); ASSERT_OK_AND_ASSIGN(auto plan, scan->CreatePlan()); ASSERT_EQ(plan->SnapshotId(), std::optional(snapshot_id)); ASSERT_FALSE(plan->Splits().empty()); + std::map read_builder_options; ReadContextBuilder read_builder(table_path); - read_builder.AddOption("read.batch-size", "1"); + read_builder_options["read.batch-size"] = "1"; + if (!projection.empty()) { read_builder.SetReadFieldNames(projection); } ASSERT_OK_AND_ASSIGN(auto read_context, read_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto read, + TableRead::Create(std::move(read_context), read_builder_options)); ASSERT_OK_AND_ASSIGN(auto reader, read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(std::move(reader))); ASSERT_TRUE(actual); @@ -3695,13 +4041,13 @@ TEST_F(ScanAndReadInteTest, TestMosaicJavaAndPythonCompatibility) { ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + TableScan::Create(std::move(scan_context), {})); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, table_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(std::move(read_context), {})); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, @@ -3754,11 +4100,11 @@ TEST_F(ScanAndReadInteTest, TestVortexJavaCompatibility) { ReadContextBuilder read_context_builder(table_path); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + TableScan::Create(std::move(scan_context), {})); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, table_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(std::move(read_context), {})); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, @@ -3783,12 +4129,12 @@ TEST_F(ScanAndReadInteTest, TestVortexJavaCompatibilityLtzAcrossTimezones) { ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + TableScan::Create(std::move(scan_context), {})); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, table_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(std::move(read_context), {})); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, @@ -3857,11 +4203,11 @@ TEST_F(ScanAndReadInteTest, TestVortexJavaCompatibilityAllTypes) { ReadContextBuilder read_context_builder(table_path); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + TableScan::Create(std::move(scan_context), {})); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, table_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(std::move(read_context), {})); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, @@ -3876,22 +4222,28 @@ TEST_F(ScanAndReadInteTest, TestAvroWithAppendTable) { auto read_data = [](int64_t snapshot_id, const std::string& result_json) { std::string table_path = GetDataDir() + "/avro/append_multiple.db/append_multiple"; // scan + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, std::to_string(snapshot_id)); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = std::to_string(snapshot_id); + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context), + scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), snapshot_id); auto splits = result_plan->Splits(); ASSERT_EQ(3, splits.size()); // read + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - read_context_builder.AddOption("test.enable-adaptive-prefetch-strategy", "false"); + read_context_builder_options["test.enable-adaptive-prefetch-strategy"] = "false"; + read_context_builder.EnablePrefetch(true).SetPrefetchBatchCount(3); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context), + read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -3951,10 +4303,13 @@ TEST_F(ScanAndReadInteTest, TestAvroWithPkTable) { std::string table_path = GetDataDir() + "/avro/pk_with_multiple_type.db/pk_with_multiple_type"; // scan + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, std::to_string(snapshot_id)); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = std::to_string(snapshot_id); + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context), + scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), snapshot_id); @@ -3965,7 +4320,7 @@ TEST_F(ScanAndReadInteTest, TestAvroWithPkTable) { ReadContextBuilder read_context_builder(table_path); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context), {})); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -4024,32 +4379,41 @@ TEST_P(ScanAndReadInteTest, TestWithPKBucketSelectByPredicate) { // Historical files use two buckets even when the current option has changed. for (const std::string current_bucket_count : {"2", "4", "8", "17"}) { SCOPED_TRACE(current_bucket_count); + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "6"); - scan_context_builder.AddOption(Options::BUCKET, current_bucket_count); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "6"; + + scan_context_builder_options[Options::BUCKET] = current_bucket_count; + scan_context_builder.SetPartitionFilter({{{"f1", "10"}}}); scan_context_builder.SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context), + scan_context_builder_options)); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); read_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context), + read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); if (EnableSnapshotLiveManifestCache()) { ASSERT_OK_AND_ASSIGN(uint64_t enabled, table_scan->GetMetrics()->GetCounter( ScanMetrics::LAST_SNAPSHOT_CACHE_ENABLED)); ASSERT_EQ(enabled, 1); + std::map cached_builder_options; ScanContextBuilder cached_builder(table_path); - cached_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "6") - .AddOption(Options::BUCKET, current_bucket_count) - .SetPartitionFilter({{{"f1", "10"}}}) - .SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto cached_context, FinishScanContext(cached_builder)); - ASSERT_OK_AND_ASSIGN(auto cached_scan, TableScan::Create(std::move(cached_context))); + cached_builder_options[Options::SCAN_SNAPSHOT_ID] = "6"; + cached_builder_options[Options::BUCKET] = current_bucket_count; + cached_builder.SetPartitionFilter({{{"f1", "10"}}}).SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto cached_context, + FinishScanContext(cached_builder, cached_builder_options)); + ASSERT_OK_AND_ASSIGN(auto cached_scan, TableScan::Create(std::move(cached_context), + cached_builder_options)); ASSERT_OK_AND_ASSIGN(result_plan, cached_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(uint64_t hit, cached_scan->GetMetrics()->GetCounter( ScanMetrics::LAST_SNAPSHOT_CACHE_HIT)); @@ -4099,16 +4463,19 @@ TEST_P(ScanAndReadInteTest, TestReadNullableMapKey) { ScanContextBuilder scan_context_builder(table_path); ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context), {})); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_TRUE(result_plan->SnapshotId()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); ASSERT_EQ(result_plan->Splits().size(), 1); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - read_context_builder.AddOption(Options::READ_BATCH_SIZE, "1"); + read_context_builder_options[Options::READ_BATCH_SIZE] = "1"; + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); const std::vector expected_rows = {R"([[0, 1, [["one", 10]]]])", @@ -4140,10 +4507,12 @@ TEST_P(ScanAndReadInteTest, TestCountRowsEmptySplits) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); // CreateCountReader with empty splits should return 0 rows. std::vector> empty_splits; @@ -4158,25 +4527,32 @@ TEST_P(ScanAndReadInteTest, TestCountRowsConsistencyWithCreateReader) { "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; // Scan latest snapshot + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // Method 1: CreateCountReader + iterate batches + std::map count_context_builder_options; ReadContextBuilder count_context_builder(table_path); - AddReadOptionsForPrefetch(&count_context_builder); + AddReadOptionsForPrefetch(&count_context_builder, count_context_builder_options); ASSERT_OK_AND_ASSIGN(auto count_read_context, count_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto count_table_read, TableRead::Create(std::move(count_read_context))); + ASSERT_OK_AND_ASSIGN(auto count_table_read, TableRead::Create(std::move(count_read_context), + count_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto count_reader, count_table_read->CreateCountReader(result_plan->Splits())); ASSERT_OK_AND_ASSIGN(int64_t count_result, count_reader->CountRows()); // Method 2: CreateReader + iterate batches + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -4192,19 +4568,24 @@ TEST_P(ScanAndReadInteTest, TestCreateCountReaderWithPredicateNotSupported) { "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; // Create splits from latest snapshot. + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // Set predicate in read context. CountReader currently does not support predicate pushdown. auto predicate = PredicateBuilder::Equal(/*field_index=*/2, /*field_name=*/"f2", FieldType::INT, Literal(static_cast(0))); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); read_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_NOK_WITH_MSG(table_read->CreateCountReader(result_plan->Splits()), "predicate pushdown is not supported"); @@ -4216,15 +4597,20 @@ TEST_P(ScanAndReadInteTest, TestCreateCountReaderWithForceKeepDeleteNotSupported "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; // Create splits from latest snapshot. + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); auto* key_value_table_read = dynamic_cast(table_read.get()); ASSERT_TRUE(key_value_table_read != nullptr); diff --git a/test/inte/scan_inte_test.cpp b/test/inte/scan_inte_test.cpp index 3f949484d..814512b12 100644 --- a/test/inte/scan_inte_test.cpp +++ b/test/inte/scan_inte_test.cpp @@ -64,15 +64,16 @@ enum class ManifestCacheMode { NoCache, Cache }; class ScanInteTest : public testing::TestWithParam { public: - Result> FinishScanContext(ScanContextBuilder& builder) { + Result> FinishScanContext( + ScanContextBuilder& builder, std::map& options) { if (GetParam() == ManifestCacheMode::Cache) { if (!cache_) { cache_ = std::make_shared(std::map{ {CacheKind::MANIFEST, 64 * 1024 * 1024}, {CacheKind::SNAPSHOT_LIVE_MANIFEST, 64 * 1024 * 1024}}); } - builder.AddOption(Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, "3") - .WithCache(cache_); + options[Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS] = "3"; + builder.WithCache(cache_); } return builder.Finish(); } @@ -269,10 +270,13 @@ TEST(ScanInteManifestCacheTest, TestRepeatedScanReusesManifestCache) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; auto run_scan = [&]() -> Result> { + std::map context_builder_options; ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1").WithCache(manifest_cache); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + context_builder.WithCache(manifest_cache); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); return table_scan->CreatePlan(); }; @@ -293,19 +297,21 @@ TEST(ScanInteManifestCacheTest, TestRepeatedScanReusesManifestCache) { Result>> RunBucketSnapshotScan( const std::string& table_path, int64_t snapshot_id, int32_t bucket, const std::shared_ptr& cache, int32_t max_snapshot_live_manifest_versions) { + std::map context_builder_options; ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, std::to_string(snapshot_id)) - .SetBucketFilter(bucket); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = std::to_string(snapshot_id); + context_builder.SetBucketFilter(bucket); if (max_snapshot_live_manifest_versions > 0) { - context_builder.AddOption(Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, - std::to_string(max_snapshot_live_manifest_versions)); + context_builder_options[Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS] = + std::to_string(max_snapshot_live_manifest_versions); } if (cache) { context_builder.WithCache(cache); } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto plan, table_scan->CreatePlan()); std::vector> data_splits; @@ -396,11 +402,14 @@ TEST(ScanInteManifestCacheTest, TestSnapshotLiveManifestCacheRetainsSnapshotsPer TEST_P(ScanInteTest, TestScanAppendWithSnapshot1) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1"); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, - FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -453,10 +462,14 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot1) { TEST_P(ScanInteTest, TestScanAppendWithSnapshot3) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "3"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "3"; + + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id ASSERT_EQ(result_plan->SnapshotId().value(), 3); @@ -514,13 +527,17 @@ TEST_P(ScanInteTest, TestScanAppendWithSpecificTableSchema) { std::string table_path = data_dir + "orc/append_09.db/append_09"; auto check_result = [&](const std::optional& specific_table_schema) { + std::map context_builder_options; ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "3"); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "3"; + if (specific_table_schema) { context_builder.SetTableSchema(specific_table_schema.value()); } - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 3); @@ -578,10 +595,14 @@ TEST_P(ScanInteTest, TestScanAppendWithSpecificTableSchema) { TEST_P(ScanInteTest, TestScanInvalidSnapshot) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "100"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "100"; + + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); ASSERT_NOK_WITH_MSG( table_scan->CreatePlan(), "The specified scan snapshotId 100 is out of available snapshotId range [1, 5]."); @@ -589,10 +610,14 @@ TEST_P(ScanInteTest, TestScanInvalidSnapshot) { TEST_P(ScanInteTest, TestBatchScanMultipleTimes) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // batch scan multiple ASSERT_NOK_WITH_MSG(table_scan->CreatePlan(), "end of scan"); @@ -600,13 +625,17 @@ TEST_P(ScanInteTest, TestBatchScanMultipleTimes) { TEST_P(ScanInteTest, TestScanAppendWithSnapshot3WithSplitTargetSize) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "3") - .AddOption(Options::SOURCE_SPLIT_OPEN_FILE_COST, "1024") - .AddOption(Options::SOURCE_SPLIT_TARGET_SIZE, "2048"); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "3"; + context_builder_options[Options::SOURCE_SPLIT_OPEN_FILE_COST] = "1024"; + context_builder_options[Options::SOURCE_SPLIT_TARGET_SIZE] = "2048"; + // open cost = 1024, and split target size is 2048, indicates at most 2 files in a split - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id ASSERT_EQ(result_plan->SnapshotId().value(), 3); @@ -670,10 +699,14 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot3WithSplitTargetSize) { TEST_P(ScanInteTest, TestScanAppendWithSnapshot3WithRowCountLimit) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "3").SetLimit(3); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "3"; + context_builder.SetLimit(3); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id ASSERT_EQ(result_plan->SnapshotId().value(), 3); @@ -716,10 +749,14 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot3WithRowCountLimit) { TEST_P(ScanInteTest, TestScanAppendWithSnapshot3WithBucketFilter) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); - context_builder.SetBucketFilter(0).AddOption(Options::SCAN_SNAPSHOT_ID, "3"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "3"; + context_builder.SetBucketFilter(0); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id ASSERT_EQ(result_plan->SnapshotId().value(), 3); @@ -759,10 +796,14 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot3WithBucketFilter) { TEST_P(ScanInteTest, TestScanAppendWithStreamWithDefaultMode) { // from snapshot is specified std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1").WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); DataSplitImpl::Builder builder1_1( BinaryRowGenerator::GenerateRow({10}, pool_.get()), @@ -871,11 +912,14 @@ TEST_P(ScanInteTest, TestScanAppendWithStreamWithDefaultMode) { TEST_P(ScanInteTest, TestScanAppendWithStreamOfLatestFullMode) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_MODE, StartupMode::LatestFull().ToString()) - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_MODE] = StartupMode::LatestFull().ToString(); + context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); DataSplitImpl::Builder builder1( BinaryRowGenerator::GenerateRow({10}, pool_.get()), @@ -925,10 +969,14 @@ TEST_P(ScanInteTest, TestScanAppendWithStreamOfLatestFullMode) { TEST_P(ScanInteTest, TestScanAppendWithBatchScanOfLatestMode) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_MODE, StartupMode::Latest().ToString()); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_MODE] = StartupMode::Latest().ToString(); + + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id ASSERT_EQ(result_plan->SnapshotId().value(), 5); @@ -981,11 +1029,14 @@ TEST_P(ScanInteTest, TestScanAppendWithBatchScanOfLatestMode) { TEST_P(ScanInteTest, TestScanAppendWithStreamOfLatestMode) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_MODE, StartupMode::Latest().ToString()) - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_MODE] = StartupMode::Latest().ToString(); + context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); std::vector> expected_snapshot_ids = {}; CheckStreamScanResult(table_scan.get(), expected_snapshot_ids, {}); @@ -994,12 +1045,15 @@ TEST_P(ScanInteTest, TestScanAppendWithStreamOfLatestMode) { TEST_P(ScanInteTest, TestScanAppendWithStreamOfFromSnapshotMode) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_MODE, StartupMode::FromSnapshot().ToString()) - .AddOption(Options::SCAN_SNAPSHOT_ID, "2") - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_MODE] = StartupMode::FromSnapshot().ToString(); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "2"; + context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); DataSplitImpl::Builder builder2_1( BinaryRowGenerator::GenerateRow({10}, pool_.get()), @@ -1066,13 +1120,16 @@ TEST_P(ScanInteTest, TestScanAppendWithStreamOfFromSnapshotMode) { TEST_P(ScanInteTest, TestScanAppendWithStreamOfFromSnapshotFullMode) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_MODE, StartupMode::FromSnapshotFull().ToString()) - .AddOption(Options::SCAN_SNAPSHOT_ID, "2") - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); + context_builder_options[Options::SCAN_MODE] = StartupMode::FromSnapshotFull().ToString(); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "2"; + context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); ASSERT_TRUE(scan_context); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); DataSplitImpl::Builder builder1_1( BinaryRowGenerator::GenerateRow({10}, pool_.get()), @@ -1151,20 +1208,26 @@ TEST_P(ScanInteTest, TestScanAppendWithStreamOfFromSnapshotFullMode) { TEST_P(ScanInteTest, TestScanAppendWithInvalidOptions) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; { + std::map context_builder_options; ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_MODE, StartupMode::FromSnapshot().ToString()) - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_MODE] = StartupMode::FromSnapshot().ToString(); + context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); ASSERT_NOK_WITH_MSG( table_scan->CreatePlan(), "scan.snapshot-id or scan.tag-name must be set when startup mode is FROM_SNAPSHOT"); } { + std::map context_builder_options; ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::BUCKET, "-2").WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), + context_builder_options[Options::BUCKET] = "-2"; + context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context), context_builder_options), "do not support bucket=-2 in scan process"); } } @@ -1177,10 +1240,14 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithEqualPredicate) { PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, Literal(FieldType::STRING, val.data(), val.size())); + std::map context_builder_options; ScanContextBuilder context_builder(table_path); - context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + context_builder.SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -1223,12 +1290,14 @@ TEST_P(ScanInteTest, TestScanAppendWithStreamWithAndPredicate) { PredicateBuilder::And({greater_than_predicate, less_than_predicate, not_in_predicate, not_equal})); + std::map context_builder_options; ScanContextBuilder context_builder(table_path); - context_builder.SetPredicate(predicate) - .AddOption(Options::SCAN_SNAPSHOT_ID, "1") - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + context_builder.SetPredicate(predicate).WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); DataSplitImpl::Builder builder1_2( BinaryRowGenerator::GenerateRow({10}, pool_.get()), @@ -1284,10 +1353,14 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithPartitionFilter) { std::map partition_keys; partition_keys["f1"] = "10"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); - context_builder.SetPartitionFilter({partition_keys}).AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + context_builder.SetPartitionFilter({partition_keys}); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -1330,10 +1403,13 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithInvalidPartitionFilter) { std::map partition_keys; partition_keys["invalid_partition_key"] = "10"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); - context_builder.SetPartitionFilter({partition_keys}).AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + context_builder.SetPartitionFilter({partition_keys}); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context), context_builder_options), "field invalid_partition_key does not exist in partition keys"); } @@ -1347,12 +1423,14 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithPartitionFilterAndPredicateF std::map partition_keys; partition_keys["f1"] = "10"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); - context_builder.SetPredicate(predicate) - .SetPartitionFilter({partition_keys}) - .AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + context_builder.SetPredicate(predicate).SetPartitionFilter({partition_keys}); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -1399,10 +1477,14 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithMultiPartitionKeys) { partition_keys["f1"] = "10"; partition_keys["f2"] = "0"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); - context_builder.SetPartitionFilter({partition_keys}).AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + context_builder.SetPartitionFilter({partition_keys}); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -1455,10 +1537,14 @@ TEST_P(ScanInteTest, TestScanAppendComplexDataWithSnapshot4WithPredicateFilter) Literal(paimon::Decimal(23, 5, 0))); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({predicate1, predicate2})); + std::map context_builder_options; ScanContextBuilder context_builder(table_path); - context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "4"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "4"; + context_builder.SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_TRUE(result_plan); // check snapshot id @@ -1516,10 +1602,14 @@ TEST_P(ScanInteTest, TestScanAppendComplexDataWithSnapshot4WithPredicateFilter2) Literal(FieldType::BINARY, "zoo", 3)); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({predicate1, predicate2})); + std::map context_builder_options; ScanContextBuilder context_builder(table_path); - context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "4"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "4"; + context_builder.SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id ASSERT_EQ(result_plan->SnapshotId().value(), 4); @@ -1571,12 +1661,16 @@ TEST_P(ScanInteTest, TestScanAppendComplexDataWithSnapshot4WithPredicateFilter2) TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithEnableStatsDenseStore) { std::string table_path = paimon::test::GetDataDir() + "orc/append_10_stats_dense_store.db/append_10_stats_dense_store"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/3, /*field_name=*/"f3", FieldType::DOUBLE, Literal(13.0)); - context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + context_builder.SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check data splits auto result_data_splits = CollectDataSplits(result_plan); @@ -1668,6 +1762,7 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithEnableStatsDenseStore) { TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithEnableStatsDenseStore2) { std::string table_path = paimon::test::GetDataDir() + "orc/append_10_stats_dense_store.db/append_10_stats_dense_store"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); auto greater_than = PredicateBuilder::GreaterThan(/*field_index=*/3, /*field_name=*/"f3", FieldType::DOUBLE, Literal(13.0)); @@ -1675,9 +1770,12 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithEnableStatsDenseStore2) { Literal(FieldType::STRING, "Emily", 5)); auto predicate = PredicateBuilder::And({greater_than, equal}).value(); - context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + context_builder.SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check data splits auto result_data_splits = CollectDataSplits(result_plan); @@ -1714,10 +1812,14 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithEnableStatsDenseStore2) { TEST_P(ScanInteTest, TestScanPKWithSnapshot1WithBucketStats) { std::string table_path = paimon::test::GetDataDir() + "orc/pk_table_with_total_buckets.db/pk_table_with_total_buckets"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1").SetBucketFilter(2); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + context_builder.SetBucketFilter(2); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); ASSERT_TRUE(result_plan->Splits().empty()); @@ -1726,19 +1828,25 @@ TEST_P(ScanInteTest, TestScanPKWithSnapshot1WithBucketStats) { TEST_P(ScanInteTest, TestScanPKWithInvalidOptions) { std::string table_path = paimon::test::GetDataDir() + "orc/pk_table_with_total_buckets.db/pk_table_with_total_buckets"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::BUCKET, "-1").WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), + context_builder_options[Options::BUCKET] = "-1"; + context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context), context_builder_options), "do not support pk table bucket=-1 in scan process"); } TEST_P(ScanInteTest, TestReadWithNoSnapshot) { std::string table_path = paimon::test::GetDataDir() + "orc/append_table_with_nested_type.db/append_table_with_nested_type"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_FALSE(result_plan->SnapshotId()); ASSERT_TRUE(result_plan->Splits().empty()); @@ -1748,6 +1856,7 @@ TEST_P(ScanInteTest, TestScanAppendWithAlterTableWithCast) { std::string table_path = paimon::test::GetDataDir() + "orc/append_table_alter_table_with_cast.db/append_table_alter_table_with_cast"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); auto child1 = @@ -1770,9 +1879,12 @@ TEST_P(ScanInteTest, TestScanAppendWithAlterTableWithCast) { FieldType::INT, Literal(80)); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({child1, child2, child3})); - context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "2"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "2"; + context_builder.SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check data splits auto result_data_splits = CollectDataSplits(result_plan); @@ -1813,6 +1925,7 @@ TEST_P(ScanInteTest, TestScanAppendWithAlterTableWithCast) { TEST_P(ScanInteTest, TestScanAppendWithAlterTableWithNoCast) { std::string table_path = paimon::test::GetDataDir() + "orc/append_table_with_alter_table.db/append_table_with_alter_table"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); auto child1 = PredicateBuilder::GreaterThan(/*field_index=*/2, /*field_name=*/"k", @@ -1821,9 +1934,12 @@ TEST_P(ScanInteTest, TestScanAppendWithAlterTableWithNoCast) { Literal(96)); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({child1, child2})); - context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "2"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "2"; + context_builder.SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check data splits auto result_data_splits = CollectDataSplits(result_plan); @@ -1891,6 +2007,7 @@ TEST_P(ScanInteTest, TestScanAppendWithAlterTableWithDenseField) { std::string table_path = paimon::test::GetDataDir() + "orc/append_table_with_alter_table_with_dense_field.db/" "append_table_with_alter_table_with_dense_field"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); auto child1 = PredicateBuilder::GreaterThan(/*field_index=*/2, /*field_name=*/"f1", @@ -1899,9 +2016,12 @@ TEST_P(ScanInteTest, TestScanAppendWithAlterTableWithDenseField) { FieldType::DOUBLE, Literal(0.0)); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({child1, child3})); - context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "2"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "2"; + context_builder.SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check data splits auto result_data_splits = CollectDataSplits(result_plan); @@ -1939,6 +2059,7 @@ TEST_P(ScanInteTest, TestScanAppendWithAlterTableWithDenseField) { TEST_P(ScanInteTest, TestScanAppendWithBitmapEmbeddedIndex) { std::string table_path = paimon::test::GetDataDir() + "orc/append_with_bitmap.db/append_with_bitmap/"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); auto child1 = PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, @@ -1947,9 +2068,12 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapEmbeddedIndex) { Literal(10)); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({child1, child2})); - context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + context_builder.SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check data splits auto result_data_splits = CollectDataSplits(result_plan); @@ -2016,6 +2140,7 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapEmbeddedIndex) { TEST_P(ScanInteTest, TestScanAppendWithBitmapEmbeddedIndexWithEmptyResult) { std::string table_path = paimon::test::GetDataDir() + "orc/append_with_bitmap.db/append_with_bitmap/"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); auto child1 = PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, @@ -2024,9 +2149,12 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapEmbeddedIndexWithEmptyResult) { Literal(10)); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({child1, child2})); - context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + context_builder.SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_TRUE(result_plan->Splits().empty()); } @@ -2035,6 +2163,7 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapNoEmbeddedIndex) { std::string table_path = paimon::test::GetDataDir() + "orc/append_with_bitmap_no_embedding.db/append_with_bitmap_no_embedding/"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); auto child1 = PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, @@ -2043,9 +2172,12 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapNoEmbeddedIndex) { Literal(10)); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({child1, child2})); - context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + context_builder.SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check data splits auto result_data_splits = CollectDataSplits(result_plan); @@ -2083,14 +2215,18 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapAndAlterTable) { std::string table_path = paimon::test::GetDataDir() + "orc/append_with_bitmap_alter_table.db/append_with_bitmap_alter_table/"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); // file0 will be removed as f5 not exists auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/3, /*field_name=*/"f5", FieldType::INT, Literal(100)); - context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "2"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "2"; + context_builder.SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check data splits @@ -2152,15 +2288,19 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapAndAlterTable3) { std::string table_path = paimon::test::GetDataDir() + "orc/append_with_bitmap_alter_table.db/append_with_bitmap_alter_table/"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); auto child1 = PredicateBuilder::IsNull(/*field_index=*/3, /*field_name=*/"f5", FieldType::INT); auto child2 = PredicateBuilder::LessThan(/*field_index=*/2, /*field_name=*/"f3", FieldType::DOUBLE, Literal(20.0)); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({child1, child2})); - context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "2"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "2"; + context_builder.SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check data splits @@ -2227,6 +2367,7 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapAndAlterTable2) { std::string table_path = paimon::test::GetDataDir() + "orc/append_with_bitmap_alter_table.db/append_with_bitmap_alter_table/"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); // in stats filter: predicate is trimmed as type for f1 is not consist: int->bigint @@ -2235,9 +2376,12 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapAndAlterTable2) { auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/0, /*field_name=*/"f1", FieldType::BIGINT, Literal(100l)); - context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "2"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "2"; + context_builder.SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check data splits @@ -2304,6 +2448,7 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapAndAlterTableWithEmptyResult) { std::string table_path = paimon::test::GetDataDir() + "orc/append_with_bitmap_alter_table.db/append_with_bitmap_alter_table/"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); // child1 will remove file1 for schema1 @@ -2314,9 +2459,12 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapAndAlterTableWithEmptyResult) { Literal(FieldType::STRING, "David", 5)); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({child1, child2})); - context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "2"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "2"; + context_builder.SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_TRUE(result_plan->Splits().empty()); } @@ -2324,11 +2472,14 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapAndAlterTableWithEmptyResult) { TEST_P(ScanInteTest, TestScanAppendWithTag1) { std::string table_path = paimon::test::GetDataDir() + "orc/append_table_with_tag.db/append_table_with_tag"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_TAG_NAME, "1"); + context_builder_options[Options::SCAN_TAG_NAME] = "1"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, - FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id ASSERT_EQ(1, result_plan->SnapshotId().value()); @@ -2337,10 +2488,14 @@ TEST_P(ScanInteTest, TestScanAppendWithTag1) { TEST_P(ScanInteTest, TestScanInvalidTag) { std::string table_path = paimon::test::GetDataDir() + "orc/append_table_with_tag.db/append_table_with_tag"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_TAG_NAME, "unknown"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_TAG_NAME] = "unknown"; + + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); ASSERT_NOK_WITH_MSG(table_scan->CreatePlan(), "Tag 'unknown' doesn't exist."); } @@ -2353,38 +2508,54 @@ TEST_P(ScanInteTest, TestWithAppendTimestampMillisBatchScan) { // EarlierOrEqual(snap3.time) → snap-3 { + std::map builder_options; ScanContextBuilder builder(table_path); - builder.AddOption(Options::SCAN_TIMESTAMP_MILLIS, std::to_string(snap3.TimeMillis())); - ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, FinishScanContext(builder)); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, TableScan::Create(std::move(ctx))); + builder_options[Options::SCAN_TIMESTAMP_MILLIS] = std::to_string(snap3.TimeMillis()); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, + FinishScanContext(builder, builder_options)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, + TableScan::Create(std::move(ctx), builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); ASSERT_EQ(plan->SnapshotId().value(), 3); } // EarlierOrEqual(snap3.time - 1) → snap-2 { + std::map builder_options; ScanContextBuilder builder(table_path); - builder.AddOption(Options::SCAN_TIMESTAMP_MILLIS, std::to_string(snap3.TimeMillis() - 1)); - ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, FinishScanContext(builder)); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, TableScan::Create(std::move(ctx))); + builder_options[Options::SCAN_TIMESTAMP_MILLIS] = std::to_string(snap3.TimeMillis() - 1); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, + FinishScanContext(builder, builder_options)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, + TableScan::Create(std::move(ctx), builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); ASSERT_EQ(plan->SnapshotId().value(), 2); } // EarlierOrEqual(INT64_MAX) → snap-5 (latest) { + std::map builder_options; ScanContextBuilder builder(table_path); - builder.AddOption(Options::SCAN_TIMESTAMP_MILLIS, - std::to_string(std::numeric_limits::max())); - ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, FinishScanContext(builder)); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, TableScan::Create(std::move(ctx))); + builder_options[Options::SCAN_TIMESTAMP_MILLIS] = + std::to_string(std::numeric_limits::max()); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, + FinishScanContext(builder, builder_options)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, + TableScan::Create(std::move(ctx), builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); ASSERT_EQ(plan->SnapshotId().value(), 5); } // EarlierOrEqual(0) → no snapshot found, expect error { + std::map builder_options; ScanContextBuilder builder(table_path); - builder.AddOption(Options::SCAN_TIMESTAMP_MILLIS, "0"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, FinishScanContext(builder)); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, TableScan::Create(std::move(ctx))); + builder_options[Options::SCAN_TIMESTAMP_MILLIS] = "0"; + + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, + FinishScanContext(builder, builder_options)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, + TableScan::Create(std::move(ctx), builder_options)); ASSERT_NOK_WITH_MSG(scan->CreatePlan(), "There is currently no snapshot earlier than or equal to timestamp"); } @@ -2400,10 +2571,14 @@ TEST_P(ScanInteTest, TestWithAppendTimestampMillisStreamScan) { // T=0: no snapshot earlier than T=0, stream starts from snapshot 1. { + std::map builder_options; ScanContextBuilder builder(table_path); - builder.AddOption(Options::SCAN_TIMESTAMP_MILLIS, "0").WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, FinishScanContext(builder)); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, TableScan::Create(std::move(ctx))); + builder_options[Options::SCAN_TIMESTAMP_MILLIS] = "0"; + builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, + FinishScanContext(builder, builder_options)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, + TableScan::Create(std::move(ctx), builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan0, scan->CreatePlan()); ASSERT_EQ(plan0->SnapshotId(), std::nullopt); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan1, scan->CreatePlan()); @@ -2411,11 +2586,14 @@ TEST_P(ScanInteTest, TestWithAppendTimestampMillisStreamScan) { } // T=snap2.time+1: EarlierThan(snap2.time+1)=snap2, stream starts from snapshot 3. { + std::map builder_options; ScanContextBuilder builder(table_path); - builder.AddOption(Options::SCAN_TIMESTAMP_MILLIS, std::to_string(snap2.TimeMillis() + 1)) - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, FinishScanContext(builder)); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, TableScan::Create(std::move(ctx))); + builder_options[Options::SCAN_TIMESTAMP_MILLIS] = std::to_string(snap2.TimeMillis() + 1); + builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, + FinishScanContext(builder, builder_options)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, + TableScan::Create(std::move(ctx), builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan0, scan->CreatePlan()); ASSERT_EQ(plan0->SnapshotId(), std::nullopt); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan1, scan->CreatePlan()); @@ -2423,11 +2601,14 @@ TEST_P(ScanInteTest, TestWithAppendTimestampMillisStreamScan) { } // T=snap3.time: EarlierThan(snap3.time)=snap2, stream starts from snapshot 3. { + std::map builder_options; ScanContextBuilder builder(table_path); - builder.AddOption(Options::SCAN_TIMESTAMP_MILLIS, std::to_string(snap3.TimeMillis())) - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, FinishScanContext(builder)); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, TableScan::Create(std::move(ctx))); + builder_options[Options::SCAN_TIMESTAMP_MILLIS] = std::to_string(snap3.TimeMillis()); + builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, + FinishScanContext(builder, builder_options)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, + TableScan::Create(std::move(ctx), builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan0, scan->CreatePlan()); ASSERT_EQ(plan0->SnapshotId(), std::nullopt); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan1, scan->CreatePlan()); @@ -2435,13 +2616,15 @@ TEST_P(ScanInteTest, TestWithAppendTimestampMillisStreamScan) { } // T=INT64_MAX: EarlierThan(INT64_MAX)=snap5, stream starts from snapshot 6 (beyond latest). { + std::map builder_options; ScanContextBuilder builder(table_path); - builder - .AddOption(Options::SCAN_TIMESTAMP_MILLIS, - std::to_string(std::numeric_limits::max())) - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, FinishScanContext(builder)); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, TableScan::Create(std::move(ctx))); + builder_options[Options::SCAN_TIMESTAMP_MILLIS] = + std::to_string(std::numeric_limits::max()); + builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, + FinishScanContext(builder, builder_options)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, + TableScan::Create(std::move(ctx), builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan0, scan->CreatePlan()); ASSERT_EQ(plan0->SnapshotId(), std::nullopt); ASSERT_TRUE(plan0->Splits().empty()); diff --git a/test/inte/write_and_read_inte_test.cpp b/test/inte/write_and_read_inte_test.cpp index 441c6d81f..bd3d9fd1b 100644 --- a/test/inte/write_and_read_inte_test.cpp +++ b/test/inte/write_and_read_inte_test.cpp @@ -154,13 +154,16 @@ class WriteAndReadInteTest Status CompactAndCommit(const std::string& table_path, const std::map& options, int64_t commit_identifier) const { + std::map write_context_builder_options; WriteContextBuilder write_context_builder(table_path, "commit_user"); write_context_builder.WithTempDirectory(LookupTempDirectory()); + write_context_builder_options = options; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, + + write_context_builder.WithStreamingMode(true).Finish()); PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr write_context, - write_context_builder.SetOptions(options).WithStreamingMode(true).Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_store_write, - FileStoreWrite::Create(std::move(write_context))); + std::unique_ptr file_store_write, + FileStoreWrite::Create(std::move(write_context), write_context_builder_options)); PAIMON_RETURN_NOT_OK(file_store_write->Compact(/*partition=*/{}, /*bucket=*/0, /*full_compaction=*/true)); PAIMON_ASSIGN_OR_RAISE( @@ -170,11 +173,15 @@ class WriteAndReadInteTest if (compact_messages.empty()) { return Status::Invalid("expected compaction commit messages"); } + std::map commit_context_builder_options; CommitContextBuilder commit_context_builder(table_path, "commit_user"); + commit_context_builder_options = options; PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, - commit_context_builder.SetOptions(options).Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_store_commit, - FileStoreCommit::Create(std::move(commit_context))); + + commit_context_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr file_store_commit, + FileStoreCommit::Create(std::move(commit_context), commit_context_builder_options)); return file_store_commit->Commit(compact_messages, commit_identifier); } @@ -185,10 +192,13 @@ class WriteAndReadInteTest std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar"); PAIMON_ASSIGN_OR_RAISE(auto plan, InnerScan(options)); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - read_context_builder.SetOptions(options).SetReadFieldNames(read_fields); + read_context_builder_options = options; + read_context_builder.SetReadFieldNames(read_fields); PAIMON_ASSIGN_OR_RAISE(auto read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context), + read_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(plan->Splits())); PAIMON_ASSIGN_OR_RAISE(auto actual, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -208,10 +218,13 @@ class WriteAndReadInteTest auto c_schema = std::make_unique(); PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*read_schema, c_schema.get())); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); + read_context_builder_options = options; + read_context_builder.SetReadSchema(std::move(c_schema)); PAIMON_ASSIGN_OR_RAISE(auto read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context), + read_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(plan->Splits())); PAIMON_ASSIGN_OR_RAISE(auto actual, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -240,11 +253,14 @@ class WriteAndReadInteTest Result> InnerScan( const std::map& options) const { std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar"); + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.SetOptions(options).AddOption(Options::SCAN_MODE, - StartupMode::LatestFull().ToString()); + scan_context_builder_options = options; + scan_context_builder_options[Options::SCAN_MODE] = StartupMode::LatestFull().ToString(); + PAIMON_ASSIGN_OR_RAISE(auto scan_context, scan_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context), + scan_context_builder_options)); return table_scan->CreatePlan(); } @@ -442,16 +458,18 @@ TEST_P(WriteAndReadInteTest, TestAppendReadWithNestedPredicateAcrossBatches) { SCOPED_TRACE(batch_size); for (bool enable_filter : {false, true}) { SCOPED_TRACE(enable_filter); + std::map builder_options; ReadContextBuilder builder(table_path); - builder.SetOptions(options) - .SetReadFieldNames({"payload", "value", "id", "key"}) + builder_options = options; + builder_options[Options::READ_BATCH_SIZE] = std::to_string(batch_size); + builder.SetReadFieldNames({"payload", "value", "id", "key"}) .SetPredicate(predicate) .EnablePredicateFilter(enable_filter) .EnablePrefetch(false) - .EnableLateMaterializing(false) - .AddOption(Options::READ_BATCH_SIZE, std::to_string(batch_size)); + .EnableLateMaterializing(false); ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(context), builder_options)); ASSERT_OK_AND_ASSIGN(auto reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(reader.get())); ASSERT_TRUE(actual); @@ -617,19 +635,24 @@ TEST_P(WriteAndReadInteTest, TestAppendVectorWithPredicate) { auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/0, /*field_name=*/"id", FieldType::INT, Literal(2)); + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.SetOptions(options) - .AddOption(Options::SCAN_MODE, StartupMode::LatestFull().ToString()) - .SetPredicate(predicate); + scan_context_builder_options = options; + scan_context_builder_options[Options::SCAN_MODE] = StartupMode::LatestFull().ToString(); + scan_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_FALSE(result_plan->Splits().empty()); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - read_context_builder.SetOptions(options).SetPredicate(predicate); + read_context_builder_options = options; + read_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -699,21 +722,26 @@ TEST_P(WriteAndReadInteTest, TestAppendWithExternalBitmapAndRangeBitmapIndexes) ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({name_predicate, score_predicate})); std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar"); + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.SetOptions(options) - .AddOption(Options::SCAN_MODE, StartupMode::LatestFull().ToString()) - .SetPredicate(predicate); + scan_context_builder_options = options; + scan_context_builder_options[Options::SCAN_MODE] = StartupMode::LatestFull().ToString(); + scan_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto plan, table_scan->CreatePlan()); ASSERT_EQ(1, plan->Splits().size()); // Keep precise post-read filtering disabled. The exact result therefore verifies that the // bitmap and range-bitmap indexes produced by the write path are consumed by the read path. + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - read_context_builder.SetOptions(options).SetPredicate(predicate); + read_context_builder_options = options; + read_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -811,12 +839,12 @@ TEST_P(WriteAndReadInteTest, TestLuminaSearchAfterRenameAcrossSchemaVersions) { auto search = std::make_shared("vec", 3, std::vector{0.0f, 0.0f}, nullptr, nullptr, VectorSearch::DistanceType::EUCLIDEAN, std::map{}); + std::map builder_options; ReadContextBuilder builder(path); - builder.SetOptions(current_options) - .SetReadFieldNames({"id", "_INDEX_SCORE"}) - .SetVectorSearch(search); + builder_options = current_options; + builder.SetReadFieldNames({"id", "_INDEX_SCORE"}).SetVectorSearch(search); ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto read, TableRead::Create(std::move(context))); + ASSERT_OK_AND_ASSIGN(auto read, TableRead::Create(std::move(context), builder_options)); ASSERT_OK_AND_ASSIGN(auto reader, read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(std::move(reader))); auto expected_type = arrow::struct_({arrow::field("_VALUE_KIND", arrow::int8()), @@ -908,13 +936,13 @@ TEST_P(WriteAndReadInteTest, TestLuminaVectorIndexAfterCompaction) { auto search = std::make_shared( "embedding", limit, std::vector{0.0f, 0.0f}, nullptr, nullptr, VectorSearch::DistanceType::EUCLIDEAN, std::map{}); + std::map builder_options; ReadContextBuilder builder(path); - builder.SetOptions(options) - .SetReadFieldNames({"id", "_INDEX_SCORE"}) - .SetVectorSearch(search); + builder_options = options; + builder.SetReadFieldNames({"id", "_INDEX_SCORE"}).SetVectorSearch(search); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, - TableRead::Create(std::move(context))); + TableRead::Create(std::move(context), builder_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, read->CreateReader(plan->Splits())); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr actual, @@ -1033,7 +1061,7 @@ TEST_P(WriteAndReadInteTest, TestAppendWithLuminaVectorIndex) { builder.SetReadFieldNames({"id", "_INDEX_SCORE"}).SetVectorSearch(search); ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr read, - TableRead::Create(std::move(context))); + TableRead::Create(std::move(context), {})); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, ReadResultCollector::CollectResult(std::move(reader))); @@ -3290,11 +3318,13 @@ TEST_P(WriteAndReadInteTest, TestPKWithParquetPageIndexFilter) { /*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, Literal(FieldType::STRING, literal_str.data(), literal_str.size())); + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_MODE, StartupMode::LatestFull().ToString()) - .SetPredicate(predicate); + scan_context_builder_options[Options::SCAN_MODE] = StartupMode::LatestFull().ToString(); + scan_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 2); ASSERT_FALSE(result_plan->Splits().empty()); @@ -3306,7 +3336,7 @@ TEST_P(WriteAndReadInteTest, TestPKWithParquetPageIndexFilter) { ReadContextBuilder read_context_builder(table_path); read_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context), {})); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -3400,11 +3430,13 @@ TEST_P(WriteAndReadInteTest, TestAppendWithParquetPageIndexFilter) { /*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, Literal(FieldType::STRING, literal_str.data(), literal_str.size())); + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_MODE, StartupMode::LatestFull().ToString()) - .SetPredicate(predicate); + scan_context_builder_options[Options::SCAN_MODE] = StartupMode::LatestFull().ToString(); + scan_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 2); ASSERT_FALSE(result_plan->Splits().empty()); @@ -3415,7 +3447,7 @@ TEST_P(WriteAndReadInteTest, TestAppendWithParquetPageIndexFilter) { ReadContextBuilder read_context_builder(table_path); read_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context), {})); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -3486,12 +3518,14 @@ TEST_P(WriteAndReadInteTest, TestAppendWithParquetPageIndexFilterAndPrefetch) { {Literal(7), Literal(11), Literal(15)}); ASSERT_TRUE(predicate); + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.SetOptions(options) - .AddOption(Options::SCAN_MODE, StartupMode::LatestFull().ToString()) - .SetPredicate(predicate); + scan_context_builder_options = options; + scan_context_builder_options[Options::SCAN_MODE] = StartupMode::LatestFull().ToString(); + scan_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_FALSE(result_plan->Splits().empty()); @@ -3500,15 +3534,17 @@ TEST_P(WriteAndReadInteTest, TestAppendWithParquetPageIndexFilterAndPrefetch) { // therefore finds no data for it and skips ahead into the next row group it owns, whose first // selected row sits in the middle of that row group. Row level filtering stays off: the // expected rows below are exactly what page-index filtering selects. + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - read_context_builder.SetOptions(options) - .SetPredicate(predicate) + read_context_builder_options = options; + read_context_builder_options["test.enable-adaptive-prefetch-strategy"] = "false"; + read_context_builder.SetPredicate(predicate) .EnablePrefetch(true) .SetPrefetchMaxParallelNum(2) - .SetPrefetchBatchCount(3) - .AddOption("test.enable-adaptive-prefetch-strategy", "false"); + .SetPrefetchBatchCount(3); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -3575,10 +3611,13 @@ TEST_P(WriteAndReadInteTest, TestAppendWithParquetMetadataCache) { auto cache = std::make_shared(CacheKind::DATA_FILE_FOOTER, 128 * 1024 * 1024); auto read_once = [&]() -> Result { + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_MODE, StartupMode::LatestFull().ToString()); + scan_context_builder_options[Options::SCAN_MODE] = StartupMode::LatestFull().ToString(); + PAIMON_ASSIGN_OR_RAISE(auto scan_context, scan_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context), + scan_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto result_plan, table_scan->CreatePlan()); if (result_plan->SnapshotId() != std::optional(1)) { return Status::Invalid("unexpected snapshot id"); @@ -3590,7 +3629,7 @@ TEST_P(WriteAndReadInteTest, TestAppendWithParquetMetadataCache) { ReadContextBuilder read_context_builder(table_path); read_context_builder.WithCache(cache); PAIMON_ASSIGN_OR_RAISE(auto read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context), {})); PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(result_plan->Splits())); PAIMON_ASSIGN_OR_RAISE(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -3897,10 +3936,15 @@ TEST_P(WriteAndReadInteTest, TestAppendMapSharedShreddingWithPredicate) { (void)helper; std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar"); + std::map write_context_builder_options; WriteContextBuilder write_context_builder(table_path, "commit_user_1"); + write_context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - write_context_builder.SetOptions(options).Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + + write_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto file_store_write, + FileStoreWrite::Create(std::move(write_context), write_context_builder_options)); auto write_one_row = [&](const std::string& data) -> Status { PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch, @@ -3917,27 +3961,36 @@ TEST_P(WriteAndReadInteTest, TestAppendMapSharedShreddingWithPredicate) { /*commit_identifier=*/0)); ASSERT_OK(file_store_write->Close()); + std::map commit_context_builder_options; CommitContextBuilder commit_context_builder(table_path, "commit_user_1"); + commit_context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - commit_context_builder.SetOptions(options).Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + + commit_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context), + commit_context_builder_options)); ASSERT_OK(commit->Commit(commit_msgs, /*commit_identifier=*/0)); auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/0, /*field_name=*/"id", FieldType::INT, Literal(10)); + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.SetOptions(options) - .AddOption(Options::SCAN_MODE, StartupMode::LatestFull().ToString()) - .SetPredicate(predicate); + scan_context_builder_options = options; + scan_context_builder_options[Options::SCAN_MODE] = StartupMode::LatestFull().ToString(); + scan_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_FALSE(result_plan->Splits().empty()); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - read_context_builder.SetOptions(options).SetPredicate(predicate); + read_context_builder_options = options; + read_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -4994,10 +5047,13 @@ TEST_P(WriteAndReadInteTest, TestMapSharedShreddingStructValueSchemaEvolutionRea auto read_fields = [&](const std::vector& field_names) -> Status { PAIMON_ASSIGN_OR_RAISE(auto plan, InnerScan(options)); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - read_context_builder.SetOptions(options).SetReadFieldNames(field_names); + read_context_builder_options = options; + read_context_builder.SetReadFieldNames(field_names); PAIMON_ASSIGN_OR_RAISE(auto read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context), + read_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(plan->Splits())); PAIMON_ASSIGN_OR_RAISE(auto actual, ReadResultCollector::CollectResult(std::move(batch_reader))); diff --git a/test/inte/write_and_read_inte_with_index_test.cpp b/test/inte/write_and_read_inte_with_index_test.cpp index efda3de03..aa0abb0d7 100644 --- a/test/inte/write_and_read_inte_with_index_test.cpp +++ b/test/inte/write_and_read_inte_with_index_test.cpp @@ -133,16 +133,17 @@ class WriteAndReadInteWithIndexTest const std::vector> splits, const std::shared_ptr& predicate, const std::shared_ptr& expected_array) const { + std::map context_builder_options; ReadContextBuilder context_builder(table_path); - context_builder.AddOption("read.batch-size", "2") - .AddOption("test.enable-adaptive-prefetch-strategy", "false") - .SetPredicate(predicate) - .EnableLateMaterializing(false); + context_builder_options["read.batch-size"] = "2"; + context_builder_options["test.enable-adaptive-prefetch-strategy"] = "false"; + context_builder.SetPredicate(predicate).EnableLateMaterializing(false); if (enable_prefetch_) { context_builder.EnablePrefetch(true).SetPrefetchBatchCount(3); } ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto result_array, @@ -1176,16 +1177,18 @@ TEST_P(WriteAndReadInteWithIndexTest, TestSimple) { &expected_array); ASSERT_TRUE(array_status.ok()); + std::map context_builder_options; ReadContextBuilder context_builder(path); - context_builder.AddOption("read.batch-size", "2") - .AddOption("test.enable-adaptive-prefetch-strategy", "false") - .AddOption("orc.read.enable-metrics", "true") - .SetPredicate(predicate); + context_builder_options["read.batch-size"] = "2"; + context_builder_options["test.enable-adaptive-prefetch-strategy"] = "false"; + context_builder_options["orc.read.enable-metrics"] = "true"; + context_builder.SetPredicate(predicate); if (enable_prefetch_) { context_builder.EnablePrefetch(true).SetPrefetchBatchCount(3); } ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(split)); ASSERT_OK_AND_ASSIGN(auto result_array, ReadResultCollector::CollectResult(batch_reader.get())); std::shared_ptr read_metrics = batch_reader->GetReaderMetrics(); @@ -1249,17 +1252,19 @@ TEST_P(WriteAndReadInteWithIndexTest, TestReadWithLimits) { auto predicate = PredicateBuilder::NotEqual(/*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, Literal(FieldType::STRING, "Alice", 5)); + std::map context_builder_options; ReadContextBuilder context_builder(path); - context_builder.AddOption(Options::FILE_FORMAT, "orc") - .AddOption("test.enable-adaptive-prefetch-strategy", "false") - .AddOption(Options::READ_BATCH_SIZE, "1") - .AddOption("orc.read.enable-metrics", "true") - .SetPredicate(predicate); + context_builder_options[Options::FILE_FORMAT] = "orc"; + context_builder_options["test.enable-adaptive-prefetch-strategy"] = "false"; + context_builder_options[Options::READ_BATCH_SIZE] = "1"; + context_builder_options["orc.read.enable-metrics"] = "true"; + context_builder.SetPredicate(predicate); if (enable_prefetch_) { context_builder.EnablePrefetch(true).SetPrefetchBatchCount(3); } ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(split)); // simulate read limits, only read 3 batches std::vector batches; @@ -1533,17 +1538,18 @@ TEST_P(WriteAndReadInteWithIndexTest, TestBitmapIndexWithLateMaterializing) { /*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, Literal(FieldType::STRING, literal_str.data(), literal_str.size())); + std::map context_builder_options; ReadContextBuilder context_builder(path); - context_builder.AddOption("read.batch-size", "2") - .AddOption("test.enable-adaptive-prefetch-strategy", "false") - .SetPredicate(predicate) - .EnablePredicateFilter(true) - .EnableLateMaterializing(true); + context_builder_options["read.batch-size"] = "2"; + context_builder_options["test.enable-adaptive-prefetch-strategy"] = "false"; + context_builder.SetPredicate(predicate).EnablePredicateFilter(true).EnableLateMaterializing( + true); if (enable_prefetch_) { context_builder.EnablePrefetch(true).SetPrefetchBatchCount(3); } ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(std::vector>{split})); ASSERT_OK_AND_ASSIGN(auto result_array, @@ -2934,15 +2940,17 @@ TEST_P(WriteAndReadInteWithIndexTest, TestWithIOException) { for (size_t i = 0; i < 200; i++) { ScopeGuard guard([&io_hook]() { io_hook->Clear(); }); io_hook->Reset(i, IOHook::Mode::RETURN_ERROR); + std::map context_builder_options; ReadContextBuilder context_builder(path); - context_builder.AddOption("read.batch-size", "2") - .AddOption("test.enable-adaptive-prefetch-strategy", "false") - .SetPredicate(predicate); + context_builder_options["read.batch-size"] = "2"; + context_builder_options["test.enable-adaptive-prefetch-strategy"] = "false"; + context_builder.SetPredicate(predicate); if (enable_prefetch_) { context_builder.EnablePrefetch(true).SetPrefetchBatchCount(3); } ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - Result> table_read = TableRead::Create(std::move(read_context)); + Result> table_read = + TableRead::Create(std::move(read_context), context_builder_options); CHECK_HOOK_STATUS(table_read.status(), i); Result> batch_reader = table_read.value()->CreateReader(split); CHECK_HOOK_STATUS(batch_reader.status(), i); diff --git a/test/inte/write_inte_test.cpp b/test/inte/write_inte_test.cpp index 31a325ec1..f21ac104d 100644 --- a/test/inte/write_inte_test.cpp +++ b/test/inte/write_inte_test.cpp @@ -289,11 +289,14 @@ class WriteInteTest : public testing::Test, public ::testing::WithParamInterface const std::vector>& data_splits, const std::string& blob_field, const std::vector>& expected_blobs) const { + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - read_context_builder.SetOptions(options).SetReadFieldNames({blob_field}); + read_context_builder_options = options; + read_context_builder.SetReadFieldNames({blob_field}); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context), + read_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(data_splits)); PAIMON_ASSIGN_OR_RAISE(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -353,7 +356,7 @@ class WriteInteTest : public testing::Test, public ::testing::WithParamInterface commit_builder.IgnoreEmptyCommit(ignore_empty_commit); PAIMON_ASSIGN_OR_RAISE(auto commit_context, commit_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(auto file_store_commit, - FileStoreCommit::Create(std::move(commit_context))); + FileStoreCommit::Create(std::move(commit_context), {})); return file_store_commit->Commit(commit_messages, commit_identifier); } @@ -2015,11 +2018,14 @@ TEST_P(WriteInteTest, TestPkTableWriteWithIOException) { ASSERT_EQ(3, batches_2.size()); // write data + std::map context_builder_options; WriteContextBuilder context_builder(root_path, "commit_user_1"); + context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.SetOptions(options).WithStreamingMode(true).Finish()); + + context_builder.WithStreamingMode(true).Finish()); Result> write = - FileStoreWrite::Create(std::move(write_context)); + FileStoreWrite::Create(std::move(write_context), context_builder_options); CHECK_HOOK_STATUS(write.status(), i); auto& file_store_write = write.value(); // round 1 @@ -2451,12 +2457,15 @@ TEST_P(WriteInteTest, TestWriteAndCommitIOException) { for (size_t i = 0; i < 200; i++) { ScopeGuard guard([&io_hook]() { io_hook->Clear(); }); io_hook->Reset(i, IOHook::Mode::RETURN_ERROR); + std::map context_builder_options; WriteContextBuilder context_builder(table_path, "commit_user_1"); + context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.SetOptions(options).WithStreamingMode(true).Finish()); + + context_builder.WithStreamingMode(true).Finish()); Result> write = - FileStoreWrite::Create(std::move(write_context)); + FileStoreWrite::Create(std::move(write_context), context_builder_options); CHECK_HOOK_STATUS(write.status(), i); auto& file_store_write = write.value(); @@ -2476,14 +2485,15 @@ TEST_P(WriteInteTest, TestWriteAndCommitIOException) { for (size_t i = 0; i < 400; i += paimon::test::RandomNumber(5, 15)) { ScopeGuard guard([&io_hook]() { io_hook->Clear(); }); io_hook->Reset(i, IOHook::Mode::RETURN_ERROR); + std::map commit_context_builder_options; CommitContextBuilder commit_context_builder(table_path, "commit_user_1"); - ASSERT_OK_AND_ASSIGN( - std::unique_ptr commit_context, - commit_context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .IgnoreEmptyCommit(false) - .Finish()); - auto commit = FileStoreCommit::Create(std::move(commit_context)); + commit_context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + commit_context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + + commit_context_builder.IgnoreEmptyCommit(false).Finish()); + auto commit = + FileStoreCommit::Create(std::move(commit_context), commit_context_builder_options); CHECK_HOOK_STATUS(commit.status(), i); Result committed = commit.value()->FilterAndCommit({{0, commit_messages}}); CHECK_HOOK_STATUS(committed.status(), i); @@ -2545,12 +2555,15 @@ TEST_P(WriteInteTest, TestWriteWithFieldId) { /*partition_keys=*/{}, /*primary_keys=*/{}, options)); + std::map context_builder_options; WriteContextBuilder context_builder(table_path, "commit_user_1"); + context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.SetOptions(options).Finish()); + + context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(std::move(write_context), context_builder_options)); ::ArrowArray c_array; ASSERT_TRUE(arrow::ExportArray(*src_array, &c_array).ok()); RecordBatchBuilder batch_builder(&c_array); @@ -2615,12 +2628,15 @@ TEST_P(WriteInteTest, TestAppendTableWriteAndReadWithExternalPath) { /*table_name=*/"append_table_with_external_path", &schema, /*partition_keys=*/{"f1"}, /*primary_keys=*/{}, options)); // write + std::map context_builder_options; WriteContextBuilder context_builder(table_path, "commit_user_1"); + context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.SetOptions(options).Finish()); + + context_builder.Finish()); std::string root_path = write_context->GetRootPath(); ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(std::move(write_context), context_builder_options)); std::shared_ptr array = arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ @@ -2652,15 +2668,21 @@ TEST_P(WriteInteTest, TestAppendTableWriteAndReadWithExternalPath) { ASSERT_TRUE(file_exist); // check read result + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::FILE_FORMAT, file_format); + scan_context_builder_options[Options::FILE_FORMAT] = file_format; + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - read_context_builder.AddOption(Options::FILE_FORMAT, file_format); + read_context_builder_options[Options::FILE_FORMAT] = file_format; + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); @@ -2712,12 +2734,15 @@ TEST_P(WriteInteTest, TestPKTableWriteAndReadWithExternalPath) { /*table_name=*/"pk_table_with_external_path", &schema, /*partition_keys=*/{"f1"}, /*primary_keys=*/{"f0", "f1"}, options)); // write + std::map context_builder_options; WriteContextBuilder context_builder(table_path, "commit_user_1"); + context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.SetOptions(options).Finish()); + + context_builder.Finish()); std::string root_path = write_context->GetRootPath(); ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(std::move(write_context), context_builder_options)); std::shared_ptr array = arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ @@ -2932,12 +2957,15 @@ TEST_P(WriteInteTest, TestWriteAndReadWithSpecialPartitionValue) { /*primary_keys=*/{}, options)); // write + std::map context_builder_options; WriteContextBuilder context_builder(table_path, "commit_user_1"); + context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.SetOptions(options).Finish()); + + context_builder.Finish()); std::string root_path = write_context->GetRootPath(); ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(std::move(write_context), context_builder_options)); auto write = [&](const std::string& json_array, const std::map& partition) -> void { std::shared_ptr array = @@ -2973,15 +3001,21 @@ TEST_P(WriteInteTest, TestWriteAndReadWithSpecialPartitionValue) { arrow::field("_VALUE_KIND", arrow::int8())); auto data_type = arrow::struct_(fields_with_row_kind); { + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::FILE_FORMAT, file_format); + scan_context_builder_options[Options::FILE_FORMAT] = file_format; + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context), + scan_context_builder_options)); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - read_context_builder.AddOption(Options::FILE_FORMAT, file_format); + read_context_builder_options[Options::FILE_FORMAT] = file_format; + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context), + read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); @@ -3020,18 +3054,24 @@ TEST_P(WriteInteTest, TestWriteAndReadWithSpecialPartitionValue) { ASSERT_TRUE(expected->Equals(read_result)) << read_result->ToString(); } { + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::FILE_FORMAT, file_format); + scan_context_builder_options[Options::FILE_FORMAT] = file_format; + std::vector> partition_filters = { {{"dt", " "}, {"hr", "a=b?"}}}; ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.SetPartitionFilter(partition_filters).Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context), + scan_context_builder_options)); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - read_context_builder.AddOption(Options::FILE_FORMAT, file_format); + read_context_builder_options[Options::FILE_FORMAT] = file_format; + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context), + read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); @@ -3047,18 +3087,24 @@ TEST_P(WriteInteTest, TestWriteAndReadWithSpecialPartitionValue) { ASSERT_TRUE(expected->Equals(read_result)) << read_result->ToString(); } { + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::FILE_FORMAT, file_format); + scan_context_builder_options[Options::FILE_FORMAT] = file_format; + std::vector> partition_filters = { {{"dt", ""}, {"hr", "a=b?"}}}; ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.SetPartitionFilter(partition_filters).Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context), + scan_context_builder_options)); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - read_context_builder.AddOption(Options::FILE_FORMAT, file_format); + read_context_builder_options[Options::FILE_FORMAT] = file_format; + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context), + read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); @@ -3074,18 +3120,24 @@ TEST_P(WriteInteTest, TestWriteAndReadWithSpecialPartitionValue) { ASSERT_TRUE(expected->Equals(read_result)) << read_result->ToString(); } { + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::FILE_FORMAT, file_format); + scan_context_builder_options[Options::FILE_FORMAT] = file_format; + std::vector> partition_filters = { {{"dt", "__DEFAULT_PARTITION__"}, {"hr", "a=b?"}}}; ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.SetPartitionFilter(partition_filters).Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context), + scan_context_builder_options)); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - read_context_builder.AddOption(Options::FILE_FORMAT, file_format); + read_context_builder_options[Options::FILE_FORMAT] = file_format; + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context), + read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); @@ -3124,12 +3176,15 @@ TEST_P(WriteInteTest, TestWriteWithNestedSchema) { /*table_name=*/"bar", &schema, /*partition_keys=*/{}, /*primary_keys=*/{}, options)); // write + std::map context_builder_options; WriteContextBuilder context_builder(table_path, "commit_user_1"); + context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.SetOptions(options).Finish()); + + context_builder.Finish()); std::string root_path = write_context->GetRootPath(); ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(std::move(write_context), context_builder_options)); std::shared_ptr array = arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ @@ -3155,15 +3210,21 @@ TEST_P(WriteInteTest, TestWriteWithNestedSchema) { /*commit_identifier=*/1)); // check read result + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::FILE_FORMAT, file_format); + scan_context_builder_options[Options::FILE_FORMAT] = file_format; + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - read_context_builder.AddOption(Options::FILE_FORMAT, file_format); + read_context_builder_options[Options::FILE_FORMAT] = file_format; + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); @@ -3220,12 +3281,15 @@ TEST_P(WriteInteTest, TestWriteWithIOException) { ScopeGuard guard([&io_hook]() { io_hook->Clear(); }); io_hook->Reset(i, IOHook::Mode::RETURN_ERROR); + std::map context_builder_options; WriteContextBuilder context_builder(table_path, "commit_user_1"); + context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.SetOptions(options).WithStreamingMode(true).Finish()); + + context_builder.WithStreamingMode(true).Finish()); Result> write = - FileStoreWrite::Create(std::move(write_context)); + FileStoreWrite::Create(std::move(write_context), context_builder_options); CHECK_HOOK_STATUS(write.status(), i); auto& file_store_write = write.value(); @@ -3273,11 +3337,14 @@ TEST_P(WriteInteTest, TestCommitWithIOException) { /*partition_keys=*/{"f1"}, /*primary_keys=*/{}, options)); // write data + std::map context_builder_options; WriteContextBuilder context_builder(table_path, "commit_user_1"); + context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.SetOptions(options).WithStreamingMode(true).Finish()); + + context_builder.WithStreamingMode(true).Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(std::move(write_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_0, MakeRecordBatch({{"Alice", 10, 1, 11.1}}, {{"f1", "10"}}, 0)); ASSERT_OK(file_store_write->Write(std::move(batch_0))); @@ -3293,14 +3360,15 @@ TEST_P(WriteInteTest, TestCommitWithIOException) { std::string tmp_table_path = tmp_dir->Str(); ScopeGuard guard([&io_hook]() { io_hook->Clear(); }); io_hook->Reset(i, IOHook::Mode::RETURN_ERROR); + std::map commit_context_builder_options; CommitContextBuilder commit_context_builder(tmp_table_path, "commit_user_1"); - ASSERT_OK_AND_ASSIGN( - std::unique_ptr commit_context, - commit_context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .IgnoreEmptyCommit(false) - .Finish()); - auto commit = FileStoreCommit::Create(std::move(commit_context)); + commit_context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + commit_context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + + commit_context_builder.IgnoreEmptyCommit(false).Finish()); + auto commit = + FileStoreCommit::Create(std::move(commit_context), commit_context_builder_options); CHECK_HOOK_STATUS(commit.status(), i); Result committed = commit.value()->FilterAndCommit({{0, commit_messages}}); CHECK_HOOK_STATUS(committed.status(), i); @@ -3348,14 +3416,16 @@ TEST_P(WriteInteTest, TestWriteMemoryUse) { /*primary_keys=*/{}, options)); std::shared_ptr write_pool = GetMemoryPool(); { + std::map context_builder_options; WriteContextBuilder context_builder(table_path, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.SetOptions(options) - .WithMemoryPool(write_pool) - .WithStreamingMode(true) - .Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(std::move(write_context))); + context_builder_options = options; + ASSERT_OK_AND_ASSIGN( + std::unique_ptr write_context, + + context_builder.WithMemoryPool(write_pool).WithStreamingMode(true).Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr file_store_write, + FileStoreWrite::Create(std::move(write_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_0, MakeRecordBatch({{"Alice", 10, 1, 11.1}}, {{"f1", "10"}}, 0)); ASSERT_OK(file_store_write->Write(std::move(batch_0))); @@ -3532,11 +3602,15 @@ TEST_F(WriteInteTest, TestBranchWrite) { std::map options = {{Options::FILE_FORMAT, "orc"}, {Options::FILE_SYSTEM, "local"}}; + std::map context_builder_options; WriteContextBuilder context_builder(table_path, "commit_user_1"); + context_builder_options = options; + context_builder_options[Options::BRANCH] = "rt"; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.SetOptions(options).WithBranch("rt").Finish()); + + context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(std::move(write_context), context_builder_options)); // write to rt branch, pk table std::string data_0 = @@ -3577,11 +3651,15 @@ TEST_F(WriteInteTest, TestBranchWrite) { ASSERT_TRUE(expected_commit_message_0->TEST_Equal(*result_commit_msgs)); // write to main branch, append table + std::map context_builder1_options; WriteContextBuilder context_builder1(table_path, "commit_user_1"); + context_builder1_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context1, - context_builder1.SetOptions(options).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write1, - FileStoreWrite::Create(std::move(write_context1))); + + context_builder1.Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr file_store_write1, + FileStoreWrite::Create(std::move(write_context1), context_builder1_options)); std::string data_1 = R"([["20240725", "watermelon", 3], ["20240725", "orange", 2] @@ -3719,7 +3797,7 @@ TEST_P(WriteInteTest, TestDataEvolutionWrite) { builder.WithWriteSchema(write_cols); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, builder.Finish()); PAIMON_ASSIGN_OR_RAISE(auto file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(std::move(write_context), {})); ArrowArray c_array; PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*write_array, &c_array)); auto record_batch = std::make_unique( @@ -3752,7 +3830,7 @@ TEST_P(WriteInteTest, TestDataEvolutionWrite) { CommitContextBuilder builder(table_path, commit_user); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_commit, - FileStoreCommit::Create(std::move(commit_context))); + FileStoreCommit::Create(std::move(commit_context), {})); ASSERT_OK(file_store_commit->Commit(commit_msgs)); auto commit_impl = dynamic_cast(file_store_commit.get()); @@ -3768,7 +3846,7 @@ TEST_P(WriteInteTest, TestDataEvolutionWrite) { ScanContextBuilder builder(table_path); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + TableScan::Create(std::move(scan_context), {})); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, table_scan->CreatePlan()); ASSERT_EQ(plan->Splits().size(), 1); auto data_split = std::dynamic_pointer_cast(plan->Splits()[0]); @@ -4144,13 +4222,14 @@ TEST_P(WriteInteTest, TestPkSpillableMapSharedShreddingReadWrite) { /*primary_keys=*/{"id"}, options)); std::string tmp_dir = PathUtil::JoinPath(dir->Str(), "tmp"); + std::map write_builder_options; WriteContextBuilder write_builder(table_path, "commit_user_1"); + write_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - write_builder.SetOptions(options) - .WithStreamingMode(true) - .WithTempDirectory(tmp_dir) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + + write_builder.WithStreamingMode(true).WithTempDirectory(tmp_dir).Finish()); + ASSERT_OK_AND_ASSIGN(auto file_store_write, + FileStoreWrite::Create(std::move(write_context), write_builder_options)); ASSERT_OK_AND_ASSIGN( auto batch_0, TestHelper::MakeRecordBatch( @@ -4213,7 +4292,8 @@ TEST_P(WriteInteTest, TestPkSpillableDiskQuotaExhaustedFallsBackToFlush) { WriteContextBuilder write_builder(table_path, "commit_user_1"); write_builder.WithStreamingMode(true).WithTempDirectory(tmp_dir); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN(auto file_store_write, + FileStoreWrite::Create(std::move(write_context), {})); auto write_array = arrow::ipc::internal::json::ArrayFromJSON(data_type, R"([ ["Alice", 10, 1, 1.0], @@ -4269,7 +4349,8 @@ TEST_P(WriteInteTest, TestPkSpillableGlobalMemoryPreemptionDataCorrectness) { WriteContextBuilder write_builder(table_path, "commit_user_1"); write_builder.WithStreamingMode(true).WithTempDirectory(tmp_dir); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN(auto file_store_write, + FileStoreWrite::Create(std::move(write_context), {})); // Write long strings to different partitions to trigger global memory preemption. std::string long_str_a(48, 'a'); @@ -4367,7 +4448,8 @@ TEST_P(WriteInteTest, TestPkSpillableTempFilesCleanedAfterPrepareCommit) { WriteContextBuilder write_builder(table_path, "commit_user_1"); write_builder.WithStreamingMode(true).WithTempDirectory(tmp_dir); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN(auto file_store_write, + FileStoreWrite::Create(std::move(write_context), {})); auto batch1 = arrow::ipc::internal::json::ArrayFromJSON(data_type, R"([["Alice", 10, 1]])").ValueOrDie(); @@ -4431,7 +4513,8 @@ TEST_P(WriteInteTest, TestPkSpillableIntermediateMergeWithTempFileTracking) { WriteContextBuilder write_builder(table_path, "commit_user_1"); write_builder.WithStreamingMode(true).WithTempDirectory(tmp_dir); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN(auto file_store_write, + FileStoreWrite::Create(std::move(write_context), {})); auto write_array_fn = [](FileStoreWrite* writer, const std::map& partition, int32_t bucket, @@ -4514,7 +4597,8 @@ TEST_P(WriteInteTest, TestPkSpillableVector) { WriteContextBuilder write_builder(table_path, "commit_user_1"); write_builder.WithStreamingMode(true).WithTempDirectory(tmp_dir); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN(auto file_store_write, + FileStoreWrite::Create(std::move(write_context), {})); auto write_array = [](FileStoreWrite* writer, const std::shared_ptr& array) { ArrowArray c_array; @@ -4599,7 +4683,8 @@ TEST_P(WriteInteTest, TestPkSpillableMultiBucketMultiRoundDataCorrectness) { .WithTempDirectory(tmp_dir) .SetWriteBufferSpillThreadNumber(3); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN(auto file_store_write, + FileStoreWrite::Create(std::move(write_context), {})); // Round 1: Bucket 0 writes Alice(dup) + Bob, Bucket 1 writes Dave(dup) + Eve auto r1_b0_batch1 = @@ -4787,14 +4872,16 @@ TEST_P(WriteInteTest, TestPkSpillableWithIOException) { // Write with spill enabled (WithTempDirectory) std::string tmp_dir = PathUtil::JoinPath(dir->Str(), "tmp"); + std::map context_builder_options; WriteContextBuilder context_builder(root_path, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.SetOptions(options) - .WithStreamingMode(true) - .WithTempDirectory(tmp_dir) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, - FileStoreWrite::Create(std::move(write_context))); + context_builder_options = options; + ASSERT_OK_AND_ASSIGN( + std::unique_ptr write_context, + + context_builder.WithStreamingMode(true).WithTempDirectory(tmp_dir).Finish()); + ASSERT_OK_AND_ASSIGN( + auto file_store_write, + FileStoreWrite::Create(std::move(write_context), context_builder_options)); // Start IOHook only for spill write/read operations. io_hook->Reset(i, IOHook::Mode::RETURN_ERROR); From c92201bfb2cf553f7be04aefc071a6aca1b47efc Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Wed, 4 Nov 2026 09:17:34 +0800 Subject: [PATCH 30/30] fix(table): bind contexts and correct branch time travel Expose the bound schema through GetSchema without a LatestSchema alias. Resolve catalog credentials only when loading tables and preserve bound storage settings for operators and dependency reads. Resolve snapshot, tag, and historical schema metadata on the effective target branch. Cover cross-branch time travel with schema and data regressions. --- docs/source/user_guide/catalog.rst | 34 +- include/paimon/catalog/table.h | 10 +- include/paimon/commit_context.h | 8 +- include/paimon/read_context.h | 15 +- include/paimon/scan_context.h | 15 +- include/paimon/table/format/format_table.h | 4 +- include/paimon/write_context.h | 9 +- src/paimon/core/catalog/catalog.cpp | 4 +- .../core/catalog/file_system_catalog.cpp | 12 +- src/paimon/core/catalog/file_system_catalog.h | 4 +- .../core/catalog/file_system_catalog_test.cpp | 7 +- src/paimon/core/operation/commit_context.cpp | 6 - .../core/operation/commit_context_test.cpp | 2 + .../core/operation/internal_read_context.cpp | 49 +-- .../core/operation/internal_read_context.h | 9 +- .../operation/internal_read_context_test.cpp | 44 +++ src/paimon/core/operation/read_context.cpp | 6 - .../core/operation/read_context_test.cpp | 12 +- src/paimon/core/operation/scan_context.cpp | 6 - .../core/operation/scan_context_test.cpp | 12 +- src/paimon/core/operation/write_context.cpp | 6 - .../core/operation/write_context_test.cpp | 10 +- src/paimon/core/table/file_store_table.cpp | 21 +- src/paimon/core/table/file_store_table.h | 3 +- .../core/table/file_store_table_factory.cpp | 15 +- src/paimon/core/table/format/format_table.cpp | 2 +- .../core/table/format/format_table_test.cpp | 15 +- .../table/system/audit_log_system_table.cpp | 26 +- .../system/read_optimized_system_table.cpp | 32 +- src/paimon/core/table/system/system_table.cpp | 21 +- src/paimon/core/table/system/system_table.h | 8 +- .../core/table/system/system_table_test.cpp | 38 +++ src/paimon/core/table/table_test.cpp | 306 +++++++++++++++--- src/paimon/core/utils/context_utils.h | 16 + src/paimon/rest/rest_catalog.cpp | 9 +- src/paimon/rest/rest_catalog_test.cpp | 22 +- src/paimon/testing/utils/test_helper.h | 2 +- test/inte/append_compaction_inte_test.cpp | 10 +- test/inte/pk_compaction_inte_test.cpp | 4 +- test/inte/write_inte_test.cpp | 14 +- 40 files changed, 567 insertions(+), 281 deletions(-) diff --git a/docs/source/user_guide/catalog.rst b/docs/source/user_guide/catalog.rst index 0b0ac6f20..aa8b5a336 100644 --- a/docs/source/user_guide/catalog.rst +++ b/docs/source/user_guide/catalog.rst @@ -33,6 +33,14 @@ Managed table copies retain the current schema and identifier when ``branch`` changes. ``FileStoreTable::SwitchToBranch`` loads the target branch's latest schema. Physical layout options such as ``merge-engine``, ``row-tracking.enabled`` and ``data-evolution.enabled`` cannot be changed by a copy. + +``table->GetSchema()`` returns the schema bound to that table instance without +metadata I/O. It does not refresh when the stored schema changes. Load a new +table through ``catalog->GetTable(identifier)`` to obtain the latest schema. +``Copy(options)`` derives a table from the current instance; time-travel options +may trigger snapshot/tag lookup and historical schema loading. These operations +can perform metadata I/O and fail, as indicated by the returned ``Result``. + Changing ``deletion-vectors.enabled`` requires the table's existing options to contain ``deletion-vectors.modifiable=true``. Set that option before copying with a different deletion-vectors mode. @@ -354,10 +362,9 @@ so its restored file references are visible to the expiration operation. Reading through the catalog ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -To read a native table with the per-table temporary credentials a catalog issues, -pass the catalog and table identifier to ``ReadContextBuilder::WithCatalog`` or -``ScanContextBuilder::WithCatalog``. This is a shorthand for asking the catalog -for that table's file system and passing it in through ``WithFileSystem``: +Load a table with ``Catalog::GetTable`` to bind its schema, location and file +system, including any per-table temporary credentials. ``WithCatalog`` supplies +catalog services for snapshot operations and dependency reads: .. code-block:: cpp @@ -377,15 +384,16 @@ for that table's file system and passing it in through ``WithFileSystem``: PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, table->NewRead(std::move(read_context))); -When ``Finish()`` builds the context, it asks the catalog for the table's file -system through ``Catalog::GetTableFileSystem`` and uses it as-is for the schema, -snapshots, manifests and data under the table path, so it signs every access with -the table's credentials and reloads them as they expire. An explicit -``WithFileSystem`` takes precedence, so the catalog is not asked. A catalog that -issues no per-table credentials returns its catalog-level file system, so this is -also how you read with the catalog's own object-store credentials. A format table -is read through the table's own ``NewRead`` or ``NewScan`` instead; see -:doc:`format_table`. +``Finish()`` builds a context without fetching a file system. Operators created +through ``Table::NewScan/NewRead/NewWrite/NewCommit`` use the table's bound path, +schema and file system. Contexts provide execution settings such as filters, +projections, caches and executors; write and commit identity checks still apply. + +The legacy static ``Create(context, options)`` factories load a native table from +the context's loading parameters. They resolve ``Catalog::GetTableFileSystem`` +at that point, unless an explicit ``WithFileSystem`` or scheme map takes +precedence. These loading parameters do not override an already loaded table. +Format tables also use their own ``NewRead`` and ``NewScan``; see :doc:`format_table`. The C++ REST catalog covers the database, table, snapshot and commit operations of the ``Catalog`` API. The parts of the Java REST catalog that have no C++ diff --git a/include/paimon/catalog/table.h b/include/paimon/catalog/table.h index 9bdd306ea..01a08e030 100644 --- a/include/paimon/catalog/table.h +++ b/include/paimon/catalog/table.h @@ -44,6 +44,9 @@ class WriteContext; /// Common interface for managed, format and system tables. /// Create operators with NewScan/NewRead/NewWrite/NewCommit; unsupported operations return errors. +/// Operators use the table's bound path, schema and file system. Context loading parameters apply +/// to static Create() factories; write and commit contexts must still identify this table. +/// Contexts supply execution parameters and optional catalog services for snapshot operations. class PAIMON_EXPORT Table { public: virtual ~Table() = default; @@ -76,13 +79,16 @@ class PAIMON_EXPORT Table { return uuid_.empty() ? std::nullopt : std::optional(uuid_); } - /// Loads the latest schema of table. - std::shared_ptr LatestSchema() const { + /// Returns the schema bound to this table instance without metadata I/O. + /// A time-travel copy exposes its historical schema. Reload through the catalog to obtain + /// the latest schema; this accessor does not refresh it. + std::shared_ptr GetSchema() const { return schema_; } /// Copies this table with dynamic_options overriding its current options. /// The table type is structural and cannot be changed by these options. + /// This may perform metadata I/O, including snapshot/tag lookup and schema loading. virtual Result> Copy( const std::map& dynamic_options) const = 0; diff --git a/include/paimon/commit_context.h b/include/paimon/commit_context.h index ee0fc517a..05c95ada5 100644 --- a/include/paimon/commit_context.h +++ b/include/paimon/commit_context.h @@ -161,7 +161,8 @@ class PAIMON_EXPORT CommitContextBuilder { /// @return Reference to this builder for method chaining. CommitContextBuilder& UseRESTCatalogCommit(bool use_rest_catalog_commit); - /// Loads the current schema from `catalog`. Catalogs with version management also load and + /// Static Create() loads the table from `catalog`; Table::NewCommit() uses its bound schema + /// and file system. Catalogs with version management also load and /// commit snapshots, rebasing on conflicts. Other catalogs use file-system snapshot commits. /// Mutually exclusive with `UseRESTCatalogCommit()`. /// @@ -203,9 +204,8 @@ class PAIMON_EXPORT CommitContextBuilder { /// @return Reference to this builder for method chaining. CommitContextBuilder& WithExecutor(const std::shared_ptr& executor); - /// Sets a custom file system instance to be used for all file operations in this commit - /// context. - /// This bypasses the global file system registry and uses the provided implementation directly. + /// Sets the file system for table loading through static Create(). + /// Table::NewCommit() uses the table's bound file system. /// /// @param file_system The file system to use. /// @return Reference to this builder for method chaining. diff --git a/include/paimon/read_context.h b/include/paimon/read_context.h index 7eb3aa17f..9de014eff 100644 --- a/include/paimon/read_context.h +++ b/include/paimon/read_context.h @@ -451,7 +451,7 @@ class PAIMON_EXPORT ReadContextBuilder { ReadContextBuilder& WithRealtimeContext( const std::shared_ptr& realtime_context); - /// Set the table schema as a string to avoid schema loading I/O operations. + /// Sets the table schema for static Create() to avoid schema loading I/O. /// /// This optimization allows the reader to use a pre-loaded schema instead of /// reading it from the table metadata, which can improve performance especially @@ -482,19 +482,18 @@ class PAIMON_EXPORT ReadContextBuilder { ReadContextBuilder& WithFileSystemSchemeToIdentifierMap( const std::map& fs_scheme_to_identifier_map); - /// Sets a custom file system instance to be used for all file operations in this read context. - /// This bypasses the global file system registry and uses the provided implementation directly. + /// Sets the file system for table loading through static Create(). + /// Table::NewRead() uses the table's bound file system. /// /// @param file_system The file system to use. /// @return Reference to this builder for method chaining. /// @note If not set, use default file system (configured in `Options::FILE_SYSTEM`) ReadContextBuilder& WithFileSystem(const std::shared_ptr& file_system); - /// Reads a native table through its own file system - including the per-table temporary - /// credentials a catalog that issues them hands out through `Catalog::GetTableFileSystem`. - /// This is a shorthand for `WithFileSystem(catalog->GetTableFileSystem(identifier))`; an - /// explicit `WithFileSystem()` takes precedence, so the catalog is not asked. - /// @param catalog Non-null catalog, read when `Finish()` builds the context. + /// Sets the catalog used for snapshot operations and table loading. + /// Static Create() resolves per-table credentials when loading the table; explicit file systems + /// and scheme maps take precedence. Table::NewRead() uses the table's bound file system. + /// @param catalog Non-null catalog, retained by the context. /// @param identifier The native table to read. /// @return Reference to this builder for method chaining. ReadContextBuilder& WithCatalog(const std::shared_ptr& catalog, diff --git a/include/paimon/scan_context.h b/include/paimon/scan_context.h index aacb40c86..b6bbafb8b 100644 --- a/include/paimon/scan_context.h +++ b/include/paimon/scan_context.h @@ -210,8 +210,8 @@ class PAIMON_EXPORT ScanContextBuilder { /// @return Reference to this builder for method chaining. /// @note if not set, executor is default executor ScanContextBuilder& WithExecutor(const std::shared_ptr& executor); - /// Sets a custom file system instance to be used for all file operations in this scan context. - /// This bypasses the global file system registry and uses the provided implementation directly. + /// Sets the file system for table loading through static Create(). + /// Table::NewScan() uses the table's bound file system. /// @param file_system The file system to use. /// @return Reference to this builder for method chaining. /// @note If not set, use default file system (configured in `Options::FILE_SYSTEM`) @@ -224,17 +224,16 @@ class PAIMON_EXPORT ScanContextBuilder { ScanContextBuilder& WithFileSystemSchemeToIdentifierMap( const std::map& fs_scheme_to_identifier_map); - /// Plans a native table through its own file system - including the per-table temporary - /// credentials a catalog that issues them hands out through `Catalog::GetTableFileSystem`. - /// This is a shorthand for `WithFileSystem(catalog->GetTableFileSystem(identifier))`; an - /// explicit `WithFileSystem()` takes precedence, so the catalog is not asked. - /// @param catalog Non-null catalog, read when `Finish()` builds the context. + /// Sets the catalog used for snapshot operations and table loading. + /// Static Create() resolves per-table credentials when loading the table; explicit file systems + /// and scheme maps take precedence. Table::NewScan() uses the table's bound file system. + /// @param catalog Non-null catalog, retained by the context. /// @param identifier The native table to scan. /// @return Reference to this builder for method chaining. ScanContextBuilder& WithCatalog(const std::shared_ptr& catalog, const Identifier& identifier); - /// Set the table schema as a string to avoid schema loading I/O operations. + /// Sets the table schema for static Create() to avoid schema loading I/O. /// /// This optimization allows the scanner to use a pre-loaded schema instead of /// reading it from the table metadata, which can improve performance especially diff --git a/include/paimon/table/format/format_table.h b/include/paimon/table/format/format_table.h index 3082eea6a..d10e355b2 100644 --- a/include/paimon/table/format/format_table.h +++ b/include/paimon/table/format/format_table.h @@ -144,8 +144,8 @@ class PAIMON_EXPORT FormatTable : public Table, public std::enable_shared_from_t return options_; } - /// Returns the data schema, including partition columns. - std::shared_ptr LatestSchema() const { + /// Returns the bound data schema, including partition columns, without metadata I/O. + std::shared_ptr GetSchema() const { return std::static_pointer_cast(schema_); } diff --git a/include/paimon/write_context.h b/include/paimon/write_context.h index 14717b284..75dd8662d 100644 --- a/include/paimon/write_context.h +++ b/include/paimon/write_context.h @@ -213,8 +213,8 @@ class PAIMON_EXPORT WriteContextBuilder { /// @return Reference to this builder for method chaining. WriteContextBuilder& WithWriteSchema(const std::vector& write_schema); - /// Sets a custom file system instance to be used for all file operations in this write context. - /// This bypasses the global file system registry and uses the provided implementation directly. + /// Sets the file system for table loading through static Create(). + /// Table::NewWrite() uses the table's bound file system. /// /// @param file_system The file system to use. /// @return Reference to this builder for method chaining. @@ -229,8 +229,9 @@ class PAIMON_EXPORT WriteContextBuilder { /// for that branch. Its schema is read from the branch directory rather than from the catalog, /// just as a read of that branch reads it. /// Data, manifests and historical metadata remain on the file system. The catalog supplies - /// that file system unless overridden by `WithFileSystem()` or - /// `WithFileSystemSchemeToIdentifierMap()`. + /// that file system when static Create() loads the table, unless overridden by + /// `WithFileSystem()` or `WithFileSystemSchemeToIdentifierMap()`. Table::NewWrite() uses + /// the table's bound schema and file system. /// @param catalog Non-null catalog, kept alive while the writer uses its snapshot loader. /// @param identifier The native table to write to. /// @return Reference to this builder for method chaining. diff --git a/src/paimon/core/catalog/catalog.cpp b/src/paimon/core/catalog/catalog.cpp index 717606612..5f2c5da34 100644 --- a/src/paimon/core/catalog/catalog.cpp +++ b/src/paimon/core/catalog/catalog.cpp @@ -68,8 +68,8 @@ Result> Catalog::Create( } PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(options, file_system, fs_scheme_to_identifier_map)); - return std::shared_ptr( - std::make_unique(core_options.GetFileSystem(), root_path, options)); + return std::shared_ptr(std::make_unique( + core_options.GetFileSystem(), root_path, options, fs_scheme_to_identifier_map)); } bool Catalog::SupportsVersionManagement() const { diff --git a/src/paimon/core/catalog/file_system_catalog.cpp b/src/paimon/core/catalog/file_system_catalog.cpp index 3be877c1e..50cc3c395 100644 --- a/src/paimon/core/catalog/file_system_catalog.cpp +++ b/src/paimon/core/catalog/file_system_catalog.cpp @@ -56,10 +56,12 @@ struct ArrowSchema; namespace paimon { FileSystemCatalog::FileSystemCatalog(const std::shared_ptr& fs, const std::string& warehouse, - const std::map& catalog_options) + const std::map& catalog_options, + const std::map& fs_scheme_map) : fs_(fs), warehouse_(warehouse), catalog_options_(catalog_options), + fs_scheme_map_(fs_scheme_map), logger_(Logger::GetLogger("FileSystemCatalog")) {} Status FileSystemCatalog::CreateDatabase(const std::string& db_name, @@ -294,7 +296,7 @@ Result> FileSystemCatalog::LoadTableSchema( PAIMON_ASSIGN_OR_RAISE( std::shared_ptr system_table, GlobalSystemTableLoader::LoadForSchema(identifier.GetObjectName(), context)); - return system_table->LatestSchema(); + return system_table->GetSchema(); } // The system table suffix is dropped when the data table identifier is rebuilt below, so the // identifier is validated as a whole here. @@ -303,7 +305,7 @@ Result> FileSystemCatalog::LoadTableSchema( if (is_system_table) { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, LoadSystemTable(identifier)); - return system_table->LatestSchema(); + return system_table->GetSchema(); } PAIMON_ASSIGN_OR_RAISE(std::optional> latest_schema, TableSchemaExists(identifier)); @@ -361,7 +363,7 @@ Result> FileSystemCatalog::GetTable(const Identifier& ide "FileSystemCatalog::GetTable")); PAIMON_ASSIGN_OR_RAISE(std::string branch, identifier.GetBranchNameOrDefault()); return FileStoreTable::Create(fs_, table_path, *latest_schema, identifier, /*uuid=*/"", branch, - /*fs_scheme_map=*/{}, /*dynamic_options=*/{}); + fs_scheme_map_, /*dynamic_options=*/{}); } Result> FileSystemCatalog::LoadSystemTable( @@ -387,7 +389,7 @@ Result> FileSystemCatalog::LoadSystemTable( PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(data_identifier)); return SystemTableLoader::Load(system_table_name.value(), fs_, table_path, latest_schema.value(), dynamic_options, - data_identifier.GetObjectName()); + data_identifier.GetObjectName(), fs_scheme_map_); } Status FileSystemCatalog::DropDatabase(const std::string& name, bool ignore_if_not_exists, diff --git a/src/paimon/core/catalog/file_system_catalog.h b/src/paimon/core/catalog/file_system_catalog.h index 47e6a179d..4c63add7f 100644 --- a/src/paimon/core/catalog/file_system_catalog.h +++ b/src/paimon/core/catalog/file_system_catalog.h @@ -42,7 +42,8 @@ class SystemTable; class FileSystemCatalog : public Catalog, public std::enable_shared_from_this { public: FileSystemCatalog(const std::shared_ptr& fs, const std::string& warehouse, - const std::map& catalog_options); + const std::map& catalog_options, + const std::map& fs_scheme_map = {}); Status CreateDatabase(const std::string& db_name, const std::map& options, @@ -107,6 +108,7 @@ class FileSystemCatalog : public Catalog, public std::enable_shared_from_this fs_; std::string warehouse_; std::map catalog_options_; + std::map fs_scheme_map_; std::shared_ptr logger_; }; diff --git a/src/paimon/core/catalog/file_system_catalog_test.cpp b/src/paimon/core/catalog/file_system_catalog_test.cpp index a4a9a3104..c1af1e721 100644 --- a/src/paimon/core/catalog/file_system_catalog_test.cpp +++ b/src/paimon/core/catalog/file_system_catalog_test.cpp @@ -240,8 +240,7 @@ TEST(FileSystemCatalogTest, TestOptionsSystemTableCatalog) { ASSERT_EQ("tbl1$options", system_table->Name()); ASSERT_EQ("tbl1$options", system_table->FullName()); ASSERT_EQ(system_table->GetRootPath(), table_path); - ASSERT_EQ((std::vector{"key", "value"}), - system_table->LatestSchema()->FieldNames()); + ASSERT_EQ((std::vector{"key", "value"}), system_table->GetSchema()->FieldNames()); ASSERT_NOK_WITH_MSG(catalog.LoadTableSchema(Identifier("db1", "tbl1$unknown")), "not exist"); ASSERT_NOK_WITH_MSG(catalog.LoadTableSchema(Identifier("db1", "missing$options")), "not exist"); @@ -621,7 +620,7 @@ TEST(FileSystemCatalogTest, TestBranchTableLoadsBranchSchema) { ASSERT_TRUE(exists); ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog.GetTable(branch_identifier)); - data_schema = std::dynamic_pointer_cast(table->LatestSchema()); + data_schema = std::dynamic_pointer_cast(table->GetSchema()); ASSERT_NE(nullptr, data_schema); ASSERT_EQ((std::vector{"b1_c0"}), data_schema->FieldNames()); ASSERT_EQ(3, data_schema->Id()); @@ -955,7 +954,7 @@ TEST(FileSystemCatalogTest, TestCreateTableWithBlob) { ASSERT_TRUE(typed_schema.Equals(loaded_schema)); ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog.GetTable(Identifier("db1", "tbl1"))); - ASSERT_OK_AND_ASSIGN(auto arrow_schema_from_get_table, table->LatestSchema()->GetArrowSchema()); + ASSERT_OK_AND_ASSIGN(auto arrow_schema_from_get_table, table->GetSchema()->GetArrowSchema()); auto schema_from_get_table = arrow::ImportSchema(arrow_schema_from_get_table.get()).ValueOrDie(); ASSERT_TRUE(typed_schema.Equals(schema_from_get_table)); diff --git a/src/paimon/core/operation/commit_context.cpp b/src/paimon/core/operation/commit_context.cpp index 459a95a4d..fac300109 100644 --- a/src/paimon/core/operation/commit_context.cpp +++ b/src/paimon/core/operation/commit_context.cpp @@ -193,12 +193,6 @@ Result> CommitContextBuilder::Finish() { "a commit either goes to the catalog of WithCatalog() or is handed back as a request " "for the caller to send, which is what UseRESTCatalogCommit() asks for"); } - if (impl_->catalog_ != nullptr && impl_->specific_file_system_ == nullptr && - impl_->fs_scheme_to_identifier_map_.empty()) { - // A scheme map takes precedence over catalog-issued per-table credentials. - PAIMON_ASSIGN_OR_RAISE(impl_->specific_file_system_, - impl_->catalog_->GetTableFileSystem(impl_->identifier_.value())); - } PAIMON_ASSIGN_OR_RAISE(impl_->root_path_, PathUtil::NormalizePath(impl_->root_path_)); if (impl_->root_path_.empty()) { return Status::Invalid("root path is empty"); diff --git a/src/paimon/core/operation/commit_context_test.cpp b/src/paimon/core/operation/commit_context_test.cpp index 8970db696..fcda83fbe 100644 --- a/src/paimon/core/operation/commit_context_test.cpp +++ b/src/paimon/core/operation/commit_context_test.cpp @@ -138,6 +138,8 @@ TEST(CommitContextTest, TestWithCatalog) { ASSERT_TRUE(ctx->GetIdentifier().has_value()); ASSERT_EQ(ctx->GetIdentifier().value(), Identifier("db1", "t1")); ASSERT_EQ(ctx->GetTableId(), std::optional("table-uuid")); + ASSERT_FALSE(ctx->GetSpecificFileSystem()); + ASSERT_TRUE(catalog->TableFileSystemRequests().empty()); CommitContextBuilder null_catalog_builder("table_root_path", "commit_user_1"); ASSERT_NOK_WITH_MSG(null_catalog_builder.WithCatalog(nullptr, Identifier("db1", "t1")).Finish(), diff --git a/src/paimon/core/operation/internal_read_context.cpp b/src/paimon/core/operation/internal_read_context.cpp index 1beb19d76..956dacff8 100644 --- a/src/paimon/core/operation/internal_read_context.cpp +++ b/src/paimon/core/operation/internal_read_context.cpp @@ -47,6 +47,7 @@ #include "paimon/core/schema/arrow_schema_validator.h" #include "paimon/core/schema/schema_manager.h" #include "paimon/core/utils/branch_manager.h" +#include "paimon/core/utils/context_utils.h" #include "paimon/core/utils/nested_projection_utils.h" #include "paimon/defs.h" #include "paimon/predicate/compound_predicate.h" @@ -280,12 +281,14 @@ std::optional InternalReadContext::TryResolveSpecialFieldByName( Result> InternalReadContext::CreateForTable( const std::shared_ptr& context, const std::shared_ptr& table_schema, CoreOptions core_options, const std::string& root_path, - const std::map& dynamic_options) { + const std::map& dynamic_options, + const std::map& fs_scheme_map) { core_options.WithCache(context->GetCache()); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr result, CreateFromCoreOptions(context, table_schema, std::move(core_options), root_path)); result->dynamic_options_ = dynamic_options; + result->fs_scheme_map_ = fs_scheme_map; return result; } @@ -305,10 +308,16 @@ Result> InternalReadContext::CreateFromOpti const std::shared_ptr& context, const std::shared_ptr& table_schema, const std::map& options, const std::shared_ptr& file_system, std::optional path) { + std::shared_ptr resolved_file_system = file_system; + if (resolved_file_system == nullptr) { + PAIMON_ASSIGN_OR_RAISE(resolved_file_system, + ContextUtils::ResolveFileSystemForLoad(*context)); + } PAIMON_ASSIGN_OR_RAISE( CoreOptions core_options, - CoreOptions::FromMap( - options, file_system != nullptr ? file_system : context->GetSpecificFileSystem())); + CoreOptions::FromMap(options, resolved_file_system, + file_system ? std::map{} + : context->GetFileSystemSchemeToIdentifierMap())); core_options.WithCache(context->GetCache()); return CreateFromCoreOptions(context, table_schema, std::move(core_options), std::move(path)); } @@ -316,23 +325,16 @@ Result> InternalReadContext::CreateFromOpti Result> InternalReadContext::CreateForFallbackBranch( const std::shared_ptr& context, const std::string& branch, const std::shared_ptr& file_system, const std::string& root_path, - const std::map& table_dynamic_options) { - std::shared_ptr table_schema; - const auto& specific_table_schema = context->GetSpecificTableSchema(); - if (branch == BranchManager::DEFAULT_MAIN_BRANCH && specific_table_schema) { - PAIMON_ASSIGN_OR_RAISE(table_schema, - TableSchema::CreateFromJson(specific_table_schema.value())); - } else { - auto schema_manager = std::make_shared(file_system, root_path, branch); - PAIMON_ASSIGN_OR_RAISE(std::optional> latest_schema, - schema_manager->Latest()); - if (!latest_schema) { - return Status::Invalid( - fmt::format("schema file not found in path {}, branch {}", root_path, branch)); - } - table_schema = latest_schema.value(); - } - assert(table_schema); + const std::map& table_dynamic_options, + const std::map& fs_scheme_map) { + SchemaManager schema_manager(file_system, root_path, branch); + PAIMON_ASSIGN_OR_RAISE(std::optional> latest_schema, + schema_manager.Latest()); + if (!latest_schema) { + return Status::Invalid( + fmt::format("schema file not found in path {}, branch {}", root_path, branch)); + } + std::shared_ptr table_schema = latest_schema.value(); auto options = table_schema->Options(); for (const auto& [key, value] : table_dynamic_options) { if (key != Options::TYPE) { @@ -348,6 +350,7 @@ Result> InternalReadContext::CreateForFallb std::unique_ptr result, CreateFromOptions(context, table_schema, options, file_system, root_path)); result->dynamic_options_ = table_dynamic_options; + result->fs_scheme_map_ = fs_scheme_map; return result; } @@ -552,6 +555,7 @@ InternalReadContext::InternalReadContext(const std::shared_ptr& rea read_schema_(read_schema), arrow_pool_(arrow_pool), options_(options), + fs_scheme_map_(read_context->GetFileSystemSchemeToIdentifierMap()), path_(std::move(path)) {} Result> InternalReadContext::CreateWithSchema( @@ -564,9 +568,12 @@ Result> InternalReadContext::CreateWithSche } // Create a new InternalReadContext sharing all properties except read_schema. // The new read_schema is the minimal column set for COUNT(*). - return std::shared_ptr( + auto result = std::shared_ptr( new InternalReadContext(original->read_context_, original->table_schema_, new_read_schema, original->options_, original->path_, original->arrow_pool_)); + result->dynamic_options_ = original->dynamic_options_; + result->fs_scheme_map_ = original->fs_scheme_map_; + return result; } } // namespace paimon diff --git a/src/paimon/core/operation/internal_read_context.h b/src/paimon/core/operation/internal_read_context.h index 8e05db14f..da6e75351 100644 --- a/src/paimon/core/operation/internal_read_context.h +++ b/src/paimon/core/operation/internal_read_context.h @@ -63,7 +63,8 @@ class InternalReadContext { static Result> CreateForTable( const std::shared_ptr& read_context, const std::shared_ptr& table_schema, CoreOptions core_options, - const std::string& root_path, const std::map& dynamic_options); + const std::string& root_path, const std::map& dynamic_options, + const std::map& fs_scheme_map); /// Builds an internal write/lookup read from effective options and the context's file system. static Result> Create( @@ -79,7 +80,8 @@ class InternalReadContext { static Result> CreateForFallbackBranch( const std::shared_ptr& context, const std::string& branch, const std::shared_ptr& file_system, const std::string& root_path, - const std::map& table_dynamic_options); + const std::map& table_dynamic_options, + const std::map& fs_scheme_map); /// Maps score output positions in the read schema to their logical search sources. const std::map& GetIndexScoreBindings() const { @@ -158,7 +160,7 @@ class InternalReadContext { /// The scheme map the file system of this read was resolved with, which a catalog reading a /// dependency table on its behalf needs to resolve that table's location the same way. const std::map& GetFileSystemSchemeToIdentifierMap() const { - return read_context_->GetFileSystemSchemeToIdentifierMap(); + return fs_scheme_map_; } bool ReadAheadCacheEnabled() const { @@ -217,6 +219,7 @@ class InternalReadContext { std::shared_ptr arrow_pool_; CoreOptions options_; std::map dynamic_options_; + std::map fs_scheme_map_; std::map index_score_bindings_; /// Optional table root path; unset for reads using the context's path. std::optional path_; diff --git a/src/paimon/core/operation/internal_read_context_test.cpp b/src/paimon/core/operation/internal_read_context_test.cpp index c4b6746f0..3e07ae54d 100644 --- a/src/paimon/core/operation/internal_read_context_test.cpp +++ b/src/paimon/core/operation/internal_read_context_test.cpp @@ -48,6 +48,50 @@ namespace paimon::test { +TEST(InternalReadContext, TestBoundReadIgnoresContextLoadingParameters) { + const std::string path = GetDataDir() + "/orc/append_09.db/append_09"; + auto file_system = std::make_shared(); + SchemaManager schema_manager(file_system, path); + ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, schema_manager.ReadSchema(0)); + auto catalog = std::make_shared(); + const std::map table_scheme_map = {{"file", "local"}}; + const std::map dynamic_options = {{Options::FILE_FORMAT, "orc"}}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, + ReadContextBuilder("another-table") + .WithCatalog(catalog, Identifier("db", "tbl")) + .WithFileSystemSchemeToIdentifierMap({{"file", "missing"}}) + .SetTableSchema("invalid schema") + .Finish()); + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, + CoreOptions::FromMap(table_schema->Options(), file_system)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr bound, + InternalReadContext::CreateForTable(std::move(context), table_schema, core_options, path, + dynamic_options, table_scheme_map)); + ASSERT_EQ(bound->GetTableSchema(), table_schema); + ASSERT_EQ(bound->GetPath(), path); + ASSERT_EQ(bound->GetCoreOptions().GetFileSystem(), file_system); + ASSERT_EQ(bound->GetFileSystemSchemeToIdentifierMap(), table_scheme_map); + ASSERT_TRUE(catalog->TableFileSystemRequests().empty()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr replaced, + InternalReadContext::CreateWithSchema(bound, bound->GetReadSchema())); + ASSERT_EQ(replaced->GetFileSystemSchemeToIdentifierMap(), table_scheme_map); +} + +TEST(InternalReadContext, TestFallbackToMainIgnoresContextSchema) { + const std::string path = GetDataDir() + "/orc/append_09.db/append_09"; + auto file_system = std::make_shared(); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr context, + ReadContextBuilder("another-table").SetTableSchema("invalid schema").Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr fallback, + InternalReadContext::CreateForFallbackBranch(std::move(context), "main", + file_system, path, {}, {})); + ASSERT_EQ(fallback->GetPath(), path); + ASSERT_EQ(fallback->GetTableSchema()->Fields().size(), 4U); + ASSERT_EQ(fallback->GetCoreOptions().GetFileSystem(), file_system); +} + TEST(InternalReadContext, TestReadWithUnspecifiedSchema) { // no read schema is specified, read all fields std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; diff --git a/src/paimon/core/operation/read_context.cpp b/src/paimon/core/operation/read_context.cpp index 3a24acbbd..186dc0943 100644 --- a/src/paimon/core/operation/read_context.cpp +++ b/src/paimon/core/operation/read_context.cpp @@ -325,12 +325,6 @@ Result> ReadContextBuilder::Finish() { if (impl_->catalog_ == nullptr && impl_->identifier_) { return Status::Invalid("cannot read through a null catalog"); } - if (impl_->catalog_ != nullptr && impl_->specific_file_system_ == nullptr) { - // A catalog issuing per-table temporary credentials only hands them out through - // GetTableFileSystem, so the read uses the credentials of this table. - PAIMON_ASSIGN_OR_RAISE(impl_->specific_file_system_, - impl_->catalog_->GetTableFileSystem(impl_->identifier_.value())); - } if (impl_->enable_prefetch_ && impl_->prefetch_max_parallel_num_ == 0) { return Status::Invalid("prefetch max parallel num should be greater than 0"); } diff --git a/src/paimon/core/operation/read_context_test.cpp b/src/paimon/core/operation/read_context_test.cpp index a2845ef36..d268ba9ca 100644 --- a/src/paimon/core/operation/read_context_test.cpp +++ b/src/paimon/core/operation/read_context_test.cpp @@ -40,28 +40,24 @@ #include "paimon/testing/utils/testharness.h" namespace paimon::test { -TEST(ReadContextTest, TestWithCatalogResolvesTableFileSystem) { - // WithCatalog is a shorthand for WithFileSystem(catalog->GetTableFileSystem(identifier)); it - // resolves the per-table file system when Finish() builds the context, which also keeps the - // catalog and identifier for the operators to resolve the table through. +TEST(ReadContextTest, TestWithCatalogDefersTableFileSystem) { auto table_fs = std::make_shared(); auto catalog = std::make_shared(); catalog->SetTableFileSystem(table_fs); ReadContextBuilder builder("table_root_path"); ASSERT_OK_AND_ASSIGN(auto ctx, builder.WithCatalog(catalog, Identifier("db1", "t1")).Finish()); - ASSERT_EQ(ctx->GetSpecificFileSystem(), table_fs); + ASSERT_FALSE(ctx->GetSpecificFileSystem()); ASSERT_EQ(ctx->GetCatalog(), catalog); ASSERT_EQ(ctx->GetIdentifier(), std::optional(Identifier("db1", "t1"))); - ASSERT_EQ(catalog->TableFileSystemRequests().size(), 1U); - ASSERT_EQ(catalog->TableFileSystemRequests().front(), Identifier("db1", "t1")); + ASSERT_TRUE(catalog->TableFileSystemRequests().empty()); // Finish() resets the builder, so the next context reads without the catalog. ASSERT_OK_AND_ASSIGN(auto next_ctx, builder.Finish()); ASSERT_FALSE(next_ctx->GetSpecificFileSystem()); ASSERT_FALSE(next_ctx->GetCatalog()); ASSERT_FALSE(next_ctx->GetIdentifier()); - ASSERT_EQ(catalog->TableFileSystemRequests().size(), 1U); + ASSERT_TRUE(catalog->TableFileSystemRequests().empty()); } TEST(ReadContextTest, TestWithFileSystemOverridesCatalog) { diff --git a/src/paimon/core/operation/scan_context.cpp b/src/paimon/core/operation/scan_context.cpp index e8a95582f..16d78da75 100644 --- a/src/paimon/core/operation/scan_context.cpp +++ b/src/paimon/core/operation/scan_context.cpp @@ -195,12 +195,6 @@ Result> ScanContextBuilder::Finish() { return Status::Invalid( "WithFileSystem() and WithFileSystemSchemeToIdentifierMap() cannot be used together"); } - if (impl_->catalog_ != nullptr && impl_->specific_file_system_ == nullptr && - impl_->fs_scheme_to_identifier_map_.empty()) { - // A scheme map takes precedence over catalog-issued per-table credentials. - PAIMON_ASSIGN_OR_RAISE(impl_->specific_file_system_, - impl_->catalog_->GetTableFileSystem(impl_->identifier_.value())); - } std::shared_ptr executor = impl_->executor_ ? impl_->executor_ : CreateDefaultExecutor(); auto ctx = std::make_unique( diff --git a/src/paimon/core/operation/scan_context_test.cpp b/src/paimon/core/operation/scan_context_test.cpp index e18e60b1a..e1fad6b4c 100644 --- a/src/paimon/core/operation/scan_context_test.cpp +++ b/src/paimon/core/operation/scan_context_test.cpp @@ -31,28 +31,24 @@ #include "paimon/testing/utils/testharness.h" namespace paimon::test { -TEST(ScanContextTest, TestWithCatalogResolvesTableFileSystem) { - // WithCatalog is a shorthand for WithFileSystem(catalog->GetTableFileSystem(identifier)); it - // resolves the per-table file system when Finish() builds the context, which also keeps the - // catalog and identifier for the operators to resolve the table through. +TEST(ScanContextTest, TestWithCatalogDefersTableFileSystem) { auto table_fs = std::make_shared(); auto catalog = std::make_shared(); catalog->SetTableFileSystem(table_fs); ScanContextBuilder builder("table_root_path"); ASSERT_OK_AND_ASSIGN(auto ctx, builder.WithCatalog(catalog, Identifier("db1", "t1")).Finish()); - ASSERT_EQ(ctx->GetSpecificFileSystem(), table_fs); + ASSERT_FALSE(ctx->GetSpecificFileSystem()); ASSERT_EQ(ctx->GetCatalog(), catalog); ASSERT_EQ(ctx->GetIdentifier(), std::optional(Identifier("db1", "t1"))); - ASSERT_EQ(catalog->TableFileSystemRequests().size(), 1U); - ASSERT_EQ(catalog->TableFileSystemRequests().front(), Identifier("db1", "t1")); + ASSERT_TRUE(catalog->TableFileSystemRequests().empty()); // Finish() resets the builder, so the next context scans without the catalog. ASSERT_OK_AND_ASSIGN(auto next_ctx, builder.Finish()); ASSERT_FALSE(next_ctx->GetSpecificFileSystem()); ASSERT_FALSE(next_ctx->GetCatalog()); ASSERT_FALSE(next_ctx->GetIdentifier()); - ASSERT_EQ(catalog->TableFileSystemRequests().size(), 1U); + ASSERT_TRUE(catalog->TableFileSystemRequests().empty()); } TEST(ScanContextTest, TestWithFileSystemOverridesCatalog) { diff --git a/src/paimon/core/operation/write_context.cpp b/src/paimon/core/operation/write_context.cpp index 328c2043c..93d4f340a 100644 --- a/src/paimon/core/operation/write_context.cpp +++ b/src/paimon/core/operation/write_context.cpp @@ -206,12 +206,6 @@ Result> WriteContextBuilder::Finish() { PAIMON_RETURN_NOT_OK( CatalogUtils::CheckNotSystemTable(impl_->identifier_.value(), "write")); } - if (impl_->catalog_ != nullptr && impl_->specific_file_system_ == nullptr && - impl_->fs_scheme_to_identifier_map_.empty()) { - // A scheme map takes precedence over catalog-issued per-table credentials. - PAIMON_ASSIGN_OR_RAISE(impl_->specific_file_system_, - impl_->catalog_->GetTableFileSystem(impl_->identifier_.value())); - } bool enable_multi_thread_spill = impl_->spill_thread_number_ > 0; if (enable_multi_thread_spill) { PAIMON_RETURN_NOT_OK_FROM_ARROW( diff --git a/src/paimon/core/operation/write_context_test.cpp b/src/paimon/core/operation/write_context_test.cpp index ea551eeeb..d4119862b 100644 --- a/src/paimon/core/operation/write_context_test.cpp +++ b/src/paimon/core/operation/write_context_test.cpp @@ -69,24 +69,22 @@ TEST(WriteContextTest, TestWithCatalog) { "cannot write through a null catalog"); } -TEST(WriteContextTest, TestWithCatalogResolvesTableFileSystem) { - // Finish() resolves the per-table file system once. +TEST(WriteContextTest, TestWithCatalogDefersTableFileSystem) { auto table_fs = std::make_shared(); auto catalog = std::make_shared(); catalog->SetTableFileSystem(table_fs); WriteContextBuilder builder("table_root_path", "commit_user_1"); ASSERT_OK_AND_ASSIGN(auto ctx, builder.WithCatalog(catalog, Identifier("db1", "t1")).Finish()); - ASSERT_EQ(ctx->GetSpecificFileSystem(), table_fs); - ASSERT_EQ(catalog->TableFileSystemRequests().size(), 1U); - ASSERT_EQ(catalog->TableFileSystemRequests().front(), Identifier("db1", "t1")); + ASSERT_FALSE(ctx->GetSpecificFileSystem()); + ASSERT_TRUE(catalog->TableFileSystemRequests().empty()); WriteContextBuilder map_builder("table_root_path", "commit_user_1"); ASSERT_OK_AND_ASSIGN(auto map_ctx, map_builder.WithCatalog(catalog, Identifier("db1", "t1")) .WithFileSystemSchemeToIdentifierMap({{"file", "local"}}) .Finish()); ASSERT_FALSE(map_ctx->GetSpecificFileSystem()); - ASSERT_EQ(catalog->TableFileSystemRequests().size(), 1U); + ASSERT_TRUE(catalog->TableFileSystemRequests().empty()); } TEST(WriteContextTest, TestSetContent) { diff --git a/src/paimon/core/table/file_store_table.cpp b/src/paimon/core/table/file_store_table.cpp index 808334130..90fca78ee 100644 --- a/src/paimon/core/table/file_store_table.cpp +++ b/src/paimon/core/table/file_store_table.cpp @@ -155,7 +155,7 @@ Result> FileStoreTable::Create( Result> FileStoreTable::CopyTableSchema( const std::shared_ptr& fs, const std::string& root_path, - const std::string& current_branch, const std::shared_ptr& schema, + const std::shared_ptr& schema, const std::map& applied_options, const std::map& dynamic_options) { std::map overrides = dynamic_options; @@ -180,11 +180,12 @@ Result> FileStoreTable::CopyTableSchema( PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merged, schema->MergeOptions(overrides)); PAIMON_ASSIGN_OR_RAISE(CoreOptions options, CoreOptions::FromMap(merged->Options(), fs)); PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateStartupMode(options)); + const std::string target_branch = BranchManager::NormalizeBranch(options.GetBranch()); // Snapshot lookup failures defer to scanning; a resolved snapshot with a missing schema // fails Copy. const std::shared_ptr resolved_fs = options.GetFileSystem(); - SnapshotManager snapshots(resolved_fs, root_path, current_branch); + SnapshotManager snapshots(resolved_fs, root_path, target_branch); auto resolve_snapshot = [&]() -> Result> { const int32_t selectors = options.GetScanSnapshotId().has_value() + options.GetScanTagName().has_value() + @@ -198,7 +199,7 @@ Result> FileStoreTable::CopyTableSchema( return std::optional(std::move(snapshot)); } if (options.GetScanTagName()) { - TagManager tags(resolved_fs, root_path, current_branch); + TagManager tags(resolved_fs, root_path, target_branch); PAIMON_ASSIGN_OR_RAISE(Tag tag, tags.GetOrThrow(*options.GetScanTagName())); PAIMON_ASSIGN_OR_RAISE(Snapshot snapshot, tag.TrimToSnapshot()); return std::optional(std::move(snapshot)); @@ -207,7 +208,7 @@ Result> FileStoreTable::CopyTableSchema( }; auto snapshot = resolve_snapshot(); if (snapshot.ok() && snapshot.value()) { - SchemaManager schemas(resolved_fs, root_path, current_branch); + SchemaManager schemas(resolved_fs, root_path, target_branch); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr historical, schemas.ReadSchema(snapshot.value()->SchemaId())); auto historical_options = merged->Options(); @@ -243,10 +244,10 @@ Result> FileStoreTable::CopyTableSchema( Result> FileStoreTable::Copy( const std::map& dynamic_options) const { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr copied_schema, - CopyTableSchema(file_system_, root_path_, CurrentBranch(), - std::static_pointer_cast(schema_), - dynamic_options_, dynamic_options)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr copied_schema, + CopyTableSchema(file_system_, root_path_, std::static_pointer_cast(schema_), + dynamic_options_, dynamic_options)); auto accumulated = dynamic_options_; for (const auto& [key, value] : dynamic_options) { if (key != Options::TYPE) { @@ -554,7 +555,7 @@ Result> FileStoreTable::NewRead( PAIMON_ASSIGN_OR_RAISE( std::shared_ptr internal_context, InternalReadContext::CreateForTable(read_context, loaded_schema, store_->GetOptions(), - root_path_, dynamic_options_)); + root_path_, dynamic_options_, fs_scheme_map_)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, CreateTableRead(internal_context, read_context->GetMemoryPool(), @@ -570,7 +571,7 @@ Result> FileStoreTable::NewRead( PAIMON_ASSIGN_OR_RAISE(std::shared_ptr fallback_context, InternalReadContext::CreateForFallbackBranch( read_context, /*branch=*/scan_fallback_branch.value(), file_system_, - root_path_, dynamic_options_)); + root_path_, dynamic_options_, fs_scheme_map_)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr fallback_table_read, CreateTableRead(fallback_context, read_context->GetMemoryPool(), diff --git a/src/paimon/core/table/file_store_table.h b/src/paimon/core/table/file_store_table.h index fc44b583d..f6b403a62 100644 --- a/src/paimon/core/table/file_store_table.h +++ b/src/paimon/core/table/file_store_table.h @@ -83,13 +83,14 @@ class FileStoreTable : public Table { /// Applies dynamic options to the current schema, retaining the table's identifier. Time /// travel selects the schema of the requested snapshot. Use SwitchToBranch to load another /// branch's latest schema. + /// Snapshot/tag lookup and historical schema loading may perform metadata I/O. Result> Copy( const std::map& dynamic_options) const override; /// Shared schema-copy semantics for managed tables and their system-table views. static Result> CopyTableSchema( const std::shared_ptr& fs, const std::string& root_path, - const std::string& current_branch, const std::shared_ptr& schema, + const std::shared_ptr& schema, const std::map& applied_options, const std::map& dynamic_options); diff --git a/src/paimon/core/table/file_store_table_factory.cpp b/src/paimon/core/table/file_store_table_factory.cpp index a1bf0895d..e99b3df78 100644 --- a/src/paimon/core/table/file_store_table_factory.cpp +++ b/src/paimon/core/table/file_store_table_factory.cpp @@ -39,6 +39,7 @@ #include "paimon/core/table/file_store_table.h" #include "paimon/core/table/system/system_table.h" #include "paimon/core/utils/branch_manager.h" +#include "paimon/core/utils/context_utils.h" #include "paimon/defs.h" #include "paimon/fs/file_system.h" #include "paimon/read_context.h" @@ -135,9 +136,11 @@ Result> ManagedTable(const Context& ctx, kind, branch)); } } - PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, - CoreOptions::FromMap(options, ctx.GetSpecificFileSystem(), - ctx.GetFileSystemSchemeToIdentifierMap())); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr file_system, + ContextUtils::ResolveFileSystemForLoad(ctx)); + PAIMON_ASSIGN_OR_RAISE( + CoreOptions tmp_options, + CoreOptions::FromMap(options, file_system, ctx.GetFileSystemSchemeToIdentifierMap())); PAIMON_ASSIGN_OR_RAISE(std::optional system_table_path, SystemTableLoader::TryParsePath(ctx.GetPath())); if (system_table_path) { @@ -155,9 +158,9 @@ Result> ManagedTable(const Context& ctx, catalog_table_schema, CatalogMainBranchSchemaJson(ctx.GetCatalog(), catalog_identifier, branch, kind)); } - return ManagedTableOf(tmp_options.GetFileSystem(), ctx.GetSpecificFileSystem(), ctx.GetPath(), - branch, options, catalog_table_schema, ctx.GetIdentifier(), - ctx.GetTableId().value_or(""), ctx.GetFileSystemSchemeToIdentifierMap()); + return ManagedTableOf(tmp_options.GetFileSystem(), file_system, ctx.GetPath(), branch, options, + catalog_table_schema, ctx.GetIdentifier(), ctx.GetTableId().value_or(""), + ctx.GetFileSystemSchemeToIdentifierMap()); } } // namespace diff --git a/src/paimon/core/table/format/format_table.cpp b/src/paimon/core/table/format/format_table.cpp index f200a85cf..4340ac145 100644 --- a/src/paimon/core/table/format/format_table.cpp +++ b/src/paimon/core/table/format/format_table.cpp @@ -221,7 +221,7 @@ Result> FormatTable::Copy( } const std::vector& FormatTable::PartitionKeys() const { - return LatestSchema()->PartitionKeys(); + return GetSchema()->PartitionKeys(); } Result> FormatTable::GetArrowSchema() const { diff --git a/src/paimon/core/table/format/format_table_test.cpp b/src/paimon/core/table/format/format_table_test.cpp index 21e809680..f77887ba5 100644 --- a/src/paimon/core/table/format/format_table_test.cpp +++ b/src/paimon/core/table/format/format_table_test.cpp @@ -754,11 +754,10 @@ TEST(FormatTableTest, TestCatalogManagedPartitionsAreRejected) { // And the same refusal when it arrives at the call rather than in the schema. Validating the // schema's own options alone would let this one through and then drop it, leaving a caller // who asked for catalog-managed partitions with a scan that read the directories anyway. - Result> dynamic_table = - FormatTable::Create(off_dir->GetFileSystem(), off_dir->Str(), Identifier("db", "tbl"), - off_table->LatestSchema(), - /*location_carries_paimon_metadata=*/true, - {{Options::METASTORE_PARTITIONED_TABLE, "true"}}); + Result> dynamic_table = FormatTable::Create( + off_dir->GetFileSystem(), off_dir->Str(), Identifier("db", "tbl"), off_table->GetSchema(), + /*location_carries_paimon_metadata=*/true, + {{Options::METASTORE_PARTITIONED_TABLE, "true"}}); ASSERT_FALSE(dynamic_table.ok()); ASSERT_TRUE(dynamic_table.status().IsNotImplemented()) << dynamic_table.status().ToString(); } @@ -2693,7 +2692,7 @@ TEST(FormatTableTest, TestAFormatTableIsWrittenCommittedAndReadBack) { "a format table sends no commit table request"); auto format_catalog = std::make_shared(); - format_catalog->SetTableSchema(table->LatestSchema()); + format_catalog->SetTableSchema(table->GetSchema()); CommitContextBuilder catalog_builder(dir->Str(), "test-user"); catalog_builder.WithCatalog(format_catalog, Identifier("db", "fmt")); ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog_context, catalog_builder.Finish()); @@ -2745,7 +2744,7 @@ TEST(FormatTableTest, TestACatalogInTheContextIsRefusedAlikeByWriteAndCommit) { CreateTable(dir->GetFileSystem(), dir->Str(), {"dt"})); auto catalog = std::make_shared(); - catalog->SetTableSchema(table->LatestSchema()); + catalog->SetTableSchema(table->GetSchema()); Identifier identifier("db", "fmt"); WriteContextBuilder write_builder(dir->Str(), "test-user"); @@ -2934,7 +2933,7 @@ TEST(FormatTableTest, TestTheTableEntryPointsRefuseWhatAFormatTableCannotHonour) "not supported for format tables"); auto catalog = std::make_shared(); - catalog->SetTableSchema(format_table->LatestSchema()); + catalog->SetTableSchema(format_table->GetSchema()); auto catalog_write_context = [&dir, &catalog]() -> Result> { WriteContextBuilder builder(dir->Str(), "test-user"); diff --git a/src/paimon/core/table/system/audit_log_system_table.cpp b/src/paimon/core/table/system/audit_log_system_table.cpp index 4da5a803a..9b4e1aaa7 100644 --- a/src/paimon/core/table/system/audit_log_system_table.cpp +++ b/src/paimon/core/table/system/audit_log_system_table.cpp @@ -398,15 +398,10 @@ Result> AuditLogSystemTable::NewScan( } std::shared_ptr scan_filter = context->GetScanFilters(); ScanContextBuilder builder(table_path_); - std::shared_ptr data_table_fs = context->GetSpecificFileSystem(); - if (data_table_fs == nullptr && context->GetFileSystemSchemeToIdentifierMap().empty()) { - data_table_fs = GetFileSystem(); - } builder.WithStreamingMode(context->IsStreamingMode()) .WithMemoryPool(context->GetMemoryPool()) .WithExecutor(context->GetExecutor()) - .WithFileSystem(data_table_fs) - .WithFileSystemSchemeToIdentifierMap(context->GetFileSystemSchemeToIdentifierMap()) + .WithFileSystem(GetFileSystem()) .WithCache(context->GetCache()); if (scan_filter) { if (scan_filter->GetBucketFilter()) { @@ -420,10 +415,8 @@ Result> AuditLogSystemTable::NewScan( builder.SetLimit(context->GetLimit().value()); } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr data_context, builder.Finish()); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr data_table, - CreateDataTable(table_schema_, options_, data_context->GetSpecificFileSystem(), - data_context->GetFileSystemSchemeToIdentifierMap())); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr data_table, + CreateDataTable(table_schema_, options_)); return data_table->NewScan(std::move(data_context)); } @@ -450,10 +443,8 @@ Result> AuditLogSystemTable::NewChangelogRead( PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(read_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr data_context, CreateDataReadContext(context, read_options)); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr data_table, - CreateDataTable(table_schema_, read_options, data_context->GetSpecificFileSystem(), - data_context->GetFileSystemSchemeToIdentifierMap())); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr data_table, + CreateDataTable(table_schema_, read_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr data_read, data_table->NewRead(std::move(data_context))); auto* key_value_read = dynamic_cast(data_read.get()); @@ -474,15 +465,10 @@ Result> AuditLogSystemTable::CreateDataReadContext( PAIMON_ASSIGN_OR_RAISE(std::shared_ptr base_read_schema, BaseReadSchema()); PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(read_options)); ReadContextBuilder builder(table_path_); - std::shared_ptr data_table_fs = context->GetSpecificFileSystem(); - if (data_table_fs == nullptr && context->GetFileSystemSchemeToIdentifierMap().empty()) { - data_table_fs = GetFileSystem(); - } builder.SetReadFieldNames(base_read_schema->field_names()) .WithMemoryPool(context->GetMemoryPool()) .WithExecutor(context->GetExecutor()) - .WithFileSystem(data_table_fs) - .WithFileSystemSchemeToIdentifierMap(context->GetFileSystemSchemeToIdentifierMap()) + .WithFileSystem(GetFileSystem()) .EnablePrefetch(context->EnablePrefetch()) .SetPrefetchBatchCount(context->GetPrefetchBatchCount()) .SetPrefetchMaxParallelNum(context->GetPrefetchMaxParallelNum()) diff --git a/src/paimon/core/table/system/read_optimized_system_table.cpp b/src/paimon/core/table/system/read_optimized_system_table.cpp index edc444369..e7bd4a06a 100644 --- a/src/paimon/core/table/system/read_optimized_system_table.cpp +++ b/src/paimon/core/table/system/read_optimized_system_table.cpp @@ -67,15 +67,10 @@ Result> ReadOptimizedSystemTable::NewScan( } auto options = ReadOptimizedOptions(); ScanContextBuilder builder(table_path_); - std::shared_ptr data_table_fs = context->GetSpecificFileSystem(); - if (data_table_fs == nullptr && context->GetFileSystemSchemeToIdentifierMap().empty()) { - data_table_fs = GetFileSystem(); - } builder.WithStreamingMode(context->IsStreamingMode()) .WithMemoryPool(context->GetMemoryPool()) .WithExecutor(context->GetExecutor()) - .WithFileSystem(data_table_fs) - .WithFileSystemSchemeToIdentifierMap(context->GetFileSystemSchemeToIdentifierMap()) + .WithFileSystem(GetFileSystem()) .WithCache(context->GetCache()); if (context->GetLimit().has_value()) { builder.SetLimit(context->GetLimit().value()); @@ -90,24 +85,15 @@ Result> ReadOptimizedSystemTable::NewScan( if (context->GetGlobalIndexResult()) { builder.SetGlobalIndexResult(context->GetGlobalIndexResult()); } - if (context->GetSpecificTableSchema().has_value()) { - builder.SetTableSchema(context->GetSpecificTableSchema().value()); - } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr base_context, builder.Finish()); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr data_table, - CreateDataTable(table_schema_, options, base_context->GetSpecificFileSystem(), - base_context->GetFileSystemSchemeToIdentifierMap())); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr data_table, + CreateDataTable(table_schema_, options)); return data_table->NewScan(std::move(base_context)); } Result> ReadOptimizedSystemTable::CreateDataReadContext( const std::shared_ptr& context) const { ReadContextBuilder builder(table_path_); - std::shared_ptr data_table_fs = context->GetSpecificFileSystem(); - if (data_table_fs == nullptr && context->GetFileSystemSchemeToIdentifierMap().empty()) { - data_table_fs = GetFileSystem(); - } builder.SetPredicate(context->GetPredicate()) .EnablePredicateFilter(context->EnablePredicateFilter()) .EnablePrefetch(context->EnablePrefetch()) @@ -117,8 +103,7 @@ Result> ReadOptimizedSystemTable::CreateDataReadCon .SetRowToBatchThreadNumber(context->GetRowToBatchThreadNumber()) .WithMemoryPool(context->GetMemoryPool()) .WithExecutor(context->GetExecutor()) - .WithFileSystem(data_table_fs) - .WithFileSystemSchemeToIdentifierMap(context->GetFileSystemSchemeToIdentifierMap()) + .WithFileSystem(GetFileSystem()) .SetReadAheadCacheEnabled(context->ReadAheadCacheEnabled()) .WithCacheConfig(context->GetCacheConfig()) .SetWarmupLevel(context->GetWarmupLevel()) @@ -132,9 +117,6 @@ Result> ReadOptimizedSystemTable::CreateDataReadCon PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*read_schema, c_read_schema.get())); builder.SetReadSchema(std::move(c_read_schema)); } - if (context->GetSpecificTableSchema().has_value()) { - builder.SetTableSchema(context->GetSpecificTableSchema().value()); - } return builder.Finish(); } @@ -145,10 +127,8 @@ Result> ReadOptimizedSystemTable::NewRead( PAIMON_RETURN_NOT_OK(CheckNoSearch(context)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr base_context, CreateDataReadContext(context)); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr data_table, - CreateDataTable(table_schema_, options_, base_context->GetSpecificFileSystem(), - base_context->GetFileSystemSchemeToIdentifierMap())); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr data_table, + CreateDataTable(table_schema_, options_)); return data_table->NewRead(std::move(base_context)); } diff --git a/src/paimon/core/table/system/system_table.cpp b/src/paimon/core/table/system/system_table.cpp index 70767c5b1..06291ebea 100644 --- a/src/paimon/core/table/system/system_table.cpp +++ b/src/paimon/core/table/system/system_table.cpp @@ -224,16 +224,14 @@ Status SystemTable::AdoptOwnSchema() { Result> SystemTable::CreateDataTable( const std::shared_ptr& table_schema, - const std::map& options, - const std::shared_ptr& specific_file_system, - const std::map& fs_scheme_map) const { + const std::map& options) const { PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merged_schema, table_schema->MergeOptions(options)); const std::string branch = BranchManager::NormalizeBranch(LoadBranch(merged_schema->Options())); - return FileStoreTable::Create(specific_file_system, GetRootPath(), - std::shared_ptr(std::move(merged_schema)), - CatalogUtils::IdentifierFromPath(GetRootPath(), branch), "", - branch, fs_scheme_map, options); + return FileStoreTable::Create( + GetFileSystem(), GetRootPath(), std::shared_ptr(std::move(merged_schema)), + CatalogUtils::IdentifierFromPath(GetRootPath(), branch), "", branch, + origin_ ? origin_->fs_scheme_map : std::map{}, options); } Result> SystemTable::Copy( @@ -258,10 +256,9 @@ Result> SystemTable::Copy( } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr current_schema, origin_->table_schema->MergeOptions(origin_->dynamic_options)); - const std::string current_branch = LoadBranch(current_schema->Options()); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr copied_schema, - FileStoreTable::CopyTableSchema(GetFileSystem(), GetRootPath(), current_branch, + FileStoreTable::CopyTableSchema(GetFileSystem(), GetRootPath(), std::shared_ptr(std::move(current_schema)), origin_->dynamic_options, dynamic_options)); auto accumulated = origin_->dynamic_options; @@ -271,7 +268,7 @@ Result> SystemTable::Copy( } } return SystemTableLoader::Load(origin_->kind, GetFileSystem(), GetRootPath(), copied_schema, - accumulated, origin_->data_table_name); + accumulated, origin_->data_table_name, origin_->fs_scheme_map); } Result> SystemTable::NewWrite( @@ -292,7 +289,8 @@ bool SystemTableLoader::IsSupported(const std::string& system_table_name) { Result> SystemTableLoader::Load( const std::string& system_table_name, const std::shared_ptr& fs, const std::string& table_path, const std::shared_ptr& table_schema, - const std::map& dynamic_options, const std::string& data_table_name) { + const std::map& dynamic_options, const std::string& data_table_name, + const std::map& fs_scheme_map) { std::optional factory = FindSystemTableFactory(system_table_name); if (factory) { const std::string effective_name = @@ -304,6 +302,7 @@ Result> SystemTableLoader::Load( origin.kind = StringUtils::ToLowerCase(system_table_name); origin.table_schema = table_schema; origin.dynamic_options = dynamic_options; + origin.fs_scheme_map = fs_scheme_map; origin.data_table_name = data_table_name; system_table->origin_.emplace(std::move(origin)); PAIMON_RETURN_NOT_OK(system_table->AdoptOwnSchema()); diff --git a/src/paimon/core/table/system/system_table.h b/src/paimon/core/table/system/system_table.h index 9cab11cd8..e76c21e83 100644 --- a/src/paimon/core/table/system/system_table.h +++ b/src/paimon/core/table/system/system_table.h @@ -96,9 +96,7 @@ class SystemTable : public Table, public std::enable_shared_from_this> CreateDataTable( const std::shared_ptr& table_schema, - const std::map& options, - const std::shared_ptr& specific_file_system, - const std::map& fs_scheme_map) const; + const std::map& options) const; friend class SystemTableLoader; friend class GlobalSystemTableLoader; @@ -110,6 +108,7 @@ class SystemTable : public Table, public std::enable_shared_from_this table_schema; std::map dynamic_options; + std::map fs_scheme_map; /// Logical data table name, including its branch; empty uses the directory name. std::string data_table_name; /// Global table context; null for table-scoped views. @@ -131,7 +130,8 @@ class SystemTableLoader { const std::string& system_table_name, const std::shared_ptr& fs, const std::string& table_path, const std::shared_ptr& table_schema, const std::map& dynamic_options, - const std::string& data_table_name); + const std::string& data_table_name, + const std::map& fs_scheme_map = {}); static Result> TryParsePath(const std::string& path); diff --git a/src/paimon/core/table/system/system_table_test.cpp b/src/paimon/core/table/system/system_table_test.cpp index 2c063c278..9f4924c61 100644 --- a/src/paimon/core/table/system/system_table_test.cpp +++ b/src/paimon/core/table/system/system_table_test.cpp @@ -61,6 +61,7 @@ #include "paimon/table/source/table_read.h" #include "paimon/table/source/table_scan.h" #include "paimon/testing/mock/mock_file_batch_reader.h" +#include "paimon/testing/mock/mock_file_system.h" #include "paimon/testing/utils/read_result_collector.h" #include "paimon/testing/utils/testharness.h" #include "paimon/write_context.h" @@ -313,6 +314,43 @@ TEST(SystemTableTest, TestReadOptimizedSystemTablePathParsing) { // defaults, so a setting that is not copied across silently reverts: a caller that asked for NONE // or DECODED would get RAW back. `$ro` has its own builder chain and `$audit_log` and `$binlog` // share one, so all three are pinned here. +TEST(SystemTableTest, TestDataReadContextsUseBoundStorage) { + const std::map options = {{Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "orc"}}; + ASSERT_OK_AND_ASSIGN(std::shared_ptr schema, CreateTableSchemaForTest(options)); + auto fs = std::make_shared(); + auto caller_fs = std::make_shared(); + ReadOptimizedSystemTable ro(fs, "/table", schema, "table", options); + AuditLogSystemTable audit(fs, "/table", schema, "table", options); + BinlogSystemTable binlog(fs, "/table", schema, "table", options); + for (bool use_scheme_map : {false, true}) { + ReadContextBuilder builder("another-table"); + builder.SetTableSchema("invalid schema").SetWarmupLevel(WarmupLevel::DECODED); + if (use_scheme_map) { + builder.WithFileSystemSchemeToIdentifierMap({{"file", "missing"}}); + } else { + builder.WithFileSystem(caller_fs); + } + ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ro_context, + ro.CreateDataReadContext(context)); + ASSERT_OK_AND_ASSIGN(auto audit_options, audit.ReadOptions()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr audit_context, + audit.CreateDataReadContext(context, audit_options)); + ASSERT_OK_AND_ASSIGN(auto binlog_options, binlog.ReadOptions()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr binlog_context, + binlog.CreateDataReadContext(context, binlog_options)); + for (const ReadContext* effective : + {ro_context.get(), audit_context.get(), binlog_context.get()}) { + ASSERT_EQ(effective->GetSpecificFileSystem(), fs); + ASSERT_TRUE(effective->GetFileSystemSchemeToIdentifierMap().empty()); + ASSERT_FALSE(effective->GetSpecificTableSchema()); + ASSERT_EQ(effective->GetPath(), "/table"); + ASSERT_EQ(effective->GetWarmupLevel(), WarmupLevel::DECODED); + } + } +} + TEST(SystemTableTest, TestNewReadPropagatesWarmupLevel) { std::map options = {{Options::FILE_SYSTEM, "local"}, {Options::FILE_FORMAT, "orc"}}; diff --git a/src/paimon/core/table/table_test.cpp b/src/paimon/core/table/table_test.cpp index b2589be4b..492ddc5a3 100644 --- a/src/paimon/core/table/table_test.cpp +++ b/src/paimon/core/table/table_test.cpp @@ -18,6 +18,7 @@ #include "paimon/catalog/table.h" +#include #include #include #include @@ -189,7 +190,7 @@ TEST(TableTest, TestCopyKeepsTheBranchTheTableServes) { ASSERT_TRUE(file_store_copy != nullptr); EXPECT_EQ(file_store_copy->CurrentBranch(), "main"); std::shared_ptr copied_schema = - std::dynamic_pointer_cast(copy->LatestSchema()); + std::dynamic_pointer_cast(copy->GetSchema()); ASSERT_TRUE(copied_schema != nullptr); EXPECT_EQ(copied_schema->Options().at(Options::BUCKET), "2"); { @@ -249,7 +250,7 @@ TEST(TableTest, TestCopyBranchRetainsCurrentSchemaAndIdentifier) { EXPECT_EQ("dev", dev_file_store_copy->CurrentBranch()); EXPECT_EQ(table->Name(), dev_copy->Name()); std::shared_ptr dev_copied_schema = - std::dynamic_pointer_cast(dev_copy->LatestSchema()); + std::dynamic_pointer_cast(dev_copy->GetSchema()); ASSERT_TRUE(dev_copied_schema != nullptr); EXPECT_EQ(dev_copied_schema->Options().at(Options::BUCKET), "1"); EXPECT_EQ(dev_copied_schema->Options().at(Options::BRANCH), "dev"); @@ -258,14 +259,14 @@ TEST(TableTest, TestCopyBranchRetainsCurrentSchemaAndIdentifier) { ASSERT_OK_AND_ASSIGN(std::shared_ptr
dev_copy_restyled, table->Copy({{Options::BRANCH, "dev"}, {Options::BUCKET, "2"}})); std::shared_ptr restyled_schema = - std::dynamic_pointer_cast(dev_copy_restyled->LatestSchema()); + std::dynamic_pointer_cast(dev_copy_restyled->GetSchema()); ASSERT_TRUE(restyled_schema != nullptr); EXPECT_EQ(restyled_schema->Options().at(Options::BUCKET), "2"); EXPECT_EQ(restyled_schema->Options().at(Options::BUCKET_KEY), "id"); ASSERT_OK_AND_ASSIGN(std::shared_ptr
missing, table->Copy({{Options::BRANCH, "missing"}})); - EXPECT_EQ(table->LatestSchema()->FieldNames(), missing->LatestSchema()->FieldNames()); + EXPECT_EQ(table->GetSchema()->FieldNames(), missing->GetSchema()->FieldNames()); } TEST(TableTest, TestACopyHandsItsDynamicOptionsToTheFallbackReader) { @@ -408,7 +409,7 @@ TEST(TableTest, TestACopySwitchingBranchesKeepsItsAccumulatedDynamicOptions) { ASSERT_TRUE(dev_file_store_copy != nullptr); EXPECT_EQ("dev", dev_file_store_copy->CurrentBranch()); std::shared_ptr dev_copied_schema = - std::dynamic_pointer_cast(dev_copy->LatestSchema()); + std::dynamic_pointer_cast(dev_copy->GetSchema()); ASSERT_TRUE(dev_copied_schema != nullptr); EXPECT_EQ("true", dev_copied_schema->Options().at(Options::KEY_VALUE_SEQUENCE_NUMBER_ENABLED)); ReadContextBuilder read_builder(path); @@ -422,7 +423,7 @@ TEST(TableTest, TestACopySwitchingBranchesKeepsItsAccumulatedDynamicOptions) { with_sequence->Copy( {{Options::BRANCH, "dev"}, {Options::KEY_VALUE_SEQUENCE_NUMBER_ENABLED, "false"}})); std::shared_ptr off_schema = - std::dynamic_pointer_cast(off->LatestSchema()); + std::dynamic_pointer_cast(off->GetSchema()); ASSERT_TRUE(off_schema != nullptr); EXPECT_EQ("false", off_schema->Options().at(Options::KEY_VALUE_SEQUENCE_NUMBER_ENABLED)); } @@ -610,7 +611,7 @@ TEST(TableTest, TestACopyServesItsS3OptionThroughManagersOfItsOwn) { EXPECT_NE(copy_file_store_table->GetSnapshotManager(), file_store_table->GetSnapshotManager()); EXPECT_EQ("main", copy_file_store_table->CurrentBranch()); - auto copy_schema = std::dynamic_pointer_cast(copy->LatestSchema()); + auto copy_schema = std::dynamic_pointer_cast(copy->GetSchema()); ASSERT_TRUE(copy_schema != nullptr); EXPECT_EQ(endpoint, copy_schema->Options().at("s3.endpoint")); } @@ -638,8 +639,8 @@ TEST(TableTest, TestSwitchToBranch) { EXPECT_EQ("dev", dev_file_store_table->CurrentBranch()); EXPECT_EQ("tbl$branch_dev", dev_table->Name()); EXPECT_EQ("tbl$branch_dev", dev_table->FullName()); - ASSERT_EQ((std::vector{"c0"}), dev_table->LatestSchema()->FieldNames()); - auto dev_table_schema = std::dynamic_pointer_cast(dev_table->LatestSchema()); + ASSERT_EQ((std::vector{"c0"}), dev_table->GetSchema()->FieldNames()); + auto dev_table_schema = std::dynamic_pointer_cast(dev_table->GetSchema()); ASSERT_TRUE(dev_table_schema != nullptr); ASSERT_EQ("dev", dev_table_schema->Options().at(Options::BRANCH)); ASSERT_TRUE(dev_file_store_table->GetSchemaManager() != nullptr); @@ -651,7 +652,7 @@ TEST(TableTest, TestSwitchToBranch) { auto same_file_store_table = std::dynamic_pointer_cast(same_table); ASSERT_TRUE(same_file_store_table != nullptr); // Switching to the same branch still rebuilds its managers. - EXPECT_EQ(same_table->LatestSchema(), dev_table->LatestSchema()); + EXPECT_EQ(same_table->GetSchema(), dev_table->GetSchema()); EXPECT_NE(same_file_store_table->GetSchemaManager(), dev_file_store_table->GetSchemaManager()); EXPECT_NE(same_file_store_table->GetSnapshotManager(), dev_file_store_table->GetSnapshotManager()); @@ -667,8 +668,8 @@ TEST(TableTest, TestSwitchToBranch) { EXPECT_EQ("main", main_again_file_store_table->CurrentBranch()); EXPECT_EQ("tbl", main_again->Name()); EXPECT_EQ("tbl", main_again->FullName()); - ASSERT_EQ((std::vector{"id"}), main_again->LatestSchema()->FieldNames()); - auto main_again_schema = std::dynamic_pointer_cast(main_again->LatestSchema()); + ASSERT_EQ((std::vector{"id"}), main_again->GetSchema()->FieldNames()); + auto main_again_schema = std::dynamic_pointer_cast(main_again->GetSchema()); ASSERT_TRUE(main_again_schema != nullptr); EXPECT_EQ(0, main_again_schema->Options().count(Options::BRANCH)); @@ -768,7 +769,7 @@ TEST(TableTest, TestOperatorsServeTheBranchTheTableWasLoadedWith) { ASSERT_OK_AND_ASSIGN(std::shared_ptr
main_copy, table->Copy({{Options::BRANCH, "main"}})); - EXPECT_EQ(table->LatestSchema()->FieldNames(), main_copy->LatestSchema()->FieldNames()); + EXPECT_EQ(table->GetSchema()->FieldNames(), main_copy->GetSchema()->FieldNames()); } TEST(TableTest, TestGetTableWithUnknownDatabase) { @@ -801,7 +802,7 @@ TEST(TableTest, TestGetTableWithUnknownDatabase) { EXPECT_EQ(table->Name(), "tbl1"); EXPECT_EQ(table->FullName(), "tbl1"); - std::shared_ptr latest_schema = table->LatestSchema(); + std::shared_ptr latest_schema = table->GetSchema(); ASSERT_NE(latest_schema, nullptr); auto data_schema = std::dynamic_pointer_cast(latest_schema); ASSERT_TRUE(data_schema != nullptr); @@ -830,11 +831,11 @@ TEST(TableTest, TestGetTableWithBranch) { ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog.GetTable(Identifier("db1", "tbl1$branch_b1"))); ASSERT_EQ("tbl1$branch_b1", table->Name()); - ASSERT_EQ((std::vector{"b1_c0"}), table->LatestSchema()->FieldNames()); + ASSERT_EQ((std::vector{"b1_c0"}), table->GetSchema()->FieldNames()); for (const auto& main_identifier : {Identifier("db1", "tbl1$branch_main"), Identifier("db1", "tbl1$branch_MAIN")}) { ASSERT_OK_AND_ASSIGN(table, catalog.GetTable(main_identifier)); - ASSERT_EQ((std::vector{"main_c0"}), table->LatestSchema()->FieldNames()); + ASSERT_EQ((std::vector{"main_c0"}), table->GetSchema()->FieldNames()); } Status missing_branch = catalog.GetTable(Identifier("db1", "tbl1$branch_missing")).status(); @@ -906,7 +907,7 @@ TEST(TableTest, TestCatalogOpenedTableServesThePerTableFileSystem) { CreateManagedTable(dir->GetFileSystem(), dir->Str(), identifier)); auto catalog = std::make_shared(); - catalog->SetTableSchema(checked_pointer_cast(table->LatestSchema())); + catalog->SetTableSchema(checked_pointer_cast(table->GetSchema())); ASSERT_OK_AND_ASSIGN(std::shared_ptr
opened, catalog->GetTable(identifier)); EXPECT_NE(opened->GetFileSystem(), nullptr); @@ -1012,6 +1013,72 @@ TEST(TableTest, TestNewScanServesTheSchemaTheTableWasLoadedWith) { ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(ctx), {}), "not found latest schema"); } +TEST(TableTest, TestStaticFactoriesResolveCatalogFileSystemWhenLoading) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::shared_ptr fs = dir->GetFileSystem(); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + CreateManagedTable(fs, dir->Str(), identifier)); + auto catalog = std::make_shared(); + catalog->SetTableFileSystem(fs); + catalog->SetTableSchema(table->GetSchema()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, + ScanContextBuilder(dir->Str()).WithCatalog(catalog, identifier).Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, + ReadContextBuilder(dir->Str()).WithCatalog(catalog, identifier).Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr write_context, + WriteContextBuilder(dir->Str(), "user").WithCatalog(catalog, identifier).Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr commit_context, + CommitContextBuilder(dir->Str(), "user").WithCatalog(catalog, identifier).Finish()); + ASSERT_TRUE(catalog->TableFileSystemRequests().empty()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
scan_table, + FileStoreTableFactory::Create(*scan_context, {})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
read_table, + FileStoreTableFactory::Create(*read_context, {})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
write_table, + FileStoreTableFactory::Create(*write_context, {})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
commit_table, + FileStoreTableFactory::Create(*commit_context, {})); + for (const auto& loaded : {scan_table, read_table, write_table, commit_table}) { + ASSERT_EQ(loaded->GetFileSystem(), fs); + } + ASSERT_EQ(catalog->TableFileSystemRequests().size(), 4U); +} + +TEST(TableTest, TestSystemScansUseBoundFileSystem) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::shared_ptr fs = dir->GetFileSystem(); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + CreateManagedTable(fs, dir->Str(), Identifier("db", "tbl"), "main", + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + WriteAndCommitTwoRows(table, dir->Str()); + auto schema = std::dynamic_pointer_cast(table->GetSchema()); + ASSERT_TRUE(schema); + for (const std::string& kind : {"ro", "audit_log", "binlog"}) { + SCOPED_TRACE(kind); + ASSERT_OK_AND_ASSIGN(std::shared_ptr system_table, + SystemTableLoader::Load(kind, fs, dir->Str(), schema, {}, "tbl")); + for (bool use_scheme_map : {false, true}) { + ScanContextBuilder builder("another-table"); + builder.SetTableSchema("invalid schema"); + if (use_scheme_map) { + builder.WithFileSystemSchemeToIdentifierMap({{"file", "missing"}}); + } else { + builder.WithFileSystem(std::make_shared()); + } + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, + system_table->NewScan(std::move(context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_FALSE(plan->Splits().empty()); + } + } +} + TEST(TableTest, TestNewScanServesTheFileSystemTheTableWasOpenedWith) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); @@ -1130,7 +1197,7 @@ TEST(TableTest, TestNewWriteCreatesAFileStoreWrite) { auto catalog = std::make_shared(); catalog->SetFileSystem(dir->GetFileSystem()); // A catalog write reads the current schema from the catalog on the main branch. - catalog->SetTableSchema(checked_pointer_cast(table->LatestSchema())); + catalog->SetTableSchema(checked_pointer_cast(table->GetSchema())); ASSERT_OK_AND_ASSIGN( std::unique_ptr ctx, @@ -1149,7 +1216,7 @@ TEST(TableTest, TestNewCommitCreatesAFileStoreCommit) { auto catalog = std::make_shared(); catalog->SetFileSystem(dir->GetFileSystem()); // Like a write, a catalog commit reads the current schema from the catalog. - catalog->SetTableSchema(checked_pointer_cast(table->LatestSchema())); + catalog->SetTableSchema(checked_pointer_cast(table->GetSchema())); ASSERT_OK_AND_ASSIGN( std::unique_ptr ctx, @@ -1181,7 +1248,7 @@ TEST(TableTest, TestNewWriteAndCommitRefuseAContextThatNamesAnotherTable) { auto catalog = std::make_shared(); catalog->SetFileSystem(fs); - catalog->SetTableSchema(checked_pointer_cast(table->LatestSchema())); + catalog->SetTableSchema(checked_pointer_cast(table->GetSchema())); { WriteContextBuilder write_builder(dir->Str(), "commit-user"); write_builder.WithCatalog(catalog, Identifier("db", "other")); @@ -1237,7 +1304,7 @@ TEST(TableTest, TestFactoriesApplyOptionsWithoutContextConfiguration) { auto managed = std::dynamic_pointer_cast(loaded); ASSERT_TRUE(managed); ASSERT_EQ(managed->CurrentBranch(), BranchManager::NormalizeBranch(branch)); - auto effective_schema = std::dynamic_pointer_cast(loaded->LatestSchema()); + auto effective_schema = std::dynamic_pointer_cast(loaded->GetSchema()); ASSERT_TRUE(effective_schema); ASSERT_EQ(effective_schema->Options().at(Options::READ_BATCH_SIZE), "17"); ASSERT_OK(TableScan::Create(std::move(scan_context), options).status()); @@ -1300,7 +1367,7 @@ TEST(TableTest, TestNewCommitRefusesABranchACatalogCannotAddress) { auto catalog = std::make_shared(); catalog->SetFileSystem(fs); - catalog->SetTableSchema(checked_pointer_cast(table->LatestSchema())); + catalog->SetTableSchema(checked_pointer_cast(table->GetSchema())); WriteContextBuilder write_builder(dir->Str(), "commit-user"); write_builder.WithCatalog(catalog, identifier); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_ctx, write_builder.Finish()); @@ -1403,18 +1470,17 @@ TEST(TableTest, TestCopyServesTheTableWithTheGivenOptionsOnTop) { ASSERT_OK_AND_ASSIGN(std::shared_ptr
copied, table->Copy({{Options::BUCKET, "2"}, {Options::TYPE, "TABLE"}})); std::shared_ptr copied_schema = - std::dynamic_pointer_cast(copied->LatestSchema()); + std::dynamic_pointer_cast(copied->GetSchema()); ASSERT_TRUE(copied_schema != nullptr); EXPECT_EQ(copied_schema->Options().at(Options::BUCKET), "2"); EXPECT_EQ(copied_schema->Options().count(Options::TYPE), 0); EXPECT_EQ(copied_schema->Options().at(Options::BUCKET_KEY), "id"); - EXPECT_EQ(copied_schema->Id(), - std::dynamic_pointer_cast(table->LatestSchema())->Id()); + EXPECT_EQ(copied_schema->Id(), std::dynamic_pointer_cast(table->GetSchema())->Id()); EXPECT_EQ(copied->FullName(), table->FullName()); EXPECT_EQ(copied->GetRootPath(), table->GetRootPath()); EXPECT_EQ(copied->GetFileSystem(), table->GetFileSystem()); EXPECT_EQ( - std::dynamic_pointer_cast(table->LatestSchema())->Options().at(Options::BUCKET), + std::dynamic_pointer_cast(table->GetSchema())->Options().at(Options::BUCKET), "1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_ctx, @@ -1424,6 +1490,168 @@ TEST(TableTest, TestCopyServesTheTableWithTheGivenOptionsOnTop) { ASSERT_FALSE(plan->Splits().empty()); } +TEST(TableTest, TestSchemaRemainsBoundUntilTableIsReloaded) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto fs = dir->GetFileSystem(); + const std::string path = dir->Str() + "/db.db/tbl"; + const Identifier identifier("db", "tbl"); + FileSystemCatalog catalog(fs, dir->Str(), {}); + ASSERT_OK(catalog.CreateDatabase("db", {}, false)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
original, CreateManagedTable(fs, path, identifier)); + const std::shared_ptr bound_schema = original->GetSchema(); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr updated, + TableSchema::Create(1, + arrow::schema({arrow::field("id", arrow::int32(), false), + arrow::field("added", arrow::utf8())}), + {}, {}, {})); + ASSERT_OK_AND_ASSIGN(std::string schema_json, updated->ToJsonString()); + SchemaManager schemas(fs, path); + ASSERT_OK(fs->AtomicStore(schemas.SchemaDirectory() + "/schema-1", schema_json)); + + EXPECT_EQ(original->GetSchema(), bound_schema); + EXPECT_EQ(original->GetSchema()->FieldNames(), (std::vector{"id"})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
copied, original->Copy({})); + EXPECT_EQ(copied->GetSchema()->FieldNames(), (std::vector{"id"})); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr
reloaded, catalog.GetTable(identifier)); + EXPECT_EQ(reloaded->GetSchema()->FieldNames(), (std::vector{"id", "added"})); + EXPECT_EQ(std::dynamic_pointer_cast(reloaded->GetSchema())->Id(), 1); + EXPECT_EQ(original->GetSchema(), bound_schema); +} + +TEST(TableTest, TestCopyTimeTravelUsesTargetBranchSchema) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto fs = dir->GetFileSystem(); + const Identifier identifier("db", "tbl"); + const std::map options = {{Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "id"}}; + ASSERT_OK_AND_ASSIGN(std::shared_ptr
main, + CreateManagedTable(fs, dir->Str(), identifier, "main", options)); + WriteAndCommitTwoRows(main, dir->Str()); + SchemaManager dev_schemas(fs, dir->Str(), "dev"); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr dev_schema, + dev_schemas.CreateTable(arrow::schema({arrow::field("id", arrow::int32(), false), + arrow::field("v", arrow::int32())}), + {}, {}, options)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr dev, + FileStoreTable::Create(fs, dir->Str(), std::shared_ptr(std::move(dev_schema)), + identifier, "", "dev", {}, {})); + WriteAndCommitRow(dev, dir->Str(), 7, 70); + + std::map> tables; + int64_t timestamp = 0; + for (const auto& original : {main, std::shared_ptr
(dev)}) { + auto managed = std::dynamic_pointer_cast(original); + const std::string branch = managed->CurrentBranch(); + ASSERT_OK_AND_ASSIGN(std::optional snapshot, + managed->GetSnapshotManager()->LatestSnapshot()); + ASSERT_TRUE(snapshot); + ASSERT_EQ(snapshot->Id(), 1); + timestamp = std::max(timestamp, snapshot->TimeMillis()); + ASSERT_OK_AND_ASSIGN(std::string snapshot_json, snapshot->ToJsonString()); + TagManager tags(fs, dir->Str(), branch); + ASSERT_OK(fs->AtomicStore(tags.TagPath("old"), snapshot_json)); + + auto schema = std::dynamic_pointer_cast(original->GetSchema()); + auto fields = schema->Fields(); + fields.emplace_back(static_cast(fields.size()), + arrow::field("added", arrow::utf8())); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr latest_schema, + TableSchema::Create(1, DataField::ConvertDataFieldsToArrowSchema(fields), {}, {}, + schema->Options())); + ASSERT_OK_AND_ASSIGN(std::string schema_json, latest_schema->ToJsonString()); + SchemaManager schemas(fs, dir->Str(), branch); + ASSERT_OK(fs->AtomicStore(schemas.ToSchemaPath(1), schema_json)); + ASSERT_OK_AND_ASSIGN( + tables[branch], + FileStoreTable::Create(fs, dir->Str(), + std::shared_ptr(std::move(latest_schema)), + identifier, "", branch, {}, {})); + } + + for (const auto& [source_branch, table] : tables) { + for (const std::string& kind : {std::string("managed"), std::string("ro")}) { + std::shared_ptr
source = table; + if (kind == "ro") { + ASSERT_OK_AND_ASSIGN( + source, + SystemTableLoader::Load( + kind, fs, dir->Str(), + std::dynamic_pointer_cast(table->GetSchema()), {}, "tbl")); + } + for (const std::string& named_branch : + {std::string("dev"), std::string("main"), std::string(""), std::string(" \t ")}) { + const std::string target_branch = BranchManager::NormalizeBranch(named_branch); + const bool target_dev = target_branch == "dev"; + const std::vector expected_fields = + target_dev ? std::vector{"id", "v"} + : std::vector{"id"}; + for (const auto& selector : std::vector>{ + {Options::SCAN_SNAPSHOT_ID, "1"}, + {Options::SCAN_TAG_NAME, "old"}, + {Options::SCAN_TIMESTAMP_MILLIS, std::to_string(timestamp)}}) { + SCOPED_TRACE(source_branch + "/" + kind + "/" + named_branch + "/" + + selector.first); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
historical, + source->Copy({{Options::BRANCH, named_branch}, selector})); + ASSERT_EQ(historical->GetSchema()->FieldNames(), expected_fields); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
again, historical->Copy({})); + ASSERT_EQ(again->GetSchema()->FieldNames(), expected_fields); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, + ScanContextBuilder(dir->Str()).Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, + historical->NewScan(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_FALSE(plan->Splits().empty()); + for (bool explicit_projection : {false, true}) { + ReadContextBuilder builder(dir->Str()); + if (explicit_projection) { + builder.SetReadFieldNames(expected_fields); + } + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, + builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, + historical->NewRead(std::move(context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, + read->CreateReader(plan->Splits())); + int64_t rows = 0; + while (true) { + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, reader->NextBatch()); + if (BatchReader::IsEofBatch(batch)) { + break; + } + ASSERT_OK_AND_ASSIGN(std::shared_ptr array, + ReadResultCollector::GetArray(std::move(batch))); + auto values = std::dynamic_pointer_cast(array); + ASSERT_TRUE(values); + ASSERT_TRUE(values->GetFieldByName("id")); + ASSERT_FALSE(values->GetFieldByName("added")); + ASSERT_EQ(values->GetFieldByName("v") != nullptr, target_dev); + if (target_dev) { + auto v = std::dynamic_pointer_cast( + values->GetFieldByName("v")); + ASSERT_TRUE(v); + for (int64_t i = 0; i < v->length(); ++i) { + ASSERT_FALSE(v->IsNull(i)); + ASSERT_EQ(v->Value(i), 70); + } + } + rows += values->length(); + } + ASSERT_EQ(rows, target_dev ? 1 : 2); + } + } + } + } + } +} + TEST(TableTest, TestCopyTimeTravelUsesHistoricalSchema) { for (const std::string& branch : {std::string("main"), std::string("dev")}) { SCOPED_TRACE(branch); @@ -1446,7 +1674,7 @@ TEST(TableTest, TestCopyTimeTravelUsesHistoricalSchema) { // Model a newly added column and its storage declarations in schema 1. auto latest_options = - std::dynamic_pointer_cast(original->LatestSchema())->Options(); + std::dynamic_pointer_cast(original->GetSchema())->Options(); latest_options[Options::BLOB_FIELD] = "payload"; latest_options[Options::FALLBACK_BLOB_DESCRIPTOR_FIELD] = "payload"; ASSERT_OK_AND_ASSIGN( @@ -1467,14 +1695,14 @@ TEST(TableTest, TestCopyTimeTravelUsesHistoricalSchema) { SCOPED_TRACE(selector.first); ASSERT_OK_AND_ASSIGN(std::shared_ptr
historical, latest->Copy({selector, {Options::READ_BATCH_SIZE, "1"}})); - auto schema = std::dynamic_pointer_cast(historical->LatestSchema()); + auto schema = std::dynamic_pointer_cast(historical->GetSchema()); ASSERT_TRUE(schema); EXPECT_EQ(schema->Id(), 0); EXPECT_EQ(schema->FieldNames(), (std::vector{"id"})); EXPECT_EQ(schema->Options().count(Options::BLOB_FIELD), 0); EXPECT_EQ(schema->Options().count(Options::FALLBACK_BLOB_DESCRIPTOR_FIELD), 0); ASSERT_OK_AND_ASSIGN(std::shared_ptr
again, historical->Copy({})); - EXPECT_EQ(std::dynamic_pointer_cast(again->LatestSchema())->Id(), 0); + EXPECT_EQ(std::dynamic_pointer_cast(again->GetSchema())->Id(), 0); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, ScanContextBuilder(dir->Str()).Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, @@ -1501,7 +1729,7 @@ TEST(TableTest, TestCopyTimeTravelUsesHistoricalSchema) { } ASSERT_OK_AND_ASSIGN(std::shared_ptr
unavailable, original->Copy({{Options::SCAN_SNAPSHOT_ID, "999"}})); - EXPECT_EQ(std::dynamic_pointer_cast(unavailable->LatestSchema())->Id(), 0); + EXPECT_EQ(std::dynamic_pointer_cast(unavailable->GetSchema())->Id(), 0); ASSERT_NOK_WITH_MSG( latest->Copy({{Options::SCAN_TAG_NAME, "old"}, {Options::BLOB_FIELD, "payload"}}), "payload"); @@ -1512,8 +1740,8 @@ TEST(TableTest, TestCopyTimeTravelUsesHistoricalSchema) { ASSERT_OK_AND_ASSIGN( std::shared_ptr options_view, SystemTableLoader::Load(OptionsSystemTable::kName, fs, dir->Str(), - std::dynamic_pointer_cast(latest->LatestSchema()), - {}, "tbl")); + std::dynamic_pointer_cast(latest->GetSchema()), {}, + "tbl")); ASSERT_OK_AND_ASSIGN(std::shared_ptr
old_options, options_view->Copy({{Options::SCAN_TAG_NAME, "old"}})); auto old_view = std::dynamic_pointer_cast(old_options); @@ -1590,7 +1818,7 @@ TEST(TableTest, TestCopyProtectsDeletionVectorsMode) { table->Copy({{"deletion-vectors.modifiable", "true"}})); ASSERT_OK_AND_ASSIGN(std::shared_ptr
toggled, modifiable->Copy({{Options::DELETION_VECTORS_ENABLED, changed}})); - EXPECT_EQ(std::dynamic_pointer_cast(toggled->LatestSchema()) + EXPECT_EQ(std::dynamic_pointer_cast(toggled->GetSchema()) ->Options() .at(Options::DELETION_VECTORS_ENABLED), changed); @@ -1621,7 +1849,7 @@ TEST(TableTest, TestSystemTableCopyUsesHistoricalSchemaForOperators) { std::shared_ptr(std::move(schema)), Identifier("db", "tbl"), "", branch, {}, {})); WriteAndCommitRow(original, dir->Str(), 1, 10); - options = std::dynamic_pointer_cast(original->LatestSchema())->Options(); + options = std::dynamic_pointer_cast(original->GetSchema())->Options(); ASSERT_OK_AND_ASSIGN( std::unique_ptr latest, TableSchema::Create(1, @@ -1705,7 +1933,7 @@ TEST(TableTest, TestCopyRefusesThePhysicalOptionsTheDataWasLaidOutWith) { table->Copy({{Options::BUCKET, "2"}})); ASSERT_OK_AND_ASSIGN(std::shared_ptr
normalized, table->Copy({{"primary-key", " , "}, {"partition", " "}})); - auto normalized_schema = std::dynamic_pointer_cast(normalized->LatestSchema()); + auto normalized_schema = std::dynamic_pointer_cast(normalized->GetSchema()); ASSERT_TRUE(normalized_schema); EXPECT_EQ(normalized_schema->Options().count("primary-key"), 0); EXPECT_EQ(normalized_schema->Options().count("partition"), 0); @@ -1806,7 +2034,7 @@ TEST(TableTest, TestCreateBuildsTheOperator) { auto catalog = std::make_shared(); catalog->SetFileSystem(dir->GetFileSystem()); // A catalog write and commit read the current schema from the catalog on the main branch. - catalog->SetTableSchema(checked_pointer_cast(table->LatestSchema())); + catalog->SetTableSchema(checked_pointer_cast(table->GetSchema())); // `TableScan::Create()` and friends build the operator straight from the context, which // carries the catalog and identifier it resolves the table through. @@ -1842,10 +2070,10 @@ TEST(TableTest, TestCreateWithSpecificTableSchemaSkipsTheCatalogSchema) { auto catalog = std::make_shared(); catalog->SetFileSystem(dir->GetFileSystem()); - catalog->SetTableSchema(checked_pointer_cast(table->LatestSchema())); + catalog->SetTableSchema(checked_pointer_cast(table->GetSchema())); // An explicit schema bypasses catalog schema loading on main. - ASSERT_OK_AND_ASSIGN(std::string schema_json, table->LatestSchema()->GetJsonSchema()); + ASSERT_OK_AND_ASSIGN(std::string schema_json, table->GetSchema()->GetJsonSchema()); WriteContextBuilder write_builder(dir->Str(), "commit-user"); ASSERT_OK_AND_ASSIGN( @@ -1873,7 +2101,7 @@ TEST(TableTest, TestCreateWithCatalogReadsTheMainBranchSchemaFromIt) { auto catalog = std::make_shared(); catalog->SetFileSystem(dir->GetFileSystem()); - catalog->SetTableSchema(checked_pointer_cast(table->LatestSchema())); + catalog->SetTableSchema(checked_pointer_cast(table->GetSchema())); // Without an explicit schema, each operator loads the catalog schema once. ScanContextBuilder scan_builder(dir->Str()); diff --git a/src/paimon/core/utils/context_utils.h b/src/paimon/core/utils/context_utils.h index 5a845e99d..f3a3cb85e 100644 --- a/src/paimon/core/utils/context_utils.h +++ b/src/paimon/core/utils/context_utils.h @@ -18,7 +18,11 @@ #pragma once +#include + #include "fmt/format.h" +#include "paimon/catalog/catalog.h" +#include "paimon/result.h" #include "paimon/status.h" namespace paimon { @@ -29,6 +33,18 @@ class ContextUtils { ContextUtils() = delete; ~ContextUtils() = delete; + template + static Result> ResolveFileSystemForLoad(const Context& context) { + if (context.GetSpecificFileSystem() != nullptr || context.GetCatalog() == nullptr || + !context.GetFileSystemSchemeToIdentifierMap().empty()) { + return context.GetSpecificFileSystem(); + } + if (!context.GetIdentifier()) { + return Status::Invalid("a catalog requires a table identifier"); + } + return context.GetCatalog()->GetTableFileSystem(context.GetIdentifier().value()); + } + /// Requires a context, memory pool and executor; kind labels errors. template static Status CheckOperatorContext(const ContextPointer& context, const char* kind) { diff --git a/src/paimon/rest/rest_catalog.cpp b/src/paimon/rest/rest_catalog.cpp index b8101dddf..a4481df3d 100644 --- a/src/paimon/rest/rest_catalog.cpp +++ b/src/paimon/rest/rest_catalog.cpp @@ -391,7 +391,7 @@ Result> RestCatalog::LoadTableSchema(const Identifier& i PAIMON_ASSIGN_OR_RAISE( std::shared_ptr system_table, GlobalSystemTableLoader::LoadForSchema(identifier.GetObjectName(), context)); - return system_table->LatestSchema(); + return system_table->GetSchema(); } PAIMON_ASSIGN_OR_RAISE(bool is_system_table, identifier.IsSystemTable()); PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); @@ -403,7 +403,7 @@ Result> RestCatalog::LoadTableSchema(const Identifier& i if (table_path != nullptr) { *table_path = system_table->GetRootPath(); } - return system_table->LatestSchema(); + return system_table->GetSchema(); } PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, LoadDataTableSchema(load_identifier, branch, table_path, table_id)); @@ -433,7 +433,8 @@ Result> RestCatalog::LoadSystemTable( Identifier data_identifier, CatalogUtils::BranchIdentifier(identifier, branch.value_or(std::string()))); return SystemTableLoader::Load(system_table_name.value(), table_file_system, data_table_path, - latest_schema, dynamic_options, data_identifier.GetObjectName()); + latest_schema, dynamic_options, data_identifier.GetObjectName(), + fs_scheme_to_identifier_map_); } Result> RestCatalog::GetTable(const Identifier& identifier) const { @@ -489,7 +490,7 @@ Result> RestCatalog::GetTable(const Identifier& identifie GetTableFileSystem(identifier)); return FileStoreTable::Create(table_file_system, table_path, schema, identifier, uuid, branch.value_or(Identifier::kDefaultMainBranch), - /*fs_scheme_map=*/{}, /*dynamic_options=*/{}); + fs_scheme_to_identifier_map_, /*dynamic_options=*/{}); } std::string RestCatalog::GetRootPath() const { diff --git a/src/paimon/rest/rest_catalog_test.cpp b/src/paimon/rest/rest_catalog_test.cpp index 7b6bf381c..6e9f420f6 100644 --- a/src/paimon/rest/rest_catalog_test.cpp +++ b/src/paimon/rest/rest_catalog_test.cpp @@ -927,8 +927,7 @@ TEST_F(RestCatalogTest, TableOperations) { std::lock_guard lock(state_->mutex); ASSERT_EQ(requests_before_get_table + 1, state_->get_table_requests); } - std::shared_ptr schema = - std::dynamic_pointer_cast(table->LatestSchema()); + std::shared_ptr schema = std::dynamic_pointer_cast(table->GetSchema()); ASSERT_NE(nullptr, schema); ASSERT_EQ((std::vector{"f0", "f1"}), schema->FieldNames()); ASSERT_EQ((std::vector{"f0"}), schema->PrimaryKeys()); @@ -946,7 +945,7 @@ TEST_F(RestCatalogTest, TableOperations) { // timeMillis is backed by the server's audit "updatedAt" instead of the current // time, keeping the conversion deterministic std::shared_ptr table_schema = - std::dynamic_pointer_cast(table->LatestSchema()); + std::dynamic_pointer_cast(table->GetSchema()); ASSERT_NE(nullptr, table_schema); ASSERT_EQ(123, table_schema->TimeMillis()); @@ -968,7 +967,7 @@ TEST_F(RestCatalogTest, TableOperations) { } ASSERT_OK_AND_ASSIGN(std::shared_ptr
commented_table, catalog->GetTable(Identifier("db1", "commented"))); - ASSERT_EQ("a table comment", commented_table->LatestSchema()->Comment().value_or("")); + ASSERT_EQ("a table comment", commented_table->GetSchema()->Comment().value_or("")); ASSERT_OK(catalog->RenameTable(identifier, Identifier("db1", "t2"), /*ignore_if_not_exists=*/false)); @@ -1073,7 +1072,7 @@ TEST_F(RestCatalogTest, SystemTableSchema) { std::shared_ptr system_table = std::dynamic_pointer_cast(table); ASSERT_NE(nullptr, system_table); ASSERT_EQ("t1$options", system_table->Name()); - ASSERT_EQ((std::vector{"key", "value"}), table->LatestSchema()->FieldNames()); + ASSERT_EQ((std::vector{"key", "value"}), table->GetSchema()->FieldNames()); ASSERT_EQ("t1$options", table->Uuid()); { std::lock_guard lock(state_->mutex); @@ -1116,8 +1115,7 @@ TEST_F(RestCatalogTest, BranchTableLoadsBranchSchemaFromServer) { ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog->GetTable(branch_identifier)); ASSERT_EQ("t1$branch_b1", table->Name()); - std::shared_ptr schema = - std::dynamic_pointer_cast(table->LatestSchema()); + std::shared_ptr schema = std::dynamic_pointer_cast(table->GetSchema()); ASSERT_NE(nullptr, schema); // the branch's own schema and schema id, not the main table's ASSERT_EQ((std::vector{"b0"}), schema->FieldNames()); @@ -1129,7 +1127,7 @@ TEST_F(RestCatalogTest, BranchTableLoadsBranchSchemaFromServer) { ASSERT_OK_AND_ASSIGN(std::shared_ptr
main_table, catalog->GetTable(Identifier("db1", "t1$branch_main"))); std::shared_ptr main_schema = - std::dynamic_pointer_cast(main_table->LatestSchema()); + std::dynamic_pointer_cast(main_table->GetSchema()); ASSERT_NE(nullptr, main_schema); ASSERT_EQ((std::vector{"f0", "f1"}), main_schema->FieldNames()); ASSERT_EQ(0, main_schema->Options().count(Options::BRANCH)); @@ -1139,7 +1137,7 @@ TEST_F(RestCatalogTest, BranchTableLoadsBranchSchemaFromServer) { ASSERT_OK_AND_ASSIGN(std::shared_ptr
main_case_table, catalog->GetTable(Identifier("db1", "t1$branch_MAIN"))); std::shared_ptr main_case_schema = - std::dynamic_pointer_cast(main_case_table->LatestSchema()); + std::dynamic_pointer_cast(main_case_table->GetSchema()); ASSERT_NE(nullptr, main_case_schema); ASSERT_EQ((std::vector{"f0", "f1"}), main_case_schema->FieldNames()); ASSERT_EQ(0, main_case_schema->Options().count(Options::BRANCH)); @@ -1395,8 +1393,7 @@ TEST_F(RestCatalogTest, NestedSchemaHighestFieldId) { ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog->GetTable(Identifier("db1", "nested"))); - std::shared_ptr schema = - std::dynamic_pointer_cast(table->LatestSchema()); + std::shared_ptr schema = std::dynamic_pointer_cast(table->GetSchema()); ASSERT_NE(nullptr, schema); ASSERT_EQ(5, schema->Id()); // 9 lives inside the value row of the map: ROW, ARRAY element and MAP key/value @@ -1490,8 +1487,7 @@ TEST_F(RestCatalogTest, PartitionKeysRoundTrip) { /*ignore_if_exists=*/false)); // the partition keys survive both the create request and the load response conversion ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog->GetTable(Identifier("db1", "pt"))); - std::shared_ptr loaded = - std::dynamic_pointer_cast(table->LatestSchema()); + std::shared_ptr loaded = std::dynamic_pointer_cast(table->GetSchema()); ASSERT_NE(nullptr, loaded); ASSERT_EQ((std::vector{"f1"}), loaded->PartitionKeys()); } diff --git a/src/paimon/testing/utils/test_helper.h b/src/paimon/testing/utils/test_helper.h index 0b9c2dd91..ad996cd6a 100644 --- a/src/paimon/testing/utils/test_helper.h +++ b/src/paimon/testing/utils/test_helper.h @@ -331,7 +331,7 @@ class TestHelper { return commit_impl->snapshot_manager_->LatestSnapshotOfUser(commit_user_); } - Result>> LatestSchema() const { + Result>> GetSchema() const { auto commit_impl = dynamic_cast(commit_.get()); return commit_impl->schema_manager_->Latest(); } diff --git a/test/inte/append_compaction_inte_test.cpp b/test/inte/append_compaction_inte_test.cpp index 70ec813e7..dd6cff70a 100644 --- a/test/inte/append_compaction_inte_test.cpp +++ b/test/inte/append_compaction_inte_test.cpp @@ -224,7 +224,7 @@ TEST_P(AppendCompactionInteTest, TestAppendTableStreamWriteFullCompaction) { auto helper, TestHelper::Create(dir->Str(), schema, partition_keys, primary_keys, options, /*is_streaming_mode=*/true)); ASSERT_OK_AND_ASSIGN(std::optional> table_schema, - helper->LatestSchema()); + helper->GetSchema()); ASSERT_TRUE(table_schema); auto gen = std::make_shared(table_schema.value(), pool_); int64_t commit_identifier = 0; @@ -551,7 +551,7 @@ TEST_P(AppendCompactionInteTest, TestAppendTableStreamWriteFullCompactionWithDv) auto helper, TestHelper::Create(dir->Str(), schema, partition_keys, primary_keys, options, /*is_streaming_mode=*/true)); ASSERT_OK_AND_ASSIGN(std::optional> table_schema, - helper->LatestSchema()); + helper->GetSchema()); ASSERT_TRUE(table_schema); auto gen = std::make_shared(table_schema.value(), pool_); int64_t commit_identifier = 0; @@ -637,7 +637,7 @@ TEST_P(AppendCompactionInteTest, TestAppendTableStreamWriteBestEffortCompaction) auto helper, TestHelper::Create(dir->Str(), schema, partition_keys, primary_keys, options, /*is_streaming_mode=*/true)); ASSERT_OK_AND_ASSIGN(std::optional> table_schema, - helper->LatestSchema()); + helper->GetSchema()); ASSERT_TRUE(table_schema); auto gen = std::make_shared(table_schema.value(), pool_); int64_t commit_identifier = 0; @@ -728,7 +728,7 @@ TEST_P(AppendCompactionInteTest, TestAppendTableStreamWriteCompactionWithExterna auto helper, TestHelper::Create(dir->Str(), schema, partition_keys, primary_keys, options, /*is_streaming_mode=*/true)); ASSERT_OK_AND_ASSIGN(std::optional> table_schema, - helper->LatestSchema()); + helper->GetSchema()); ASSERT_TRUE(table_schema); auto gen = std::make_shared(table_schema.value(), pool_); int64_t commit_identifier = 0; @@ -816,7 +816,7 @@ TEST_F(AppendCompactionInteTest, TestAppendTableCompactionWithIOException) { TestHelper::Create(dir->Str(), schema, partition_keys, primary_keys, options, /*is_streaming_mode=*/true)); ASSERT_OK_AND_ASSIGN(std::optional> table_schema, - helper->LatestSchema()); + helper->GetSchema()); ASSERT_TRUE(table_schema); auto gen = std::make_shared(table_schema.value(), pool_); diff --git a/test/inte/pk_compaction_inte_test.cpp b/test/inte/pk_compaction_inte_test.cpp index 28987f6f1..c3f96b258 100644 --- a/test/inte/pk_compaction_inte_test.cpp +++ b/test/inte/pk_compaction_inte_test.cpp @@ -3203,7 +3203,7 @@ TEST_F(PkCompactionInteTest, TestKeyValueTableCompactionWithIOException) { TestHelper::Create(dir->Str(), schema, partition_keys, primary_keys, options, /*is_streaming_mode=*/true)); ASSERT_OK_AND_ASSIGN(std::optional> table_schema, - helper->LatestSchema()); + helper->GetSchema()); ASSERT_TRUE(table_schema); auto gen = std::make_shared(table_schema.value(), pool_); @@ -3265,7 +3265,7 @@ TEST_P(PkCompactionFormatInteTest, TestKeyValueTableStreamWriteFullCompaction) { auto helper, TestHelper::Create(dir_->Str(), schema, partition_keys, primary_keys, options, /*is_streaming_mode=*/true)); ASSERT_OK_AND_ASSIGN(std::optional> table_schema, - helper->LatestSchema()); + helper->GetSchema()); ASSERT_TRUE(table_schema); auto gen = std::make_shared(table_schema.value(), pool_); int64_t commit_identifier = 0; diff --git a/test/inte/write_inte_test.cpp b/test/inte/write_inte_test.cpp index f21ac104d..7e915e708 100644 --- a/test/inte/write_inte_test.cpp +++ b/test/inte/write_inte_test.cpp @@ -687,7 +687,7 @@ TEST_P(WriteInteTest, TestAppendTableStreamWriteWithPartitionAndMultiBuckets) { /*is_streaming_mode=*/true)); int64_t commit_identifier = 0; ASSERT_OK_AND_ASSIGN(std::optional> table_schema, - helper->LatestSchema()); + helper->GetSchema()); ASSERT_TRUE(table_schema); DataGenerator gen(table_schema.value(), pool_); @@ -997,7 +997,7 @@ TEST_P(WriteInteTest, TestPkTableStreamWrite) { /*is_streaming_mode=*/true)); int64_t commit_identifier = 0; ASSERT_OK_AND_ASSIGN(std::optional> table_schema, - helper->LatestSchema()); + helper->GetSchema()); ASSERT_TRUE(table_schema); // round 1 write @@ -1275,7 +1275,7 @@ TEST_P(WriteInteTest, TestPkTableBatchWrite) { /*is_streaming_mode=*/false)); int64_t commit_identifier = 0; ASSERT_OK_AND_ASSIGN(std::optional> table_schema, - helper->LatestSchema()); + helper->GetSchema()); ASSERT_TRUE(table_schema); DataGenerator gen(table_schema.value(), pool_); @@ -1438,7 +1438,7 @@ TEST_P(WriteInteTest, TestPkTableWriteWithNoPartitionKey) { /*is_streaming_mode=*/true)); int64_t commit_identifier = 0; ASSERT_OK_AND_ASSIGN(std::optional> table_schema, - helper->LatestSchema()); + helper->GetSchema()); ASSERT_TRUE(table_schema); DataGenerator gen(table_schema.value(), pool_); @@ -1853,7 +1853,7 @@ TEST_P(WriteInteTest, TestPkTableForceLookup) { /*is_streaming_mode=*/true)); int64_t commit_identifier = 0; ASSERT_OK_AND_ASSIGN(std::optional> table_schema, - helper->LatestSchema()); + helper->GetSchema()); ASSERT_TRUE(table_schema); DataGenerator gen(table_schema.value(), pool_); @@ -1917,7 +1917,7 @@ TEST_P(WriteInteTest, TestPkTableEnableDeletionVector) { /*is_streaming_mode=*/true)); int64_t commit_identifier = 0; ASSERT_OK_AND_ASSIGN(std::optional> table_schema, - helper->LatestSchema()); + helper->GetSchema()); ASSERT_TRUE(table_schema); DataGenerator gen(table_schema.value(), pool_); @@ -3515,7 +3515,7 @@ TEST_P(WriteInteTest, TestPkTablePostponeBucket) { /*is_streaming_mode=*/false)); int64_t commit_identifier = 0; ASSERT_OK_AND_ASSIGN(std::optional> table_schema, - helper->LatestSchema()); + helper->GetSchema()); ASSERT_TRUE(table_schema); std::string data = R"([