From 9d3a6fb6fcb42e1bda97e7ad9c64c52c0e2f05dd Mon Sep 17 00:00:00 2001 From: kingjaiteh Date: Mon, 7 Sep 2026 17:56:50 -0400 Subject: [PATCH 1/2] fix(duckdb): do not cascade table/view drops in DuckLake catalogs DuckLake does not implement `DROP TABLE ... CASCADE` or `DROP VIEW ... CASCADE` and raises "Cascade Drop not supported in DuckLake". Since #5133 the janitor always deletes snapshot tables with cascade=True, so on a DuckLake catalog it can never reclaim space and silently reports "Cleanup complete." until a snapshot expires. Cascade support is declared per engine adapter, but on DuckDB it varies per attached catalog. Resolve the target catalog's type from duckdb_databases() and omit CASCADE for TABLE/VIEW drops in DuckLake catalogs. SCHEMA cascade is still supported by DuckLake and is left untouched, as is behaviour for native DuckDB catalogs. Fixes #6032 Signed-off-by: kingjaiteh --- sqlmesh/core/engine_adapter/duckdb.py | 39 +++++++++++++----- tests/core/engine_adapter/test_duckdb.py | 51 ++++++++++++++++++++++++ 2 files changed, 81 insertions(+), 9 deletions(-) diff --git a/sqlmesh/core/engine_adapter/duckdb.py b/sqlmesh/core/engine_adapter/duckdb.py index ebfcaa7901..11a2cb2a8e 100644 --- a/sqlmesh/core/engine_adapter/duckdb.py +++ b/sqlmesh/core/engine_adapter/duckdb.py @@ -173,16 +173,8 @@ def _create_table( track_rows_processed: bool = True, **kwargs: t.Any, ) -> None: - catalog = self.get_current_catalog() - catalog_type_tuple = self.fetchone( - exp.select("type") - .from_("duckdb_databases()") - .where(exp.column("database_name").eq(catalog)) - ) - catalog_type = catalog_type_tuple[0] if catalog_type_tuple else None - partitioned_by_exps = None - if catalog_type == "ducklake": + if self._get_catalog_type(self.get_current_catalog()) == "ducklake": partitioned_by_exps = kwargs.pop("partitioned_by", None) super()._create_table( @@ -215,6 +207,35 @@ def _create_table( ) self.execute(f"ALTER TABLE {table_name_str} SET PARTITIONED BY ({partitioned_by_str});") + def _drop_object( + self, + name: TableName | SchemaName, + exists: bool = True, + kind: str = "TABLE", + cascade: bool = False, + **drop_args: t.Any, + ) -> None: + # DuckLake catalogs do not implement DROP TABLE / DROP VIEW ... CASCADE and raise + # "Cascade Drop not supported in DuckLake". Views in DuckDB are late-binding, so + # dropping the underlying table without CASCADE is safe there. + if cascade and kind.upper() in ("TABLE", "VIEW"): + catalog = exp.to_table(name).catalog or self.get_current_catalog() + if self._get_catalog_type(catalog) == "ducklake": + cascade = False + + super()._drop_object(name=name, exists=exists, kind=kind, cascade=cascade, **drop_args) + + def _get_catalog_type(self, catalog: t.Optional[str]) -> t.Optional[str]: + """Returns the type of the given catalog (e.g. 'duckdb', 'ducklake') as reported by duckdb_databases().""" + if not catalog: + return None + catalog_type_tuple = self.fetchone( + exp.select("type") + .from_("duckdb_databases()") + .where(exp.column("database_name").eq(catalog)) + ) + return catalog_type_tuple[0] if catalog_type_tuple else None + @property def _is_motherduck(self) -> bool: return self._extra_config.get("is_motherduck", False) diff --git a/tests/core/engine_adapter/test_duckdb.py b/tests/core/engine_adapter/test_duckdb.py index 9fd65a6e66..b4760788fa 100644 --- a/tests/core/engine_adapter/test_duckdb.py +++ b/tests/core/engine_adapter/test_duckdb.py @@ -154,3 +154,54 @@ def test_ducklake_partitioning(adapter: EngineAdapter, duck_conn, tmp_path): f"SELECT * FROM __ducklake_metadata_{catalog}.main.ducklake_partition_info" ).fetchdf() assert partition_info.shape[0] == 1 + + +def test_drop_table_ducklake_no_cascade(adapter: EngineAdapter, duck_conn, tmp_path): + # DuckLake does not implement DROP TABLE/VIEW ... CASCADE, so the adapter must + # omit CASCADE for objects in a DuckLake catalog while keeping it for native catalogs. + catalog = "a_ducklake_db" + + duck_conn.install_extension("ducklake") + duck_conn.load_extension("ducklake") + duck_conn.execute( + f"ATTACH 'ducklake:{tmp_path}/{catalog}.ducklake' AS {catalog} (DATA_PATH '{tmp_path}');" + ) + + duck_conn.execute(f"CREATE SCHEMA {catalog}.phys") + duck_conn.execute(f"CREATE SCHEMA {catalog}.virt") + duck_conn.execute(f"CREATE TABLE {catalog}.phys.t (i INTEGER)") + duck_conn.execute(f"CREATE VIEW {catalog}.virt.v AS SELECT * FROM {catalog}.phys.t") + + # native catalog, cascade is passed through + duck_conn.execute("CREATE TABLE memory.main.native_t (i INTEGER)") + duck_conn.execute("CREATE VIEW memory.main.native_v AS SELECT * FROM memory.main.native_t") + + adapter.drop_table(f"{catalog}.phys.t", cascade=True) + adapter.drop_view(f"{catalog}.virt.v", cascade=True) + adapter.drop_table("memory.main.native_t", cascade=True) + adapter.drop_view("memory.main.native_v", cascade=True) + + assert not adapter.table_exists(f"{catalog}.phys.t") + assert not adapter.table_exists(f"{catalog}.virt.v") + assert not adapter.table_exists("memory.main.native_t") + assert not adapter.table_exists("memory.main.native_v") + + +def test_drop_object_cascade_by_catalog_type(make_mocked_engine_adapter: t.Callable): + adapter = make_mocked_engine_adapter(DuckDBEngineAdapter) + adapter.fetchone = lambda *_args, **_kwargs: ("ducklake",) # type: ignore + + adapter.drop_table("lake.phys.t", cascade=True) + adapter.drop_view("lake.virt.v", cascade=True) + # schema cascade is supported by DuckLake and must be preserved + adapter.drop_schema("lake.virt", cascade=True) + + adapter.fetchone = lambda *_args, **_kwargs: ("duckdb",) # type: ignore + adapter.drop_table("native.phys.t", cascade=True) + + assert to_sql_calls(adapter) == [ + 'DROP TABLE IF EXISTS "lake"."phys"."t"', + 'DROP VIEW IF EXISTS "lake"."virt"."v"', + 'DROP SCHEMA IF EXISTS "lake"."virt" CASCADE', + 'DROP TABLE IF EXISTS "native"."phys"."t" CASCADE', + ] From 938ef89ecf058453c79a59597b6e6182119ec2d9 Mon Sep 17 00:00:00 2001 From: kingjaiteh Date: Sat, 3 Oct 2026 10:50:00 -0400 Subject: [PATCH 2/2] fix(duckdb): match catalog names case-insensitively in _get_catalog_type Catalog aliases keep their case in duckdb_databases() (e.g. MyLake), but SQLMesh lowercases table names, so the DuckLake lookup missed and CASCADE was emitted again. Signed-off-by: kingjaiteh --- sqlmesh/core/engine_adapter/duckdb.py | 6 +++++- tests/core/engine_adapter/test_duckdb.py | 21 +++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/sqlmesh/core/engine_adapter/duckdb.py b/sqlmesh/core/engine_adapter/duckdb.py index 11a2cb2a8e..3666a801dd 100644 --- a/sqlmesh/core/engine_adapter/duckdb.py +++ b/sqlmesh/core/engine_adapter/duckdb.py @@ -232,7 +232,11 @@ def _get_catalog_type(self, catalog: t.Optional[str]) -> t.Optional[str]: catalog_type_tuple = self.fetchone( exp.select("type") .from_("duckdb_databases()") - .where(exp.column("database_name").eq(catalog)) + .where( + exp.func("LOWER", exp.column("database_name")).eq( + exp.func("LOWER", exp.Literal.string(catalog)) + ) + ) ) return catalog_type_tuple[0] if catalog_type_tuple else None diff --git a/tests/core/engine_adapter/test_duckdb.py b/tests/core/engine_adapter/test_duckdb.py index b4760788fa..696689219b 100644 --- a/tests/core/engine_adapter/test_duckdb.py +++ b/tests/core/engine_adapter/test_duckdb.py @@ -187,6 +187,27 @@ def test_drop_table_ducklake_no_cascade(adapter: EngineAdapter, duck_conn, tmp_p assert not adapter.table_exists("memory.main.native_v") +def test_drop_table_ducklake_no_cascade_mixed_case_alias( + adapter: EngineAdapter, duck_conn, tmp_path +): + # Catalog aliases keep their case in duckdb_databases(), but SQLMesh normalizes + # table names to lowercase, so the catalog type lookup must be case-insensitive. + duck_conn.install_extension("ducklake") + duck_conn.load_extension("ducklake") + duck_conn.execute( + f"ATTACH 'ducklake:{tmp_path}/my_lake.ducklake' AS \"MyLake\" (DATA_PATH '{tmp_path}');" + ) + duck_conn.execute("CREATE SCHEMA MyLake.phys") + duck_conn.execute("CREATE TABLE MyLake.phys.t (i INTEGER)") + duck_conn.execute("CREATE VIEW MyLake.phys.v AS SELECT * FROM MyLake.phys.t") + + adapter.drop_table("mylake.phys.t", cascade=True) + adapter.drop_view("mylake.phys.v", cascade=True) + + assert not adapter.table_exists("mylake.phys.t") + assert not adapter.table_exists("mylake.phys.v") + + def test_drop_object_cascade_by_catalog_type(make_mocked_engine_adapter: t.Callable): adapter = make_mocked_engine_adapter(DuckDBEngineAdapter) adapter.fetchone = lambda *_args, **_kwargs: ("ducklake",) # type: ignore