ducklake integration

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "third_party/duckdb"]
 	path = third_party/duckdb
 	url = https://github.com/duckdb/duckdb.git
+[submodule "third_party/ducklake"]
+	path = third_party/ducklake
+	url = https://github.com/duckdb/ducklake.git

Makefile

@@ -5,6 +5,8 @@ EXTENSION = pg_duckdb
 DATA = pg_duckdb.control $(wildcard sql/pg_duckdb--*.sql)
 
 SRCS = $(wildcard src/*.cpp src/*/*.cpp)
+DUCKLAKE_SRCS_DIR = third_party/ducklake/src
+SRCS += $(wildcard $(DUCKLAKE_SRCS_DIR)/common/*.cpp $(DUCKLAKE_SRCS_DIR)/functions/*.cpp $(DUCKLAKE_SRCS_DIR)/storage/*.cpp $(DUCKLAKE_SRCS_DIR)/metadata_manager/*.cpp)
 OBJS = $(subst .cpp,.o, $(SRCS))
 
 C_SRCS = $(wildcard src/*.c src/*/*.c)
@@ -60,7 +62,7 @@ endif
 
 COMPILER_FLAGS=-Wno-sign-compare -Wshadow -Wswitch -Wunused-parameter -Wunreachable-code -Wno-unknown-pragmas -Wall -Wextra ${ERROR_ON_WARNING}
 
-override PG_CPPFLAGS += -Iinclude -isystem third_party/duckdb/src/include -isystem third_party/duckdb/third_party/re2 -isystem $(INCLUDEDIR_SERVER) ${COMPILER_FLAGS}
+override PG_CPPFLAGS += -Iinclude -I$(DUCKLAKE_SRCS_DIR)/include -isystem third_party/duckdb/src/include -isystem third_party/duckdb/third_party/re2 -isystem $(INCLUDEDIR_SERVER) ${COMPILER_FLAGS}
 override PG_CXXFLAGS += -std=c++17 ${DUCKDB_BUILD_CXX_FLAGS} ${COMPILER_FLAGS} -Wno-register -Weffc++
 # Ignore declaration-after-statement warnings in our code. Postgres enforces
 # this because their ancient style guide requires it, but we don't care. It
@@ -112,7 +114,7 @@ schedulecheck:
 duckdb: $(FULL_DUCKDB_LIB)
 
 .git/modules/third_party/duckdb/HEAD:
-	git submodule update --init --recursive
+	git submodule update --init --recursive --depth 1
 
 $(FULL_DUCKDB_LIB): .git/modules/third_party/duckdb/HEAD third_party/pg_duckdb_extensions.cmake
 	OVERRIDE_GIT_DESCRIBE=$(DUCKDB_VERSION) \

include/pgduckdb/pgduckdb_ruleutils.h

@@ -15,8 +15,9 @@ char *pgduckdb_get_tabledef(Oid relation_id);
 char *pgduckdb_get_alter_tabledef(Oid relation_oid, AlterTableStmt *alter_stmt);
 char *pgduckdb_get_rename_tabledef(Oid relation_oid, RenameStmt *rename_stmt);
 bool pgduckdb_is_not_default_expr(Node *node, void *context);
-List *pgduckdb_db_and_schema(const char *postgres_schema_name, bool is_duckdb_table);
-const char *pgduckdb_db_and_schema_string(const char *postgres_schema_name, bool is_duckdb_table);
+List *pgduckdb_db_and_schema(const char *postgres_schema_name, bool is_duckdb_table, bool is_ducklake_table);
+const char *pgduckdb_db_and_schema_string(const char *postgres_schema_name, bool is_duckdb_table,
+                                          bool is_ducklake_table);
 bool pgduckdb_is_duckdb_row(Oid type_oid);
 bool pgduckdb_is_unresolved_type(Oid type_oid);
 bool pgduckdb_is_fake_type(Oid type_oid);

include/pgducklake/pgducklake_catalog.hpp

@@ -0,0 +1,27 @@
+#pragma once
+
+#include "storage/ducklake_catalog.hpp"
+
+#include "pgduckdb/utility/cpp_only_file.hpp" // Must be last include.
+
+namespace pgduckdb {
+
+class PgDuckLakeCatalog : public duckdb::DuckLakeCatalog {
+public:
+	PgDuckLakeCatalog(duckdb::AttachedDatabase &db_p, duckdb::DuckLakeOptions options_p)
+	    : duckdb::DuckLakeCatalog(db_p, std::move(options_p)) {
+	}
+
+	void Initialize(duckdb::optional_ptr<duckdb::ClientContext> context, bool load_builtin) override;
+
+	duckdb::optional_ptr<duckdb::SchemaCatalogEntry> LookupSchema(duckdb::CatalogTransaction transaction,
+	                                                              const duckdb::EntryLookupInfo &schema_lookup,
+	                                                              duckdb::OnEntryNotFound if_not_found) override;
+
+private:
+	std::mutex schemas_lock;
+	//! Map of schema index -> schema
+	std::unordered_map<duckdb::idx_t, duckdb::unique_ptr<duckdb::DuckLakeCatalogSet>> schemas;
+};
+
+} // namespace pgduckdb

include/pgducklake/pgducklake_handler.hpp

@@ -0,0 +1,10 @@
+#pragma once
+
+#include "pgduckdb/pg/declarations.hpp"
+
+namespace pgduckdb {
+
+bool IsDuckLakeTable(Relation relation);
+bool IsDuckLakeTable(Oid oid);
+
+} // namespace pgduckdb

include/pgducklake/pgducklake_metadata_manager.hpp

@@ -0,0 +1,46 @@
+#pragma once
+
+#include "pgduckdb/pg/declarations.hpp"
+
+#include "storage/ducklake_metadata_info.hpp"
+#include "storage/ducklake_metadata_manager.hpp"
+
+namespace pgduckdb {
+
+class PgDuckLakeMetadataManager : public duckdb::DuckLakeMetadataManager {
+public:
+	PgDuckLakeMetadataManager(duckdb::DuckLakeTransaction &transaction);
+
+	duckdb::DuckLakeCatalogInfo GetCatalogForSnapshot(duckdb::DuckLakeSnapshot snapshot) override;
+	duckdb::vector<duckdb::DuckLakeGlobalStatsInfo> GetGlobalTableStats(duckdb::DuckLakeSnapshot snapshot) override;
+	duckdb::vector<duckdb::DuckLakeFileListEntry> GetFilesForTable(duckdb::DuckLakeTableEntry &table_entry,
+	                                                               duckdb::DuckLakeSnapshot snapshot,
+	                                                               const duckdb::string &filter) override;
+	duckdb::unique_ptr<duckdb::DuckLakeSnapshot> GetSnapshot() override;
+	duckdb::unique_ptr<duckdb::DuckLakeSnapshot> GetSnapshot(duckdb::BoundAtClause &at_clause) override;
+
+	void WriteNewSchemas(duckdb::DuckLakeSnapshot commit_snapshot,
+	                     const duckdb::vector<duckdb::DuckLakeSchemaInfo> &new_schemas);
+	void WriteNewTables(duckdb::DuckLakeSnapshot commit_snapshot,
+	                    const duckdb::vector<duckdb::DuckLakeTableInfo> &new_tables) override;
+
+	void InsertSnapshot(duckdb::DuckLakeSnapshot commit_snapshot) override;
+	void WriteSnapshotChanges(duckdb::DuckLakeSnapshot commit_snapshot,
+	                          const duckdb::SnapshotChangeInfo &change_info) override;
+	void WriteNewDataFiles(duckdb::DuckLakeSnapshot commit_snapshot,
+	                       const duckdb::vector<duckdb::DuckLakeFileInfo> &new_files) override;
+	void UpdateGlobalTableStats(const duckdb::DuckLakeGlobalStatsInfo &stats) override;
+
+	bool GetDuckLakeTableInfo(const duckdb::DuckLakeSnapshot &snapshot, duckdb::DuckLakeSchemaEntry &schema,
+	                          duckdb::DuckLakeTableInfo &table_info);
+
+private:
+	void WriteNewSchema(duckdb::DuckLakeSnapshot commit_snapshot, const duckdb::DuckLakeSchemaInfo &schema_info);
+	void WriteNewTable(duckdb::DuckLakeSnapshot commit_snapshot, const duckdb::DuckLakeTableInfo &table_info);
+	int GetDuckLakeSchemas(const duckdb::DuckLakeSnapshot &snapshot, duckdb::DuckLakeCatalogInfo &catalog_info);
+	int GetDuckLakeTables(const duckdb::DuckLakeSnapshot &snapshot, duckdb::DuckLakeCatalogInfo &catalog);
+
+	Snapshot snapshot;
+};
+
+} // namespace pgduckdb

include/pgducklake/pgducklake_schema_entry.hpp

@@ -0,0 +1,22 @@
+#pragma once
+
+#include "storage/ducklake_schema_entry.hpp"
+
+namespace pgduckdb {
+
+class PgDuckLakeSchemaEntry : public duckdb::DuckLakeSchemaEntry {
+public:
+	PgDuckLakeSchemaEntry(duckdb::Catalog &catalog, duckdb::CreateSchemaInfo &info, duckdb::SchemaIndex schema_id,
+	                      duckdb::string schema_uuid, duckdb::string data_path);
+
+	duckdb::optional_ptr<duckdb::CatalogEntry> LookupEntry(duckdb::CatalogTransaction transaction,
+	                                                       const duckdb::EntryLookupInfo &lookup_info) override;
+
+private:
+	duckdb::optional_ptr<duckdb::CatalogEntry> LoadTableEntry(duckdb::CatalogTransaction transaction,
+	                                                          const duckdb::EntryLookupInfo &lookup_info);
+
+	duckdb::DuckLakeCatalogSet tables;
+};
+
+} // namespace pgduckdb

include/pgducklake/pgducklake_storage.hpp

@@ -0,0 +1,14 @@
+#pragma once
+
+#include "duckdb/storage/storage_extension.hpp"
+
+#include "pgduckdb/utility/cpp_only_file.hpp" // Must be last include.
+
+namespace pgduckdb {
+
+class PgDuckLakeStorageExtension : public duckdb::StorageExtension {
+public:
+	PgDuckLakeStorageExtension();
+};
+
+} // namespace pgduckdb

include/pgducklake/pgducklake_table.hpp

@@ -0,0 +1,12 @@
+#pragma once
+
+#include "pgduckdb/pg/declarations.hpp"
+
+namespace pgduckdb {
+
+class PgDuckLakeTable {
+public:
+	static void CreateTable(Relation rel);
+};
+
+} // namespace pgduckdb

include/pgducklake/pgducklake_transaction.hpp

@@ -0,0 +1,17 @@
+#pragma once
+
+#include "storage/ducklake_transaction.hpp"
+
+namespace pgduckdb {
+
+class PgDuckLakeTransaction : public duckdb::DuckLakeTransaction,
+                              public std::enable_shared_from_this<PgDuckLakeTransaction> {
+public:
+	PgDuckLakeTransaction(duckdb::DuckLakeCatalog &ducklake_catalog, duckdb::TransactionManager &manager,
+	                      duckdb::ClientContext &context);
+	~PgDuckLakeTransaction() override {};
+
+	void Start() override;
+};
+
+} // namespace pgduckdb

include/pgducklake/pgducklake_transaction_manager.hpp

@@ -0,0 +1,23 @@
+#pragma once
+
+#include "storage/ducklake_transaction_manager.hpp"
+
+#include "pgducklake/pgducklake_transaction.hpp"
+
+namespace pgduckdb {
+
+class PgDuckLakeTransactionManager : public duckdb::DuckLakeTransactionManager {
+public:
+	PgDuckLakeTransactionManager(duckdb::AttachedDatabase &db_p, duckdb::DuckLakeCatalog &ducklake_catalog);
+
+	duckdb::Transaction &StartTransaction(duckdb::ClientContext &context) override;
+	duckdb::ErrorData CommitTransaction(duckdb::ClientContext &context, duckdb::Transaction &transaction) override;
+	void RollbackTransaction(duckdb::Transaction &transaction) override;
+
+private:
+	duckdb::DuckLakeCatalog &ducklake_catalog;
+	std::mutex transaction_lock;
+	duckdb::reference_map_t<duckdb::Transaction, duckdb::shared_ptr<PgDuckLakeTransaction>> transactions;
+};
+
+} // namespace pgduckdb