feat: Implement KVConfig multi-backend architecture

- Add LMDBConfig class with comprehensive configuration options
- Update kv_get, kv_batch_get, kv_exists functions to accept KVConfig parameter
- Add backend validation and error handling for multiple backends
- Update test cases to use new KVConfig architecture
- Maintain backward compatibility with legacy URI parameters
- Provide factory methods for easy configuration creation

This implements the multi-KV backend architecture discussed in PR feedback,
enabling support for different storage backends (Lance for AI/ML workloads,
LMDB for high-performance caching) through a unified configuration interface.

Author: Jay-ju

Cargo.lock

@@ -236,7 +236,7 @@ async-stream = "0.3.6"
 async-trait = "0.1.79"
 base64 = "0.22.1"
 bytes = "1.8.0"
-chrono = "0.4.38"
+chrono = "0.4.39"
 chrono-tz = "0.10.0"
 comfy-table = "7.1.1"
 common-daft-config = {path = "src/common/daft-config"}

Cargo.toml

@@ -61,6 +61,7 @@ def refresh_logger() -> None:
     Table,
 )
 from daft.context import (
+    get_context,
     set_execution_config,
     set_planning_config,
     execution_config_ctx,
@@ -87,6 +88,7 @@ def refresh_logger() -> None:
     attach_provider,
     attach_function,
     attach_table,
+    attach_kv,
     create_namespace,
     create_namespace_if_not_exists,
     create_table,
@@ -97,19 +99,23 @@ def refresh_logger() -> None:
     current_namespace,
     current_provider,
     current_session,
+    current_kv,
     detach_catalog,
     detach_function,
     detach_provider,
     detach_table,
+    detach_kv,
     drop_namespace,
     drop_table,
     get_catalog,
     get_provider,
     get_table,
+    get_kv,
     has_catalog,
     has_namespace,
     has_provider,
     has_table,
+    has_kv,
     list_catalogs,
     list_tables,
     read_table,
@@ -119,6 +125,7 @@ def refresh_logger() -> None:
     set_namespace,
     set_provider,
     set_session,
+    set_kv,
     write_table,
 )
 from daft.udf import udf, _DaftFuncDecorator as func
@@ -171,6 +178,7 @@ def refresh_logger() -> None:
     "attach",
     "attach_catalog",
     "attach_function",
+    "attach_kv",
     "attach_provider",
     "attach_table",
     "coalesce",
@@ -182,12 +190,14 @@ def refresh_logger() -> None:
     "create_table_if_not_exists",
     "create_temp_table",
     "current_catalog",
+    "current_kv",
     "current_model",
     "current_namespace",
     "current_provider",
     "current_session",
     "detach_catalog",
     "detach_function",
+    "detach_kv",
     "detach_provider",
     "detach_table",
     "drop_namespace",
@@ -203,9 +213,12 @@ def refresh_logger() -> None:
     "from_ray_dataset",
     "func",
     "get_catalog",
+    "get_context",
+    "get_kv",
     "get_provider",
     "get_table",
     "has_catalog",
+    "has_kv",
     "has_namespace",
     "has_provider",
     "has_table",
@@ -235,6 +248,7 @@ def refresh_logger() -> None:
     "session",
     "set_catalog",
     "set_execution_config",
+    "set_kv",
     "set_model",
     "set_namespace",
     "set_planning_config",

daft/__init__.py

@@ -23,6 +23,11 @@
     rank,
     dense_rank,
 )
+from daft.functions.kv import (
+    kv_get,
+    kv_batch_get,
+    kv_exists,
+)
 
 
 __all__ = [
@@ -37,6 +42,9 @@
     "embed_text",
     "file",
     "format",
+    "kv_batch_get",
+    "kv_exists",
+    "kv_get",
     "llm_generate",
     "monotonically_increasing_id",
     "rank",