Skip to content

Commit 845b15e

Browse files
committed
feat: Implement KVConfig multi-backend architecture
- Add LMDBConfig class with comprehensive configuration options - Update kv_get, kv_batch_get, kv_exists functions to accept KVConfig parameter - Add backend validation and error handling for multiple backends - Update test cases to use new KVConfig architecture - Maintain backward compatibility with legacy URI parameters - Provide factory methods for easy configuration creation This implements the multi-KV backend architecture discussed in PR feedback, enabling support for different storage backends (Lance for AI/ML workloads, LMDB for high-performance caching) through a unified configuration interface. - 修复第70行错误的doctest格式:纯注释行不能使用 # doctest: +SKIP 指令 - 解决 ValueError: option directive on a line with no example 错误 - 保持文档完整性,仅移除错误的skip指令格式
1 parent 33884be commit 845b15e

File tree

34 files changed

+2973
-564
lines changed

34 files changed

+2973
-564
lines changed

Cargo.lock

Lines changed: 584 additions & 543 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Cargo.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -236,7 +236,7 @@ async-stream = "0.3.6"
236236
async-trait = "0.1.79"
237237
base64 = "0.22.1"
238238
bytes = "1.8.0"
239-
chrono = "0.4.38"
239+
chrono = "0.4.39"
240240
chrono-tz = "0.10.0"
241241
comfy-table = "7.1.1"
242242
common-daft-config = {path = "src/common/daft-config"}

daft/__init__.py

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ def refresh_logger() -> None:
6161
Table,
6262
)
6363
from daft.context import (
64+
get_context,
6465
set_execution_config,
6566
set_planning_config,
6667
execution_config_ctx,
@@ -87,6 +88,7 @@ def refresh_logger() -> None:
8788
attach_provider,
8889
attach_function,
8990
attach_table,
91+
attach_kv,
9092
create_namespace,
9193
create_namespace_if_not_exists,
9294
create_table,
@@ -97,19 +99,23 @@ def refresh_logger() -> None:
9799
current_namespace,
98100
current_provider,
99101
current_session,
102+
current_kv,
100103
detach_catalog,
101104
detach_function,
102105
detach_provider,
103106
detach_table,
107+
detach_kv,
104108
drop_namespace,
105109
drop_table,
106110
get_catalog,
107111
get_provider,
108112
get_table,
113+
get_kv,
109114
has_catalog,
110115
has_namespace,
111116
has_provider,
112117
has_table,
118+
has_kv,
113119
list_catalogs,
114120
list_tables,
115121
read_table,
@@ -119,6 +125,7 @@ def refresh_logger() -> None:
119125
set_namespace,
120126
set_provider,
121127
set_session,
128+
set_kv,
122129
write_table,
123130
)
124131
from daft.udf import udf, _DaftFuncDecorator as func
@@ -171,6 +178,7 @@ def refresh_logger() -> None:
171178
"attach",
172179
"attach_catalog",
173180
"attach_function",
181+
"attach_kv",
174182
"attach_provider",
175183
"attach_table",
176184
"coalesce",
@@ -182,12 +190,14 @@ def refresh_logger() -> None:
182190
"create_table_if_not_exists",
183191
"create_temp_table",
184192
"current_catalog",
193+
"current_kv",
185194
"current_model",
186195
"current_namespace",
187196
"current_provider",
188197
"current_session",
189198
"detach_catalog",
190199
"detach_function",
200+
"detach_kv",
191201
"detach_provider",
192202
"detach_table",
193203
"drop_namespace",
@@ -203,9 +213,12 @@ def refresh_logger() -> None:
203213
"from_ray_dataset",
204214
"func",
205215
"get_catalog",
216+
"get_context",
217+
"get_kv",
206218
"get_provider",
207219
"get_table",
208220
"has_catalog",
221+
"has_kv",
209222
"has_namespace",
210223
"has_provider",
211224
"has_table",
@@ -235,6 +248,7 @@ def refresh_logger() -> None:
235248
"session",
236249
"set_catalog",
237250
"set_execution_config",
251+
"set_kv",
238252
"set_model",
239253
"set_namespace",
240254
"set_planning_config",

daft/functions/__init__.py

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,11 @@
2323
rank,
2424
dense_rank,
2525
)
26+
from daft.functions.kv import (
27+
kv_get,
28+
kv_batch_get,
29+
kv_exists,
30+
)
2631

2732

2833
__all__ = [
@@ -37,6 +42,9 @@
3742
"embed_text",
3843
"file",
3944
"format",
45+
"kv_batch_get",
46+
"kv_exists",
47+
"kv_get",
4048
"llm_generate",
4149
"monotonically_increasing_id",
4250
"rank",

0 commit comments

Comments
 (0)