From 881bd9cd17e0561e7045377dadeb913120b7c12e Mon Sep 17 00:00:00 2001 From: Jax Liu Date: Fri, 4 Jul 2025 18:41:52 +0800 Subject: [PATCH 1/4] introduce duckdb connector --- ibis-server/app/model/__init__.py | 8 +- ibis-server/app/model/connector.py | 54 +++++++++++- ibis-server/app/model/metadata/factory.py | 14 +++ .../app/model/metadata/object_storage.py | 80 ++++++++++++++++++ .../test_file_source/jaffle_shop.duckdb | Bin 0 -> 1847296 bytes .../routers/v2/connector/test_local_file.py | 33 ++++++++ .../v3/connector/local_file/test_query.py | 45 ++++++++++ 7 files changed, 228 insertions(+), 6 deletions(-) create mode 100644 ibis-server/tests/resource/test_file_source/jaffle_shop.duckdb diff --git a/ibis-server/app/model/__init__.py b/ibis-server/app/model/__init__.py index c4b06edd6..1f2030358 100644 --- a/ibis-server/app/model/__init__.py +++ b/ibis-server/app/model/__init__.py @@ -378,7 +378,7 @@ class LocalFileConnectionInfo(BaseConnectionInfo): description="the root path of the local file", default="/", examples=["/data"] ) format: str = Field( - description="File format", default="csv", examples=["csv", "parquet", "json"] + description="File format", default="csv", examples=["csv", "parquet", "json", "duckdb"] ) @@ -387,7 +387,7 @@ class S3FileConnectionInfo(BaseConnectionInfo): description="the root path of the s3 bucket", default="/", examples=["/data"] ) format: str = Field( - description="File format", default="csv", examples=["csv", "parquet", "json"] + description="File format", default="csv", examples=["csv", "parquet", "json", "duckdb"] ) bucket: SecretStr = Field( description="the name of the s3 bucket", examples=["my-bucket"] @@ -408,7 +408,7 @@ class MinioFileConnectionInfo(BaseConnectionInfo): description="the root path of the minio bucket", default="/", examples=["/data"] ) format: str = Field( - description="File format", default="csv", examples=["csv", "parquet", "json"] + description="File format", default="csv", examples=["csv", "parquet", "json", "duckdb"] ) ssl_enabled: bool = Field( description="use the ssl connection or not", @@ -434,7 +434,7 @@ class GcsFileConnectionInfo(BaseConnectionInfo): description="the root path of the gcs bucket", default="/", examples=["/data"] ) format: str = Field( - description="File format", default="csv", examples=["csv", "parquet", "json"] + description="File format", default="csv", examples=["csv", "parquet", "json", "duckdb"] ) bucket: SecretStr = Field( description="the name of the gcs bucket", examples=["my-bucket"] diff --git a/ibis-server/app/model/connector.py b/ibis-server/app/model/connector.py index 39864acf8..35834bad2 100644 --- a/ibis-server/app/model/connector.py +++ b/ibis-server/app/model/connector.py @@ -1,5 +1,6 @@ import base64 import importlib +import os from contextlib import closing from functools import cache from json import loads @@ -8,6 +9,7 @@ import ibis import ibis.expr.datatypes as dt import ibis.expr.schema as sch +import opendal import pandas as pd import pyarrow as pa import sqlglot.expressions as sge @@ -204,10 +206,23 @@ def __init__(self, connection_info: ConnectionInfo): if isinstance(connection_info, GcsFileConnectionInfo): init_duckdb_gcs(self.connection, connection_info) + if connection_info.format == "duckdb": + # For duckdb format, we attach the database files + self._attach_database(connection_info) + @tracer.start_as_current_span("duckdb_query", kind=trace.SpanKind.INTERNAL) - def query(self, sql: str, limit: int) -> pa.Table: + def query(self, sql: str, limit: int | None) -> pa.Table: try: - return self.connection.execute(sql).fetch_arrow_table().slice(length=limit) + if limit is None: + # If no limit is specified, we return the full result + return self.connection.execute(sql).fetch_arrow_table() + else: + # If a limit is specified, we slice the result + # DuckDB does not support LIMIT in fetch_arrow_table, so we use slice + # to limit the number of rows returned + return ( + self.connection.execute(sql).fetch_arrow_table().slice(length=limit) + ) except IOException as e: raise UnprocessableEntityError(f"Failed to execute query: {e!s}") except HTTPException as e: @@ -222,6 +237,41 @@ def dry_run(self, sql: str) -> None: except HTTPException as e: raise QueryDryRunError(f"Failed to execute query: {e!s}") + def _attach_database(self, connection_info: ConnectionInfo) -> None: + db_files = self._list_duckdb_files(connection_info) + if not db_files: + raise UnprocessableEntityError( + "No DuckDB files found in the specified path." + ) + + for file in db_files: + try: + self.connection.execute( + f"ATTACH DATABASE '{file}' AS \"{os.path.splitext(os.path.basename(file))[0]}\" (READ_ONLY);" + ) + except IOException as e: + raise UnprocessableEntityError(f"Failed to attach database: {e!s}") + except HTTPException as e: + raise UnprocessableEntityError(f"Failed to attach database: {e!s}") + + def _list_duckdb_files(self, connection_info: ConnectionInfo) -> list[str]: + # This method should return a list of file paths in the DuckDB database + op = opendal.Operator("fs", root=connection_info.url.get_secret_value()) + files = [] + try: + for file in op.list("/"): + if file.path != "/": + stat = op.stat(file.path) + if not stat.mode.is_dir() and file.path.endswith(".duckdb"): + full_path = ( + f"{connection_info.url.get_secret_value()}/{file.path}" + ) + files.append(full_path) + except Exception as e: + raise UnprocessableEntityError(f"Failed to list files: {e!s}") + + return files + class RedshiftConnector: def __init__(self, connection_info: RedshiftConnectionUnion): diff --git a/ibis-server/app/model/metadata/factory.py b/ibis-server/app/model/metadata/factory.py index 7f78d2a00..911d42747 100644 --- a/ibis-server/app/model/metadata/factory.py +++ b/ibis-server/app/model/metadata/factory.py @@ -7,6 +7,7 @@ from app.model.metadata.mssql import MSSQLMetadata from app.model.metadata.mysql import MySQLMetadata from app.model.metadata.object_storage import ( + DuckDBMetadata, GcsFileMetadata, LocalFileMetadata, MinioFileMetadata, @@ -41,6 +42,19 @@ class MetadataFactory: @staticmethod def get_metadata(data_source: DataSource, connection_info) -> Metadata: try: + if ( + data_source + in [ + DataSource.local_file, + DataSource.s3_file, + DataSource.minio_file, + DataSource.gcs_file, + ] + and connection_info.format == "duckdb" + ): + # DuckDBMetadata is used for local file, S3, Minio, and GCS with DuckDB format + return DuckDBMetadata(connection_info) + return mapping[data_source](connection_info) except KeyError: raise NotImplementedError(f"Unsupported data source: {data_source}") diff --git a/ibis-server/app/model/metadata/object_storage.py b/ibis-server/app/model/metadata/object_storage.py index 8ed8f5759..a454f3056 100644 --- a/ibis-server/app/model/metadata/object_storage.py +++ b/ibis-server/app/model/metadata/object_storage.py @@ -2,6 +2,7 @@ import duckdb import opendal +import pyarrow as pa from loguru import logger from app.model import ( @@ -11,6 +12,7 @@ S3FileConnectionInfo, UnprocessableEntityError, ) +from app.model.connector import DuckDBConnector from app.model.metadata.dto import ( Column, RustWrenEngineColumnType, @@ -271,3 +273,81 @@ def _get_full_path(self, path): path = path[1:] return f"gs://{self.connection_info.bucket.get_secret_value()}/{path}" + + +class DuckDBMetadata(ObjectStorageMetadata): + def __init__(self, connection_info: LocalFileConnectionInfo): + super().__init__(connection_info) + self.connection = DuckDBConnector(connection_info) + + def get_table_list(self) -> list[Table]: + sql = """ + SELECT + t.table_catalog, + t.table_schema, + t.table_name, + c.column_name, + c.data_type, + c.is_nullable, + c.ordinal_position + FROM + information_schema.tables t + JOIN + information_schema.columns c + ON t.table_schema = c.table_schema + AND t.table_name = c.table_name + WHERE + t.table_type IN ('BASE TABLE', 'VIEW') + AND t.table_schema NOT IN ('information_schema', 'pg_catalog'); + """ + response = ( + self.connection.query( + sql, + ) + .to_pandas() + .to_dict(orient="records") + ) + + unique_tables = {} + for row in response: + # generate unique table name + schema_table = self._format_compact_table_name( + row["table_schema"], row["table_name"] + ) + # init table if not exists + if schema_table not in unique_tables: + unique_tables[schema_table] = Table( + name=schema_table, + columns=[], + properties=TableProperties( + schema=row["table_schema"], + catalog=row["table_catalog"], + table=row["table_name"], + ), + primaryKey="", + ) + + # table exists, and add column to the table + unique_tables[schema_table].columns.append( + Column( + name=row["column_name"], + type=self._to_column_type(row["data_type"]), + notNull=row["is_nullable"].lower() == "no", + properties=None, + ) + ) + return list(unique_tables.values()) + + def _format_compact_table_name(self, schema: str, table: str): + return f"{schema}.{table}" + + def get_constraints(self): + return [] + + def get_version(self): + df: pa.Table = self.connector.query("SELECT version()") + if df is None: + raise UnprocessableEntityError("Failed to get DuckDB version") + if df.num_rows == 0: + raise UnprocessableEntityError("DuckDB version is empty") + return df.column(0).to_pylist()[0] diff --git a/ibis-server/tests/resource/test_file_source/jaffle_shop.duckdb b/ibis-server/tests/resource/test_file_source/jaffle_shop.duckdb new file mode 100644 index 0000000000000000000000000000000000000000..2a226b84032819786a4769ecf713652900e70901 GIT binary patch literal 1847296 zcmeF)3xHGQo$&uN7ho7r5tV|=+7uDdj!}_rk!s5zAPk^ZsN!1R@sOOEiR3aS8Dv_0 zMWrfYQLv?Gt!vTlVr#9s)!NIpcD*o&bzR-Ic5C~$Vpn(Fi}wD$>%Q~Rx z%p)23Mjw)#Jm)#T=kr`9=OpK3?M**gbn~|_+`QluXDyxQly_TuYIs)ol!5bpsO3W+ zI(63SIkP@Ia2{Sk009ILKmY**5I_I{1Q0*~fp<^fx!?cpb65Yv=(Xee-{kDEJNRNS2V6^IQRT-=VL!D>Wqsx)lsJ^>QuKlwFx(tj5$uX z+#+U~`QLNKUFNp5#J%QBTdKV`Q!rPjm&`WzpJhU_ zBeB{`H>X;188W7jLkkOeUr*HWSc{j9CA{XfZoI>f@_oG-4T@QJef}me|1+w)yTyr0 z9n10>?MLWijg2~W{xoYHTCr z+8&a2uV@Y$(kHHwE6JT!#sO>+xG17YJ{`y3{y4g!( zp6WRdO22P$?khXy@h0YWw=*Hb%Qqb$>VdoRq4~9&6JEA079)(bG|7G0g4*5KbYuayZZmP(qXr-n}t z&o;~3)2Wu2Olf8|GA|U&aOUN#n{M@TbCqA1x7I98N(H2C%#wsWjE4)>3itHAW-r<1 zCL>;S+Et+wPp+>Ih2(!3fHGi1Yh&IO#TyGxGjr~=@U*_`H#bWqnzM=aX(2Zgnkl77 zQS~RC6ngFNe(?I|cD`}LSAYG$9j|}?zEHaxS>v{Pq1W%&@cK74y!QP~|FZMTzrJfz zea4G>k*wKz@3hp;HBaRKz8jEoLm9WlYmUa&%7_Xj+(&!(X>L#KpN=>cDkWkSi0cswX7YlVCBM9E6p>%u{*M{c&2!x%?6=a@?Lpm==hV4H+O?= zuD5O9jji?co&DQbz;Hsov3mNepby$kQMWVGyxPlN;Yn{P>Xv>3nLatw{H2th+tM=g zc*%ZzL%#d_9b)cW^XFdvY9`tnLVey#6ucTb!H>=Fof2(NfIWTOL{eYhl1?Rt8sF0- zv=cs3-{njUO3Cz08P7PQ{dZ0N+oSwH@>e?lNM`Rn!Mm~+dCSq|jFw5T)^qk$wK}&~ zcR5vs?;N_jdtcmXUMzS6R`}YZ=C$58VE#+hp0lm0#W}pssA+Y6Z?gNdul$idt<~94 z)8$mlf>E9`rPaAhp7n0_dh=|cw^!ZWyG_qnel_x%ubM@D-Z~bt z{eJKeo;7u=?(Tjrw?TR0Z>+@`AGEreQCgk1YP*~onMj`Tt5@SW<6E8I$+BRDR_Ddh z0(|9_x$yE4Kiki;UytgW@0W$kL~?souCa34xmI44?etlGYi@a-;`8QNamO-m_vO#H z&yj1t2PBheKUrGXetk23uO#C!gU@?rBL$oJRms{_Cz+Rvtoy6b+olTrRJ1#{JYUfb zf?7OC`i6g}ED=Vg`>Z3oi~mt zv(8^A$*8Y$&&C4@zwPrQXHw6|DVlX#oo#ipFWNKBKjYL$we!=8yi(X)H8M@-J5+86 zJS-U`cOK(!%(KEBPBTwLog<1?dA?|sADCx-cS8$S?5&DF=g53>!`QCeuJ-PNUFbbU zyV>2{Kg!L4&pM+E$6mo4-rHz>r|F_erqC_F-*5K#We-sHU7gWp4|jB%JaS}ml$p2H zF=q0z%#^0~^5`}Hh=5TryN$t`SDQ7fi`Fz*a<|@W@@Hbb6xHff6=m&m>Kfg2Dk%}| zcIqyS#p5xVIi7JG`4ji7Q=cnk-!}I~o!MM%pUs7HwKto~KMXb}6ScYR+vd6?*Z~ON zxv>}V<&F72J;mD*p6AE)@ECT#4RQgQe~Toa_xFl9orjlQr=2psTAlim=JKnZYp}OI)>AQm!Q-pu4Y1Y6jvRce zV`QwC)6u0BB%@TR#XJzGEt3~tv}{=}oczDqQ({IHtv&RUnxV54<)|)tHuMv1{PjZ@ zS(GDx0#VVj;e2Ade65c*zqUUNr(J(#jqY#ZAj`D%)#lf>ueSed%h!&_LFoE@N$b;Z zA;Y*(TVHMb+Wgx7sqGKi@o2kzZBuGJZP!;Dzt&S*Uu}Nv{E$CWR$J||^K1Y9YU``@ zx1Ha1{Mz|Vn_oM=%I*mJm{HrGwe{78rEOnr`Hd55RsUE&xZwfSxPXyYOO@Cw*%-+iB$ zZT|_Yv?*HKsG0NQ|4Xyo$Hpi_v^|H|E?-$)b#`rk)3&enFu1w` z$`$tg-Mn%`WoEa1*uTs$DmI(8|7iQCHotbfYszBYjI4-X+>GZhMXz>E&xZwXLZg`d?__gh2d;YgQUbO9HyS}!|*S4=VzwP%++WtR0{b%_9YHHj0 z0Pb7#IXS(Y{R-{4I@t1S$JN2+Gt%p8yM0G``6Io3i5~$35I_I{1o{dL@>xYL^MDtj zvaxreeYd3DJL_Bj!Ix;`8N9Uj-m8`3Q+hOG_>Tj;yKSdE8lpYgqODD(w6iwuea&y% zv(n}*ZGE-fIUYOcpD*Pk5|b=K!^vWi^UxafJT zG!I`y&tyhExkyz zzvk6u%|R~`o!fxt+$u#`-^h017+@oXr@QpMezLMMJ4LylOHfBriT1dc_2d%NnYLJa zyBF=b;tuL=_QnvTfE!vB!gVExz85o?&lVF z`d2D{H@CRm@AjH4?yu^D{*}u$TU@>`s$)y9TIXNmy6;f-C#r})2XGne{JgBa45#fs zaei2G?NQs?3H~nS8HT*m=C8C}5$#(mZGE-*we73jwa}KY-8CJu!<1yu`jo6b>`%4z z)yA*QukD}O{-7O?w#(NxsMgbVeYNpxJ+<}K=GV>-Tn>Bi7PsBL+VZvYn>N38d=0L* z|F3HMv$nq42({x;TfX+thPM3N?&}EJvCs|>ZGLToR@#n*c7SL*jMm3?er^2P{8~@j z_0>*j+WxQg(bl)p%GbuD_3zJTY#AHvZ@s}*v)#_xHv-!DwdD_H(cf#f$+b(~9+KzP*KmdV4f&O?cdKXdN%ZvN}EM1EgtQG(4ocOTRF4`(zUg>-5<0EI7a%r zGOa)PhgU$`e{A>vN;|+hyxNst+DLETN)y)3=eFaw-M-rR54LzJtv_hnYu_#?(eGIo z!KO?0GJ@0a``gQ4?;=3s9s&CTL%%!B9Yrt~XsM1k)pGPe9<(^M2{)FEnJa8`cSp=J z^S|eeyUcBAiF?hNwp4p>rY>h}q$86}CA@UTTmocF)EO5nXpTkA1xF4sR}XR8a@TyR zZ;7Qd+2*91@XTvd3SS!=cL#cDTyC{gI_jm(dWRLRcX%Qt>!rPjm&`WzpJhU_BeB{` zH>X;188W62xhP6+VDIaRIv#8Bvay8Myw;6(_))&EH={u@>#on=#(U2H$i+(JGB#3_ zdC`8mvIPVXKmY**5I_I{1Q0*~0R&V6hxq&TauwL9<6r%9h)b}(=I`GIJHT>8z+dxf zvu`};C0KJC@SOGjC0MK4WOH)y*{T-DX?41sx<)shN=n>v)zS-N@p#O0dahbpZ#LF* z)l%i=-tez>S=(oG;hd4#T>jxRj_(o>Fr^joK$ zhnHQaoky6ggIt_+z@JFzQ@h8G9DJ)|`aPoETbrPx`v!Z!D?GL@Ob{=WJf}^dkHovxgwf$dP zzIHrnTS@DqEnk~oTVHMb+Wgx7sqGKi@o2kzZF^}wZP!;Dzt&S*Uu}Nv{4mn5AEfoS zonPBt+IY0{n>N38d}+svHovw%YwN3xN87&I^0nveYRk{<_LS~W47pQV&aQ=)zU=&m z7FV*;x~R5GXye!BFV;Hn&&qD+e5hK_17E}92r8|<+9}kwzil7wc+tK^*ZSD5uQs0I z-a7oxwDr~I*G8-@U;CC!>#vPRTVHMb+VZvW+n#^4{@VPuJ+(biX6 zzP7&F{%JcNZT#AJwE2hUTljU2Lwt}`&Mt;FblY8Rq?fPt)aJMCqm76B!z*C7eGl8` zU~T^itF$Rv3uW6!8_!73ul3aSA8q+sPi;Kf@(;HBcIQRCrZyhi{Y_tk%34F~$@m)H z_AB@lqvyM4UAeEP75)lB&zINA*|*TP>G0NQ|4Xyo$82BCS1T~m=O}G|)3&enFu2xV zo8R_f7HvGX5BqE5skHps{-f=m+Wgw_t}Wm8{HBdx+aI*$Ykjow?0fzRs#Pnk9@=?b zRawcbokz9(LK}~^zS{V;y%5e{FkdJ#GIAf;OH(_AUB7S%rNIt#IG` z+A-3%gyCJH9pkp=sF7a2)>GS>+Br&_UmK5h-$GmdNYAhB58A(R+V-^_kG8$E@o4kg zjz`;G+IVc|w;jK>y=>3_w#SRMy=>RlcKO=&)#kVTeo5Q^ho}D>uI(m6=GV6KyLk3R z>E{-`r*r2|PA_M_Vn~34JcD*z9pqdiuKKpycchm;;u@5I5kLR|1Q0-ANP$5|JQzEot}8hU6&8ppB`o(Yb#41m_YvZw_U(jje~&>vwFqwJGPYG+&U(4tZg+POa@1TZS*|}jChFvWaA>{!;*ea~ zxafJTI{@`m2 z!C#Nyjb*R{5&WSI-tY(cgNv31>jiHp#z{?sWkIF8$Nb6PT=1NG1Q0*~0R#|0009IL zKmY**5O`Mvjx1v&wwVpvclzXq81GV|fjU&hxe5@5bAL`z=&rsWcw_n%xA8r5Cwy)M- z8;|YzO&h54Ubp4^8 zYkjo&ZO<#(ai;ax<{!$t^mW@;sMGF?^uKRW7fB`B<6hR=?bK!3V(sl-)Op5n_Bd6~ zI)|E7%Gt#ztvb&SZUODk9qC%ysdJ>)m-rFbAAylRKiD4cw#(n2wqsoc5I~?%p#NP9 z|6_|FTX*;F+%N6q7JPJGZk_)2F7m4!d(pCG!P@@)?4oBwKdH1?w9?|y9ta&;{JoV! zdmvp~d)xg%dw^r4zbn)FlYe*xwEf3+|F5(Iti!8a`K68Y_N_Ew?R;)Ke%tMem zNBeg}zn3xCH>URYvkcohl#jrBNkBW=-%E9`gt{DT@0M);HMQ+&JN`=Ollv0`KKrBB zcil?cyXYGsOL_#3?s;VW!NJxq^m`f0`Nov>UO3POI3HsmncQ+d%NX28`zr4`1{WLY z?~1j1skZyWNcSAPx9tC`+V;|Jaiq7ew!XIWSK7NtZTo8TYdvk3Z#y1s{FT-pv_AWG zK8Ak=cUeF1DHo|7+xzpJqdn)xM;87Xr!08%sFDR=oK&*ly7!eV_)Y18*CvJ~XuU4A*V)fwF;kL0IGt6ChV)#=Kez%IW!nm?8OHTm7p z+)aMH)DeT+s#lv;tBY0*@|g|y+y_~? zM*sl?5I_I{1Q0*~0R#|00D;OD_%kOsZsH$0y1zPi+(eEU$dQpPlNXnMnYH(@Kza4j zevGO;?5_P7LmR*LH`#0-cC=l-)<>IP+aJp7W&55>`!Tk*zS{iS_SN=(ZTZ^qsBHGt%;QF0}aC{FT-@wVgp5 zzc#4rUv2!_^0o2Xo`1Cd+WfXXwdHH$*S43od~N^H)>m7;w!Yf_X*(Wm z{MvZ5`S@};B9xdkzT&mQ=8wmk2W6i53fL_wXf|zUX?aEYg^N{ zk2ao>o?q*!?LWokA7hP68<({}-XjH9+PtWZr?}s}M;c`qtD*H|d<`SM6qWlV!{28p z>x4OTI$2Hc8FQ08XX~FAp1LLreqXxaUyd;+c*%~;`ut7Ke#@VAcAtM@TdjG0h!flP zmY&#_TlH$Q>YyjK-2q9>uI~beIp!viPlqFUu}Nv{P1plaV~hPRa&3k=RuCr`;QIY0KAF-0tg_000Iag zfB*srAb7U%|H0o_y1Mxl%tJbn_oMnX~&CJOdEftm9LFQ>#z0E z=C|Fx+P47O__gJ0+sk%5w*76_SL>sl58thi6w7te*0_&h zS!_=>ZTYs_S6jX|zqWm~v!T|rryMpW61- z`fKB{J-=z=*LDMK`C1=sJR`k)t*3TA*LvC>FWPvt_0`6&EngeIwtcnvZO)NKoeM1v`QO_D+QB>0HMEmw+4Y@Pc1|)PFyaJ8 z`aEH6+~-@;cKIW&!w@h62vn>)QfUS4ih|=Ik^cQT?GZ5Tn;~s}?I!}ZXJOl);AnlcN6fVCt1Vx<^Pr7K z>oY9H{Sa^$kE@umSk8$rVs=WKmY**5I_I{1Q0*~0R#}x37k4ecJ0o6?;(Rh z3N3g^n_qi;N_+f8dpyo|`2|#EJ=A*Iu5VdE7i7|UYU``buRT6E($AmM`rFR0Z7*#+ z+WAeJUpu}EhC*2nwee{Cv$p)Qf-cBpyM47^Z!HL|tcSS+;-%kk9qf`Ee56i$>RLldKa(Fv;7sY?XR7u_qRR=dVSlT z2fBXyyM282SFm_}?cagowRwg>WeI5e^B{*#f{OKg)jPLf?!4SO{#Uajo)a15i`lb- zGDhe`n`}`|3z<-@Y|E zuuCfK-K4gCwfVK$w#&C2k2d~F>knF=1mDjT|b9K$CE@%A&^Kw8KbsYbD zQDb{AaC*4s{P@Vi%a_W6M~^C5@Wn|b3$BxkYZjI9o6-fZO)kmtlPM(&x{fYc@SQ(1 zm-Un@c={*E-S1y0wPwI8eOC2+{p&SZC3nkT=`*+L)k9t=HNOeZdCb31YM(2ZHoECl zQbgqnrWeNI@tEiaS1_&LzYV@6>$f_6HyF;fpV?skVX#4o(A+dNtQq3O!`!xxcLG2E zi*xtOHBF5R*EG!!B~C!hLkw^i%q;`F+-Gs6?OtnlXSMa!=GXrD(!QP7mOs+p4)^JR zLoCqNSKGeY{M!Dh?GM`VXuJF&ny1gpT2I^c?GxY-3$&iv`fBrQ=Lc&iAm-h)e}A?0 z)%x4cZ##bN{HD#X9bZEXkNtjG+n=@d)dr|-Uv2pZ=9(sTL+vY-w{xMz*XFOZ&Z+GT z+W58kwVlm&er@||J#ELYE&pAt?@Uc>+DX)QJJ@#8jvMWpbghr=`fB6R^qqaltF5m# zzcwaq`Pw&$T7PXk+WKna*Osr1-}d~Y_1EUN?Wrwa8^5-_wB>92kG8(r^0oEV_D|dK zXyez$qs_l>pJ|nTeo@yr#OEXB?PO?!x82!Bdih#UZGPK6+IYx6yaJWhzPA5(RodjN zZB5%g+IU8Keyyjr{}h*hf;BE}T-E}4j}%;K^P)DM;(qrYX_R5ChSrnuHH`RDRPc?e z+*h4~vIqX+ocUT*P4Bnfj`N(Y^1I$e3w~d^;9rh0U-XhKtk2)%TX6X=xS3xXk2Mh+prr$5n zMN*0OxR>?h3v`*bSbMt{?fJ&uq52ID^<&^cyZF1E_sJLMOf~r8Tz({JKe)Gj48?Z& z+CO*N{MyeXf(gymz3tCDv_9JQ)&BX_mT&vdwbsX0KM2az)>j+9HovxiYWstBJlZZl zXoqs|wVt-?TW;JzKCP#=zS{iS`Qcsu;@pXW+fet~dTZOkc7EIOYv(y_e(g9Lsy+Aj zb#1@a)>j*!wtcnbADA!D4fdm82YlC}(uOR%7K3+c?GVxCAAIZk|EhM%(Z;XMubtAg z<3%f`jla^$*T$pu*ZOGl+iqX&TL5kR+VZvSWjh|*{+Wx7HUt7NI@jfI-Nd|3y*3SPWn>*l756mt_uG$}z51Hh;5dD7sQr-s_ z+TM1+xl01D-FwEAY;UNi+R;8#d8Myw%eURW+VZvewe72&4Yi)7?J?AI?QE#cKh*Y1 zU)LTux9w9J-+_Lv?LXT7scm1azcwD*^P4t)ZToA>*ZOGV8R_L~J+*35*+y_&S$(6%%H_c{-IyEO~6n`i5ML9Lc1kb5C`Qbm(zdxrv0;YX4q|LAW zM8NheZ2J=&t&jGInYMkk8;`a>Ys>F9 z$Uzp`ZeQ&uSA&Gw?@PJ;kkU`O20Jx}U5gUPk-lqDQk@~6YP-BPzwIe!$O@KZ(7t8U z)>oTfI|XY0q-e{pv?;cv`G)D6@sJ;ZgIR#_crdsA{?}dmW^DhrNC^laFkAwR-{F!R!Q}*Bn<}Wn zAfI3K{lZmwyB4zxiXVv&wQrSd=eKQWduq`7*sibb@@;$CE??_oJ09(EC2jl%TmRJh z*gj5YyZ=;LeYNq}&R=Q%+WOiazqWmBm#_8I#-sJOJ)dantIe;iuQtCn9^3h~pK}DK zW|Vtgjycbw;JlmSueCU z=3P;|weU1kgK6PuMXNS9O9h&| zOzr(_9mzGx)D=npMYG{Nng5zp-d|o|k}F!mE_@*qY4Z}U zZ1jYW)OR_D1l#bVdd3+QbwXz5livg@`A5#a^Bc3OMULBaIiuuRt>^5Kr-zl)sMYzm zsxGJM@IJNg?v|Ctc+S!mXMC_-Q@2)UYju~LdgpJ^bDS3E@ID}>KlQ9VuAuU*&bLce z9+PJ(UZudL)%kJBDm9{4l>Lx=_MBF$^B*NE`2E?evD>iJb$9|tNmTe1VQXT1t*E^PSBPn!Z zs3o0B3@{+m?iB?=)z_cUP+u>jPWn|eSCBs{d&Z<0qM=aCe=RpFcxjm}%vj0YT4L!; zZcgc47I*WD%rp_oMU#tFwigWeAk6&6dxlt1)t@nQqMXFqJGu%6R@9OA#-nouSD8zs zO5Za7=Z0LBJRIjaT~)2l@2hs3{^oZ+dHRlwH@`>jb`B}x*yD^VXuqCbD(9V=F6KC6 z^0D^aSdDBfKfcEmFts|nWF%Bo$D;ml^mDX2ubO4`y>NxE{lUC8wg<2`=$mSG2X!qC z%uFx^wY^hv7S#40c^U-Xv*xo-O<|k`!4z(LfMNB+13#=%>F7Rdi>YZExR)nK@}ZKg{X^zgFk_ea2XeQ`oy(ou8N-WAeUzhFg6vY~gD+ z)yPVtbHKgh@QY=S!><>Q!*BLK4r_ZVQW$bUNIek@FdFZweb?V17EF@ z_Tg__u;#Fj$e`e9sxg^gt7udP{j7J)1-TMlwk;)p3VZ$*o^(>^Qa6$6NMZ(rhS}4P{fIXeu6e)0v*qawUaM4xM_+ zDP~T{J>Q3+KHd!V0HkEF`|n)k0uK8!*ne9**#B?uU>|P=d$7e}8SeD~dMw)NJlSW! zw>rNx%X-FA;W&QFymDx;ey-8{*T;I!&GKepbPiUEFa!SNXPoL_mT7f<>dyhW4E{Ri z{dlYMiYzv7z-7Ml-+;d+e{Z|HpEd73ZSD zICCGYXC4Q600IagfB*srAb@Sq2Nj{|7m{Jh6Gs2&0%N`Ub@qMFN2 zu)*j31*P}DlOf+(m9LSQ?~M45f*(}j{Qd&Do`>4ER@(eM8?yAk_S8^m^|i#<-I z+vWFEq|67ly-29 z{GX)dcl`FqPkQ=4n_2Lqd4<3K#qanH^h?kDj-UD2!e+kb&-eWK9Y33U%I~eRgZ=*C zh->iO{owojrQh=(===P^5lr))|KRAbITs)(*W3qrdIbIN`TsfJ^X~!S0RjjhfB*sr zAb?|rA{m3KD3e&%!CXKsFT-7T-5zWL45 zZ;_YYob&x|Su0<9-@7`;sWnTRsvYOm8pjzUkH5>Bk19c;NP11uze)NpNxzl!CrSS!sk+K>>LeW|=?Fc z^g~IzBt0kTMM?iGX|JSzmGqXRKT7&g?k1PI*|v0Na(OJ$=6dn@X)hUd7sg|sa#wrVww19&D%;k1 zPAZ*n^BJ2`nYIpBf@n*{-Aq$Qv~!u+sGD37NoBK5Zqi%o#p9j%tPO||oEmdiXB@&&DQ8%&NO?RH1_F7Zv z&P8cA8Syd~nr(S2Q_<*oZbw{RZ*|jN*5qw+GpXcCDL~3Rt1TVNWMjzyM3mOdchjrg zw0pLfcH_~-UffHXShG!DCgYj){RkIEuW-|T(dVaRv1l!GJJRmrlvzAqw%sY4jCsDt z0;#imc084ixn`leEY%UqcniE_GL{kDR6M#uw%25yw|MEE%B^fmCB#j(-4ms0x&W$C z)=XXyi)2&j71G$b`br>f(5OwRwq&xiG2u!(nL^{9muyUkj~5NLWvtdnOd?=HXbZBf zXmhU!8e00F+-6AnXGu{>W{i)L)VSdTqfb~gb-@d#x9)so)e%oU@~w#tr##vCxmDxC zQ|dmscYM_^#(nLxPfVP;>CP>aKb4s|XUzOH8#?Z2YW-5=`ug2w;PrEpuIo=MW-gOC z#;KQN=9NPwO^`HElG*+wNs}cVEorKxV?X6op$R# zEqn6Qi3=W^vgOEY)-OE&ME8hGCZBQU-*2j~UbJKG5w#tS>+ku*Pm|BL$G`RR1AiPh z`i7?>>7QL0o=~@;`kHw&3-9t<>3b)2HKdMAiT7{Ute3laOD4$R3@zDk%Envk8mmLj#CM81>0!CN=3hK>GB=CG1ew4B z&oQBKZ{6y4)L!@Owa2!vZ27`UU$YZx9lptyRn~zhj+@tuF{-+S9eZ6 zVo=LE$Eh_LHso_v$-RlkY5edXzII8??CVy2_)K@%)6I$1_g_%^@VS4ScV+ziuN;3= zXwUj5FFyCd^KXnTm=*c@q|4JE9QXbQy1H(@bM_BE@Z&9^8z+B!?O(5Y^snCI5s8@=yLY>ZWRE{P&!HKhHVu?yAQf=O@k2zx3eU*ZueUO_HHu z-9Oj-Y^+ne{!r(y+W+l^YyQeve~44>IKQacZdQN%QuA-gZNi5p{KbS1Pw+c~ykz>r z#EGW1x!b5*H<8Bho5|@7#nj5b-5NY_oNB*UeNq-kl)a1lK019~ZrR-Y{nU9i&ikCd zbh?8$`qWz(s+@T>uZ(u8*N>Ut9Jzk1;~X~K#Iyd|#~tT;qt?qt@2(0troAq`Ozuvv zcJ6l$bDTS?`wK<-nYjhq`<$#WaboZ+-zg;|bDQiuo~yk7Y+l&EbDUH1_x@e)6A5$R zTf@BEYah+u2lYQ=-dtyebCI*!$%WMu*OijLu>D8Z9~IqFb;%bV9rfa+C(MtWF+Lsp z&K1iKIc?RsU-;x469G5hBqxU}}Q?9q>3^8CF={oy z8@s9>u0QXTsT0ONb?vEbbB?+7#_0U}R)x3U+;Y#avS)ne$uUPx{Osc6UcB$Rru#0q zf7_d*HtlZt<{c+>Jn`%B+>0;W_2&P%>|>XnS66fB#xL*K5IJLFV)0*Y-SNrf_AmeS zZ>LNcUpMNBsx9X=Oup{=y;U0*&-mO;zrJ>N+g}`g&aM@UublURf4p(!tS`;o`MJxU zy6oHk>%r~uMPK{b`5*q|fA09(QER_<*Ejx9Gx{^9ymIU06PxBdee6G8fA4W8JoD@8 zr(8DouPlW-*6{SX8K3czj+8V-(lklOOFBu?EJ<@EeN56>k`_r?D(T~rE|k1&c6lJtnA?@Ic6NnMhjlk}pbpGo?qq`i{f zkn|stek1AsNcta1)zYqYk`9q{xTK>b{kf!LB^@s*ENPac(T5u_TpZ$$xTWTUMAzsPkYIzyD)l% zn~pAw$3Eq*_Ofj&V~JF@t+O%eCYHL{wsdFmJhvm>l*+VqxDsMpD(+^^Pi4HuWYSya zcBI|KDYO23381s7BigylMB>*d=ADyDC)^d0R5rWRlb09Ba`)_bDjjpBDlvCisw0;1 z7I?{IEVH65mX3R!5^yRWT_F`%*_KMU858VTZRuDh8%w&&Vi|Y7m%PkP#FCAYF?m5O zl1-%-xNBq4<*~R_IJ!bGFEkOD_}mL(>DE{><~F6;lF81cvBYXG9q(*RxKizvKHrM0 zx7KS+h_WX-$#iFviKfL%8}>C4rEJ$k(`GEZv@}aoERpCm&*Gk!T+!xUkz6h+t{0bX z<;;+DzO?Dfa&N}MC`pYQ8gC3eHFbJZ(}K@`{)N>WXWuhE^XeI6zI@T%1ura`{PC?X z)P8)?NoQ66+0*wAv^yyP3pMCzS#ity;VDg5#^Pg?paqoha7oBj@ zw#BDCbIS40WhUJ5p>w7uHXMH33CkyV|1@j*gYL3BzC8Y?7tETm<#Q80u=<;0uDg8t zXFgMV#QtquYEpcwmZxTd94$%aN~d0u8TW@unjmSSBvbE6k|s+!TGCWW$4ClEGV{W8 zNycemcv<+Ia8r0?_}uXP@Y&(U@Y3+&@PhD)@P*+;;q$|1g_noV3rj|B2q1s}0tg_0 z00IagfB*srAbEor1fvFJL8yJcI_SByUC{vf0+_z&C|mi zr(bu&(C@9zaca#A4S6?nXWX2|5C7q7m(K`^WXie)KQy zd*QbFTfKR&e*O4WCprC+GMVddG^hE^~5+3l-; z_SK^Z$#hyL>Lu8;nrX7it3 zy0_`v6X$KK+W4)zF8#`c7jKL{{g*o@U7r5nmp*gYh0*AubMAQM)_+?(VZqHaKeD3X z;zxF@dFh4ouYDl){NLe)h`+($7xqY?yJ|?#RRImQC8UYxS(o zhGTBq6?uErvKgCpuby>#!=zhwM}D)aVa6@HR{v(*8Ix|2*VdV5X5lfLcE9%a_A_Q| zdg|4OwoA^ZBEQ)#S$0QGopQ>ITb_!}>RcwzR-3}iYwOGk(Niy4HsiLZUOlyQ*)f}* zdQD!F9{H-2dCV>HY@JycIra84B#7uktIn7tIUkaRx9yS@8fMfy{n|s@rLf(vo_dkL zb@Q4Oed_HE5^nV2RcFX+k%zXQAsgEJ_9_`bPwh3&%tDD;D)t+RQ>rOBa}|>no{E-w z1nJjo-(#?`aORAByW%>zQ?_GS;^yOQpFUkKuBllp-js;t%@7d=4{L#d*OU|FZ zWXq9D?&!%A1AG7M>^Tu?j@tKWNRdqXpei@V6it~eWe&~ z2q1s}0tg_000IagfB*srAbe6k~cTe<|?u;IB+et?pmt1o1alhIYU9qiw z$|Ey>wl#Xu!wX*9cHEL}4YzmQe9^>DB(7|Ic{&_l=sXqVw*&_xj`J zJ$=hNbp(Y`EsQ4cF~hxpef)Pfe`bwP5AADP8N^S5<%G z)~e{bg^$dvciyU>5+4(d->~C%jgK^3^WFFj%aR>*V+dceEh*DCvNY$WaZL1?{r-fy6(bz7hNV% zN#^Zc6IacjGqdTD=;N0(TqD6ha_cqY>uxws!kc&fg)2AhaGY=4`op#Jo?bh%>HRDB zOnhw5+AhbDuDb<` z(DO*<6Vu(fA%Fk^2q1s}0tg_000IagfB*srAbATi@N?-91_!&oR$8zd2{~ z>vJA>ea_~+b2sljee+)Pbo1ViJRlF=Jo7Gj{M)(T-@DE%-}_F@EAMQ6{mkdO&)odx zx?5g9ee;{A-y$!)Ip_P`vR1zIzISzwQ)`wsRXfhBHI6ez9*>uFq@*d5-Y;poqy|Z+ zNIFf@M zNqRuiBa*%?>4%bbNqSDwi<16X(q2jbD(Njrf0Xp!k{mG}Eva78VUi|Enks3!qz_6u zS<>l}=1E#4$+St6=O)jIMN)BZMO!Q#_d1*0$mJbgJeKs9#WL=BZLxSf)_y@O-5N{A z+{Sdats|Y!x!6soI~$WpuQ3@-dv5M&+)XZZvu)|lGHo5M1ksj?yP2krXy-DsQ8&3FlFDYA+@!bEi^n_jTb=JE zFLM*IW@s-s+~ktjXKtW>U$OQh=0sR$Drj$;Ofa zh$yX@@1|F~Y4>a|?Z%^vy||Y&v1XgROvW?o`w=dTUg4(wqR&ssV$oXWcBI|KDYJOK zY`ar78S{LP1yX1C?0702bIn3`S*jzJ@fLW=WGo}Psd#jSY_G{YZ}HMSm0Q`CN{E|m zyC+K1bOBVOteLzZ7Rjd4E2Ob=^_4)}pi!GrZOLS3W5ShoGKI!HFWHz7A1@kg%UG?E zm_)#Y&=zD{(dJ$eG_>?Txy_LD&yu2&%orafsd2*xMxU@~>Vg+eZ{7LGsw19ygOj8iYk%qxdVnjmSSB(wcVk|s+!TGCWW$4ClEnkLE2J0{+9!VAO8!e@n> z!Yjk)hR+UP7@i+)3@;5Y4lf9=2rmksA6_0lFDx15wjp=tT)7}OmpKbVKY69$p1%vt z{F`sCI_=hfTK43p6Bj%-Wy_J*tY3KkiS7}XOg`hxzu#0}y=ceWBWgPu*WdGrpC+Gg z4^2FD_q`AQYQy;_ue$kypTB+O<99!}F8UW=`iGt0yzK9iyIy;E$|3p9^tqF|8d685 zadZ2adHh!H@j>-(&<-2&?GS1_`4ev)a@!+QXUxdXXi0%F2!vDXZm*bi9yjj>L$OIO6jtPx>>sGg;_PS@UJ+^&i%NMR& zw0=|e=xd*Uddr0AvwyH<>HVkvYSy;RyMJ@v*Q<|yt}eT7_R6V?UwOEC{k+hGy4vb% z2EOxKsh@wol1N3p`2Dr{m-3U!K|9a%Bg7&E5I_I{1Q0*~0R#|0009ILKmY**5I_I{ z1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009IL zKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_Kd|GynNkO2Sy02t(NeUGmYGGM@f0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* s1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`I3*0_*z2v;Y7A literal 0 HcmV?d00001 diff --git a/ibis-server/tests/routers/v2/connector/test_local_file.py b/ibis-server/tests/routers/v2/connector/test_local_file.py index 73c854335..4055d1b63 100644 --- a/ibis-server/tests/routers/v2/connector/test_local_file.py +++ b/ibis-server/tests/routers/v2/connector/test_local_file.py @@ -447,3 +447,36 @@ async def test_list_json_files(client): assert columns[21]["type"] == "UUID" assert columns[22]["name"] == "c_varchar" assert columns[22]["type"] == "STRING" + + +async def test_duckdb_metadata_list_tables(client): + response = await client.post( + url=f"{base_url}/metadata/tables", + json={ + "connectionInfo": { + "url": "tests/resource/test_file_source", + "format": "duckdb", + }, + }, + ) + assert response.status_code == 200 + + result = next(filter(lambda x: x["name"] == "main.customers", response.json())) + assert result["name"] == "main.customers" + assert result["primaryKey"] is not None + assert result["description"] is None + assert result["properties"] == { + "catalog": "jaffle_shop", + "schema": "main", + "table": "customers", + "path": None, + } + assert len(result["columns"]) == 7 + assert result["columns"][1] == { + "name": "number_of_orders", + "nestedColumns": None, + "type": "INT64", + "notNull": False, + "description": None, + "properties": None, + } diff --git a/ibis-server/tests/routers/v3/connector/local_file/test_query.py b/ibis-server/tests/routers/v3/connector/local_file/test_query.py index 3b0111fe3..460ed47bd 100644 --- a/ibis-server/tests/routers/v3/connector/local_file/test_query.py +++ b/ibis-server/tests/routers/v3/connector/local_file/test_query.py @@ -178,3 +178,48 @@ async def test_dry_run(client, manifest_str): ) assert response.status_code == 422 assert response.text is not None + + +async def test_query_duckdb_format(client): + manifest = { + "catalog": "wren", + "schema": "public", + "models": [ + { + "name": "customers", + "tableReference": { + "catalog": "jaffle_shop", + "schema": "main", + "table": "customers", + }, + "columns": [ + {"name": "customer_id", "type": "integer"}, + {"name": "customer_lifetime_value", "type": "double"}, + {"name": "first_name", "type": "varchar"}, + {"name": "first_order", "type": "date"}, + {"name": "last_name", "type": "varchar"}, + {"name": "most_recent_order", "type": "date"}, + {"name": "number_of_orders", "type": "integer"}, + ], + }, + ], + "relationships": [], + "views": [], + } + response = await client.post( + f"{base_url}/query", + json={ + "manifestStr": base64.b64encode(orjson.dumps(manifest)).decode("utf-8"), + "sql": "SELECT * FROM customers LIMIT 1", + "connectionInfo": { + "url": "tests/resource/test_file_source", + "format": "duckdb", + }, + }, + ) + assert response.status_code == 200 + result = response.json() + assert len(result["columns"]) == len(manifest["models"][0]["columns"]) + assert len(result["data"]) == 1 + + From f1e92edd787c23aab6df0b0132c7147c83936670 Mon Sep 17 00:00:00 2001 From: Jax Liu Date: Fri, 4 Jul 2025 19:04:22 +0800 Subject: [PATCH 2/4] fix fmt --- ibis-server/app/model/__init__.py | 16 ++++++++++++---- .../v3/connector/local_file/test_query.py | 2 -- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/ibis-server/app/model/__init__.py b/ibis-server/app/model/__init__.py index 1f2030358..96f2260f3 100644 --- a/ibis-server/app/model/__init__.py +++ b/ibis-server/app/model/__init__.py @@ -378,7 +378,9 @@ class LocalFileConnectionInfo(BaseConnectionInfo): description="the root path of the local file", default="/", examples=["/data"] ) format: str = Field( - description="File format", default="csv", examples=["csv", "parquet", "json", "duckdb"] + description="File format", + default="csv", + examples=["csv", "parquet", "json", "duckdb"], ) @@ -387,7 +389,9 @@ class S3FileConnectionInfo(BaseConnectionInfo): description="the root path of the s3 bucket", default="/", examples=["/data"] ) format: str = Field( - description="File format", default="csv", examples=["csv", "parquet", "json", "duckdb"] + description="File format", + default="csv", + examples=["csv", "parquet", "json", "duckdb"], ) bucket: SecretStr = Field( description="the name of the s3 bucket", examples=["my-bucket"] @@ -408,7 +412,9 @@ class MinioFileConnectionInfo(BaseConnectionInfo): description="the root path of the minio bucket", default="/", examples=["/data"] ) format: str = Field( - description="File format", default="csv", examples=["csv", "parquet", "json", "duckdb"] + description="File format", + default="csv", + examples=["csv", "parquet", "json", "duckdb"], ) ssl_enabled: bool = Field( description="use the ssl connection or not", @@ -434,7 +440,9 @@ class GcsFileConnectionInfo(BaseConnectionInfo): description="the root path of the gcs bucket", default="/", examples=["/data"] ) format: str = Field( - description="File format", default="csv", examples=["csv", "parquet", "json", "duckdb"] + description="File format", + default="csv", + examples=["csv", "parquet", "json", "duckdb"], ) bucket: SecretStr = Field( description="the name of the gcs bucket", examples=["my-bucket"] diff --git a/ibis-server/tests/routers/v3/connector/local_file/test_query.py b/ibis-server/tests/routers/v3/connector/local_file/test_query.py index 460ed47bd..e3174c8d8 100644 --- a/ibis-server/tests/routers/v3/connector/local_file/test_query.py +++ b/ibis-server/tests/routers/v3/connector/local_file/test_query.py @@ -221,5 +221,3 @@ async def test_query_duckdb_format(client): result = response.json() assert len(result["columns"]) == len(manifest["models"][0]["columns"]) assert len(result["data"]) == 1 - - From d4a0ddfa775e8ebe8602b458bdbbe9d67176df7c Mon Sep 17 00:00:00 2001 From: Jax Liu Date: Fri, 4 Jul 2025 19:05:28 +0800 Subject: [PATCH 3/4] fix get version --- ibis-server/app/model/metadata/object_storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ibis-server/app/model/metadata/object_storage.py b/ibis-server/app/model/metadata/object_storage.py index a454f3056..7faceaa22 100644 --- a/ibis-server/app/model/metadata/object_storage.py +++ b/ibis-server/app/model/metadata/object_storage.py @@ -345,7 +345,7 @@ def get_constraints(self): return [] def get_version(self): - df: pa.Table = self.connector.query("SELECT version()") + df: pa.Table = self.connection.query("SELECT version()") if df is None: raise UnprocessableEntityError("Failed to get DuckDB version") if df.num_rows == 0: From 9d6459fe10d37b154c2bbbbfe4c9ed135f72ae0f Mon Sep 17 00:00:00 2001 From: Jax Liu Date: Mon, 7 Jul 2025 10:33:52 +0800 Subject: [PATCH 4/4] fix test --- ibis-server/app/model/metadata/object_storage.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/ibis-server/app/model/metadata/object_storage.py b/ibis-server/app/model/metadata/object_storage.py index 7faceaa22..24da66ee0 100644 --- a/ibis-server/app/model/metadata/object_storage.py +++ b/ibis-server/app/model/metadata/object_storage.py @@ -301,11 +301,7 @@ def get_table_list(self) -> list[Table]: AND t.table_schema NOT IN ('information_schema', 'pg_catalog'); """ response = ( - self.connection.query( - sql, - ) - .to_pandas() - .to_dict(orient="records") + self.connection.query(sql, limit=None).to_pandas().to_dict(orient="records") ) unique_tables = {}