Skip to content

Commit

Permalink
feat(python): Support lakefs:// URI for delta scanner (#20757)
Browse files Browse the repository at this point in the history
  • Loading branch information
ion-elgreco authored Jan 18, 2025
1 parent 226449b commit d9f3e3d
Showing 1 changed file with 8 additions and 1 deletion.
9 changes: 8 additions & 1 deletion py-polars/polars/io/delta.py
Original file line number Diff line number Diff line change
Expand Up @@ -338,8 +338,15 @@ def _split_schema(
# Required because main_schema cannot contain hive columns currently
main_schema, hive_schema = _split_schema(polars_schema, partition_columns)

file_uris = dl_tbl.file_uris()

# LakeFS has an S3 compatible API, for reading therefore it's safe to do this.
# Deltalake internally has an integration for writing commits
if dl_tbl.table_uri.startswith("lakefs://"):
file_uris = [file_uri.replace("lakefs://", "s3://") for file_uri in file_uris]

return scan_parquet(
dl_tbl.file_uris(),
file_uris,
schema=main_schema,
hive_schema=hive_schema if len(partition_columns) > 0 else None,
allow_missing_columns=True,
Expand Down

0 comments on commit d9f3e3d

Please sign in to comment.