fix: open additional storage options provider related apis in lance dataset

python/python/lance/__init__.py

@@ -6,7 +6,7 @@
 import logging
 import os
 import warnings
-from typing import TYPE_CHECKING, Dict, List, Optional, Union
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
 
 from . import io, log
 from .blob import Blob, BlobArray, BlobColumn, BlobFile, blob_array, blob_field
@@ -99,6 +99,7 @@ def dataset(
     session: Optional[Session] = None,
     namespace: Optional[LanceNamespace] = None,
     table_id: Optional[List[str]] = None,
+    storage_options_provider: Optional[Any] = None,
 ) -> LanceDataset:
     """
     Opens the Lance dataset from the address specified.
@@ -166,6 +167,11 @@ def dataset(
     table_id : optional, List[str]
         The table identifier when using a namespace (e.g., ["my_table"]).
         Must be provided together with `namespace`. Cannot be used with `uri`.
+    storage_options_provider : optional
+        A storage options provider for automatic credential refresh. Must implement
+        `fetch_storage_options()` method that returns a dict of storage options.
+        If provided along with `namespace`, this takes precedence over the
+        namespace-created provider.
 
     Notes
     -----
@@ -191,7 +197,6 @@ def dataset(
         )
 
     # Handle namespace resolution in Python
-    storage_options_provider = None
     if namespace is not None:
         if table_id is None:
             raise ValueError(
@@ -208,9 +213,10 @@ def dataset(
         namespace_storage_options = response.storage_options
 
         if namespace_storage_options:
-            storage_options_provider = LanceNamespaceStorageOptionsProvider(
-                namespace=namespace, table_id=table_id
-            )
+            if storage_options_provider is None:
+                storage_options_provider = LanceNamespaceStorageOptionsProvider(
+                    namespace=namespace, table_id=table_id
+                )
             if storage_options is None:
                 storage_options = namespace_storage_options
             else:

python/python/lance/dataset.py

@@ -437,6 +437,7 @@ def __init__(
         uri = os.fspath(uri) if isinstance(uri, Path) else uri
         self._uri = uri
         self._storage_options = storage_options
+        self._storage_options_provider = storage_options_provider
 
         # Handle deprecation warning for index_cache_size
         if index_cache_size is not None:
@@ -2283,6 +2284,35 @@ def storage_options_accessor(self):
         """
         return self._ds.storage_options_accessor()
 
+    def new_file_session(self):
+        """
+        Create a new file session for reading and writing files in this dataset.
+
+        The file session will use the dataset's storage options and provider
+        for credential management, enabling automatic credential refresh for
+        long-running operations.
+
+        Returns
+        -------
+        LanceFileSession
+            A file session configured for this dataset's storage location.
+
+        Examples
+        --------
+        >>> import lance
+        >>> dataset = lance.dataset("s3://bucket/data.lance",
+        ...                        storage_options_provider=lambda: get_creds())
+        >>> session = dataset.new_file_session()
+        >>> session.upload_file("/local/file.txt", "remote/file.txt")
+        """
+        from lance.file import LanceFileSession
+
+        return LanceFileSession(
+            base_path=self._uri,
+            storage_options=self.latest_storage_options(),
+            storage_options_provider=self._storage_options_provider,
+        )
+
     def checkout_version(
         self, version: int | str | Tuple[Optional[str], Optional[int]]
     ) -> "LanceDataset":