Skip to content

Commit

Permalink
Extend AIStore serialization backend to writing (lhotse-speech#1435)
Browse files Browse the repository at this point in the history
Signed-off-by: Piotr Żelasko <[email protected]>
  • Loading branch information
pzelasko authored and Your Name committed Jan 7, 2025
1 parent 66a78fe commit b64d6ab
Showing 1 changed file with 18 additions and 7 deletions.
25 changes: 18 additions & 7 deletions lhotse/serialization.py
Original file line number Diff line number Diff line change
Expand Up @@ -780,15 +780,26 @@ class AIStoreIOBackend(IOBackend):
"""

def open(self, identifier: str, mode: str):
assert "r" in mode, "We only support reading from AIStore at this time."
client, version = get_aistore_client()
object = client.fetch_object_by_url(identifier)
request = object.get()
if version >= parse_version("1.9.1"):
# AIStore SDK 1.9.1 supports ObjectFile for improved read fault resiliency
return request.as_file()
else:
return request.raw()
if "r" in mode:
try:
# AIStore >= 1.10.0
request = object.get_reader()
except AttributeError:
# AIStore < 1.10.0 deprecated method
request = object.get()
if version >= parse_version("1.9.1"):
# AIStore SDK 1.9.1 supports ObjectFile for improved read fault resiliency
return request.as_file()
else:
return request.raw()
if "w" in mode:
assert version >= parse_version("1.10.0"), (
f"Writing to AIStore requires at least version 1.10.0 of AIStore Python SDK, "
f"but your version is {version}"
)
return object.get_writer().as_file()

@classmethod
def is_available(cls) -> bool:
Expand Down

0 comments on commit b64d6ab

Please sign in to comment.