From 09c3bb95e7d7453058d0e1b6fd46fa6910f503f3 Mon Sep 17 00:00:00 2001 From: Sebastian Tramp Date: Tue, 9 Apr 2024 17:21:41 +0200 Subject: [PATCH] clean up + refactorings --- .idea/imr.iml | 6 +- imr/imr.py | 42 ++++++----- imr/imr_cli.py | 73 ++++++++++--------- poetry.lock | 11 +-- pyproject.toml | 2 +- tests/__init__.py | 4 + .../fixtures/repository/modela/version1/.keep | 0 .../fixtures/repository/modela/version2/.keep | 0 .../fixtures/repository/modelb/version3/.keep | 0 .../fixtures/repository/modelb/version4/.keep | 0 tests/fixtures/repository/modelc/ignore-me | 0 .../fixtures/repository/modelc/version5/.keep | 0 tests/test_repository.py | 15 ++++ 13 files changed, 87 insertions(+), 66 deletions(-) create mode 100644 tests/fixtures/repository/modela/version1/.keep create mode 100644 tests/fixtures/repository/modela/version2/.keep create mode 100644 tests/fixtures/repository/modelb/version3/.keep create mode 100644 tests/fixtures/repository/modelb/version4/.keep create mode 100644 tests/fixtures/repository/modelc/ignore-me create mode 100644 tests/fixtures/repository/modelc/version5/.keep create mode 100644 tests/test_repository.py diff --git a/.idea/imr.iml b/.idea/imr.iml index e85cfa8..250a2ea 100644 --- a/.idea/imr.iml +++ b/.idea/imr.iml @@ -2,7 +2,8 @@ - + + @@ -11,7 +12,4 @@ - - \ No newline at end of file diff --git a/imr/imr.py b/imr/imr.py index d1a1f1f..e636b16 100644 --- a/imr/imr.py +++ b/imr/imr.py @@ -1,5 +1,4 @@ -# Inteligent Model Registry (imr) -import os +"""Intelligent Model Registry (imr)""" import shutil from pathlib import Path @@ -39,12 +38,12 @@ def pull(self, directory: str, package: str, version: str = "latest") -> None: auth=(self.user, self.password), auth_type=HTTPBasicAuth, ) - file_path = directory + "/" + package + "/" + version + file_path: Path = Path(directory) / package / version if not Path(file_path).exists(): Path.mkdir(file_path, parents=True) - with path.open() as fd, Path.open(file_path + "/" + "model.zip", "wb") as out: + with path.open() as fd, Path.open(file_path / "model.zip", "wb") as out: out.write(fd.read()) - shutil.unpack_archive(file_path + "/" + "model.zip", file_path + "/model") + shutil.unpack_archive(file_path / "model.zip", file_path / "model") def rm(self, package: str, version: str = "latest") -> None: """Remove a model from the remote repository.""" @@ -59,30 +58,37 @@ def rm(self, package: str, version: str = "latest") -> None: class IMRLocal: """Local repository class.""" + repo: Path + home: Path + def __init__(self, repo: str | None = None): self.home = Path.home() if repo is None: - self.repo = str(self.home) + "/.imr" + self.repo = self.home / ".imr" else: - self.repo = repo + self.repo = Path(repo) if not Path(self.repo).exists(): - Path.mkdir(self.repo, parents=True) + Path(self.repo).mkdir(parents=True) - def list(self) -> list: - """List local models.""" - return [ - entry[0].replace(self.repo + "/", "") - for entry in os.walk(self.repo) - if len(entry[1]) == 0 and entry[0] not in self.repo - ] + def list(self) -> list[str]: + """List local models. + + returns all the paths in the repository after main directory + BASE/modela/version1 + BASE/modela/version2 + BASE/modelb/version3 + + results in ["modela/version1", "modela/version2", "modelb/version3"] + """ + return ["/".join(version.parts[-2:]) for version in self.repo.rglob("*/*/")] def push(self, directory: str, package: str, version: str = "latest") -> None: """Push a model in a directory to the local repository.""" - shutil.copytree(directory, self.repo + "/" + package + "/" + version) + shutil.copytree(src=directory, dst=self.repo / package / version) def rm(self, package: str, version: str = "latest") -> None: """Remove a model from the local repository.""" if version is None: - shutil.rmtree(self.repo + "/" + package) + shutil.rmtree(self.repo / package) else: - shutil.rmtree(self.repo + "/" + package + "/" + version) + shutil.rmtree(self.repo / package / version) diff --git a/imr/imr_cli.py b/imr/imr_cli.py index c60e854..fb04bd8 100644 --- a/imr/imr_cli.py +++ b/imr/imr_cli.py @@ -1,29 +1,29 @@ -from pathlib import Path -""" Path module. +"""Path module. For handling local files and directory operations. """ +from pathlib import Path + import click -import yaml -from imr import IMRLocal, IMRRemote +from imr.imr import IMRLocal, IMRRemote -home = Path.home() -imr_dir = home + "/.imr" -imr_local: IMRLocal = IMRLocal(imr_dir) -imr_remote: IMRRemote = None -imr_config = None -def load_params() -> None: - """Load the default parameters from conf.yaml file.""" - with Path.open(imr_dir + "/config.yaml") as stream: - imr_config = yaml.safe_load(stream) +class Context: + """The context for all CLI commands.""" + + home = Path.home() + imr_dir = home / ".imr" + imr_local: IMRLocal = IMRLocal(str(imr_dir)) + imr_remote: IMRRemote @click.group() -def cli() -> None: +@click.pass_context +def cli(ctx: click.Context) -> None: """Get the cli command options.""" + ctx.obj = Context() # add loadParams() later @@ -33,10 +33,11 @@ def local() -> None: @local.command("list") -def list_local() -> None: +@click.pass_obj +def list_local(obj: Context) -> None: """List local packages.""" - for package in imr_local.list(): - print(package) + for package in obj.imr_local.list(): + click.echo(package) @local.command() @@ -44,27 +45,29 @@ def list_local() -> None: @click.option( "-v", "--version", type=str, default="latest", help="version of the model.", show_default=True ) -def remove(package: str, version: str) -> None: +@click.pass_obj +def remove(obj: Context, package: str, version: str) -> None: """Remove local packages.""" - local.rm(package, version) + obj.imr_local.rm(package, version) @cli.group() @click.argument("host") @click.argument("user") @click.argument("password") -def remote(host: str, user: str, password: str) -> None: +@click.pass_obj +def remote(obj: Context, host: str, user: str, password: str) -> None: """Get remote command cli options.""" - global imr_remote - imr_remote = IMRRemote(host, user, password) + obj.imr_remote = IMRRemote(host, user, password) @remote.command("list") -def list_remote() -> None: +@click.pass_obj +def list_remote(obj: Context) -> None: """List remote packages.""" - packages = imr_remote.list() + packages = obj.imr_remote.list() for p in packages: - print(p) + click.echo(p) @remote.command() @@ -72,9 +75,10 @@ def list_remote() -> None: @click.option( "-v", "--version", type=str, default="latest", help="version of the model.", show_default=True ) -def rm(package: str, version: str) -> None: +@click.pass_obj +def rm(obj: Context, package: str, version: str) -> None: """Remove remote package.""" - imr_remote.rm(package, version) + obj.imr_remote.rm(package, version) @remote.command() @@ -83,9 +87,10 @@ def rm(package: str, version: str) -> None: @click.option( "-v", "--version", type=str, default="latest", help="version of the model.", show_default=True ) -def push(model_dir: str, package: str, version: str) -> None: +@click.pass_obj +def push(obj: Context, model_dir: str, package: str, version: str) -> None: """Push model to remote repository.""" - imr_remote.push(model_dir, package, version) + obj.imr_remote.push(model_dir, package, version) @remote.command() @@ -94,17 +99,13 @@ def push(model_dir: str, package: str, version: str) -> None: "-d", "--dir", type=str, - default=imr_dir, help="directory to pull the model in.", show_default=True, ) @click.option( "-v", "--version", type=str, default="latest", help="version of the model.", show_default=True ) -def pull(package: str, model_dir: str, version: str) -> None: +@click.pass_obj +def pull(obj: Context, package: str, model_dir: str, version: str) -> None: """Pull model from remote repository.""" - imr_remote.pull(model_dir, package, version) - - -if __name__ == "__main__": - cli() + obj.imr_remote.pull(model_dir, package, version) diff --git a/poetry.lock b/poetry.lock index 047f28c..f72356c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. [[package]] name = "certifi" @@ -470,14 +470,12 @@ files = [ {file = "memray-1.11.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:eedea42d13b3630faa5591e298659f34e6ead06aa867050def12de6cc03e1a97"}, {file = "memray-1.11.0-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:9fbb2a1a82e24f0f90a9bb4ca7af6174ce91c5f3b3ce58e0b16361e989ea7cc1"}, {file = "memray-1.11.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6f46e00d4a10a7fb73b560e57689a68ca3661bf969e228093d20fc1313c42f0b"}, - {file = "memray-1.11.0-cp311-cp311-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:016a68de76fc800554fcc7dc473e48092d749b3b4302a6babd2e592a5fe8ae5e"}, {file = "memray-1.11.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7824202d23e3060c7a0380e1a9bb6f131f47ee29c6f30b322e844648ea3aa9da"}, {file = "memray-1.11.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c5b8860e3cc7df4f7f451e043aabe60a3812f99c3e308f0c4c0e7a03d72c1563"}, {file = "memray-1.11.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3fc83741aedd6daa9c49ecee0a8e0048f278b6eb1ae22bdcf9b4523be7ba7106"}, {file = "memray-1.11.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:39bbf9e74c3933a84c22e047920a0f6e2d491ba943a39f4aa041f1c0422c8403"}, {file = "memray-1.11.0-cp312-cp312-macosx_10_14_x86_64.whl", hash = "sha256:0ed869e4a82722a4558da749f39d6079f2ef5e767d1399d2d090b04742e2b3f2"}, {file = "memray-1.11.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ad1f2bb1223759e6b9755b6139087f6bcbaca1718cfed70c31aba0943542b431"}, - {file = "memray-1.11.0-cp312-cp312-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:9c577e81f8f7cd1418c7bfae4651d9bb3f16b72200e4b8d9b80c71aeeab64bb8"}, {file = "memray-1.11.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1534520c3c3e6b8234fe13c6d36bd74ab855dc19cef6e9d190a2a0b48fd2d83d"}, {file = "memray-1.11.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c3dfb2c20fbbb128489f7b9f5bd2704bae6f77dba11c253cccf8eb8299697fe4"}, {file = "memray-1.11.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b8e02e8bbe03826c5e65c2cc28760b1d0bc59f9bee6d6769c01e800b50542f5b"}, @@ -851,7 +849,6 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, @@ -998,13 +995,13 @@ files = [ [[package]] name = "textual" -version = "0.56.2" +version = "0.56.3" description = "Modern Text User Interface framework" optional = false python-versions = "<4.0,>=3.8" files = [ - {file = "textual-0.56.2-py3-none-any.whl", hash = "sha256:cff8ee993796fc02430aacd7f148894478ff7f3a6b77865da525a8965ecc0ccf"}, - {file = "textual-0.56.2.tar.gz", hash = "sha256:5d3efe7266aafc7b5a52ad622a155963fd22ec6043c769ecd7142490a3d9e17e"}, + {file = "textual-0.56.3-py3-none-any.whl", hash = "sha256:bb876b16382b4a0b8d1e667055af28dad8d2a720f4298ec950e00d791b95b7ac"}, + {file = "textual-0.56.3.tar.gz", hash = "sha256:9549a005a31658cd4dce7f73a247bc699b6173c74afc52c5dfdddb0afd4a67e2"}, ] [package.dependencies] diff --git a/pyproject.toml b/pyproject.toml index 45d6cc9..58ebff3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ readme = "README-public.md" homepage = "https://github.com/eccenca/imr" [tool.poetry.scripts] -imr = 'imr-cli:main' +imr = 'imr.imr_cli:cli' [tool.poetry.dependencies] # if you need to change python version here, change it also in .python-version diff --git a/tests/__init__.py b/tests/__init__.py index 30ea453..0c7d3b9 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1 +1,5 @@ """tests""" + +from pathlib import Path + +FIXTURE_DIR = Path(__file__).parent / "fixtures" diff --git a/tests/fixtures/repository/modela/version1/.keep b/tests/fixtures/repository/modela/version1/.keep new file mode 100644 index 0000000..e69de29 diff --git a/tests/fixtures/repository/modela/version2/.keep b/tests/fixtures/repository/modela/version2/.keep new file mode 100644 index 0000000..e69de29 diff --git a/tests/fixtures/repository/modelb/version3/.keep b/tests/fixtures/repository/modelb/version3/.keep new file mode 100644 index 0000000..e69de29 diff --git a/tests/fixtures/repository/modelb/version4/.keep b/tests/fixtures/repository/modelb/version4/.keep new file mode 100644 index 0000000..e69de29 diff --git a/tests/fixtures/repository/modelc/ignore-me b/tests/fixtures/repository/modelc/ignore-me new file mode 100644 index 0000000..e69de29 diff --git a/tests/fixtures/repository/modelc/version5/.keep b/tests/fixtures/repository/modelc/version5/.keep new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_repository.py b/tests/test_repository.py new file mode 100644 index 0000000..866166d --- /dev/null +++ b/tests/test_repository.py @@ -0,0 +1,15 @@ +"""Repository tests.""" + +from imr.imr import IMRLocal +from tests import FIXTURE_DIR + + +def test_basic_list() -> None: + """Test list items in repository.""" + number_of_models = 5 + directory = FIXTURE_DIR / "repository" + repository = IMRLocal(repo=str(directory)) + models = repository.list() + assert len(models) == number_of_models + assert "modela/version1" in models + assert "ignore-me" not in models