From 2ca29f00b08a787d39cefbd6062ee0912f1092b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maik=20Fr=C3=B6be?= Date: Tue, 4 Jun 2024 13:22:29 +0200 Subject: [PATCH] prepare usage of podman with tests #647 --- python-client/Makefile | 1 + .../tests/docker_integration_test.py | 2 +- ...ter_notebook_pipeline_construction_test.py | 24 +++++++------ .../tira/local_execution_integration.py | 35 ++++++++++++++++--- 4 files changed, 47 insertions(+), 15 deletions(-) diff --git a/python-client/Makefile b/python-client/Makefile index bb60198d..10192244 100644 --- a/python-client/Makefile +++ b/python-client/Makefile @@ -8,6 +8,7 @@ build-pypi-package: run-tests run-tests: docker run -u root --rm -v /var/run/docker.sock:/var/run/docker.sock -v ${PWD}:/app -w /app --entrypoint pytest webis/tira:python-client-dev-0.0.5 docker run -u root --rm -v /var/run/docker.sock:/var/run/docker.sock -v ${PWD}:/app -w /app --entrypoint pytest webis/tira:python-client-dev-0.0.5-python3.7 + docker run -u root --rm -v /var/run/docker.sock:/run/user/0/podman/podman.sock -v ${PWD}:/app -w /app --entrypoint pytest webis/tira:python-client-dev-0.0.5-python3.7 run-tests-legacy: ../application/venv/bin/coverage run --data-file=tests/test-coverage/.coverage ../application/venv/bin/pytest \ diff --git a/python-client/tests/docker_integration_test.py b/python-client/tests/docker_integration_test.py index 1c404612..3e1db60b 100644 --- a/python-client/tests/docker_integration_test.py +++ b/python-client/tests/docker_integration_test.py @@ -30,7 +30,7 @@ def test_extraction_of_entrypoint(self): expected = '/docker-entrypoint.sh' tira = Client() - actual = tira.extract_entrypoint(image='bash:alpine3.16') + actual = tira.extract_entrypoint(image='docker.io/bash:alpine3.16') assert actual == expected diff --git a/python-client/tests/jupyter_notebook_pipeline_construction_test.py b/python-client/tests/jupyter_notebook_pipeline_construction_test.py index e09536ed..1be889dd 100644 --- a/python-client/tests/jupyter_notebook_pipeline_construction_test.py +++ b/python-client/tests/jupyter_notebook_pipeline_construction_test.py @@ -2,9 +2,13 @@ from pathlib import Path import unittest from subprocess import check_output +from tira.local_execution_integration import LocalExecutionIntegration as Client TEST_DIR = Path(__file__).parent.resolve() +def build_docker_image(image): + Client()._LocalExecutionIntegration__docker_client().images.build(path='.', dockerfile='tests/resources/'+ image, tag=image) + class JupyterNotebookPipelineConstructionTest(unittest.TestCase): def test_no_notebook_is_extracted_for_none_command(self): command = None @@ -262,7 +266,7 @@ def test_no_previous_stage_for_bash_file(self): def test_integration_against_custom_docker_image_01(self): image = 'dockerfile_bash_script_absolute' - check_output(['docker', 'build', '-t', image, '-f', f'tests/resources/{image}', '.']) + build_docker_image(image) expected = ['ir-benchmarks/tira-ir-starter/Index (tira-ir-starter-pyterrier)'] actual = extract_previous_stages_from_docker_image(image, '/usr/bin/retrieve-with-pyterrier-bash.sh') @@ -271,7 +275,7 @@ def test_integration_against_custom_docker_image_01(self): def test_integration_against_custom_docker_image_02(self): image = 'dockerfile_bash_script_absolute' - check_output(['docker', 'build', '-t', image, '-f', f'tests/resources/{image}', '.']) + build_docker_image(image) expected = ['ir-benchmarks/tira-ir-starter/Index (tira-ir-starter-pyterrier)'] actual = extract_previous_stages_from_docker_image(image) @@ -280,7 +284,7 @@ def test_integration_against_custom_docker_image_02(self): def test_integration_against_custom_docker_image_03(self): image = 'dockerfile_bash_script_absolute' - check_output(['docker', 'build', '-t', image, '-f', f'tests/resources/{image}', '.']) + build_docker_image(image) expected = [] actual = extract_previous_stages_from_docker_image(image, '/etc/hostname') @@ -289,7 +293,7 @@ def test_integration_against_custom_docker_image_03(self): def test_integration_against_custom_docker_image_04(self): image = 'jupyter_script_relative' - check_output(['docker', 'build', '-t', image, '-f', f'tests/resources/{image}', '.']) + build_docker_image(image) expected = ['ir-benchmarks/tira-ir-starter/Index (tira-ir-starter-pyterrier)'] actual = extract_previous_stages_from_docker_image(image) @@ -298,7 +302,7 @@ def test_integration_against_custom_docker_image_04(self): def test_integration_against_custom_docker_image_05(self): image = 'jupyter_script_relative' - check_output(['docker', 'build', '-t', image, '-f', f'tests/resources/{image}', '.']) + build_docker_image(image) expected = ['ir-benchmarks/tira-ir-starter/Index (tira-ir-starter-pyterrier)'] actual = extract_previous_stages_from_docker_image(image, 'python3 /usr/bin/retrieve-with-pyterrier-index.py') @@ -307,7 +311,7 @@ def test_integration_against_custom_docker_image_05(self): def test_integration_against_custom_docker_image_06(self): image = 'jupyter_script_relative' - check_output(['docker', 'build', '-t', image, '-f', f'tests/resources/{image}', '.']) + build_docker_image(image) expected = ['ir-benchmarks/tira-ir-starter/Index (tira-ir-starter-pyterrier)'] actual = extract_previous_stages_from_docker_image(image, 'python3 /usr/bin/retrieve-with-pyterrier-index.py; sleep3') @@ -316,7 +320,7 @@ def test_integration_against_custom_docker_image_06(self): def test_integration_against_custom_docker_image_07(self): image = 'jupyter_script_relative' - check_output(['docker', 'build', '-t', image, '-f', f'tests/resources/{image}', '.']) + build_docker_image(image) expected = ['ir-benchmarks/tira-ir-starter/Index (tira-ir-starter-pyterrier)'] actual = extract_previous_stages_from_docker_image(image, 'python3 /usr/bin/retrieve-with-pyterrier-index.py&&sleep3') @@ -325,7 +329,7 @@ def test_integration_against_custom_docker_image_07(self): def test_integration_against_custom_docker_image_08(self): image = 'jupyter_script_relative' - check_output(['docker', 'build', '-t', image, '-f', f'tests/resources/{image}', '.']) + build_docker_image(image) expected = ['ir-benchmarks/tira-ir-starter/Index (tira-ir-starter-pyterrier)'] actual = extract_previous_stages_from_docker_image(image, 'python3 "/usr/bin/retrieve-with-pyterrier-index.py"&&sleep3') @@ -335,7 +339,7 @@ def test_integration_against_custom_docker_image_08(self): def test_integration_against_custom_docker_image_09(self): image = 'jupyter_script_relative' - check_output(['docker', 'build', '-t', image, '-f', f'tests/resources/{image}', '.']) + build_docker_image(image) expected = ['ir-benchmarks/tira-ir-starter/Index (tira-ir-starter-pyterrier)'] actual = extract_previous_stages_from_docker_image(image, "python3 '/usr/bin/retrieve-with-pyterrier-index.py'&&sleep3") @@ -364,4 +368,4 @@ def test_pyterrier_from_run_output_01(self): expected = 'ir-benchmarks/tira-ir-starter/BM25 (tira-ir-starter-pyterrier)' actual = parse_extraction_of_tira_approach(python_line) - self.assertEqual(expected, actual) \ No newline at end of file + self.assertEqual(expected, actual) diff --git a/python-client/tira/local_execution_integration.py b/python-client/tira/local_execution_integration.py index 32e86416..895f4bc1 100644 --- a/python-client/tira/local_execution_integration.py +++ b/python-client/tira/local_execution_integration.py @@ -84,6 +84,15 @@ def ensure_image_available_locally(self, image, client=None): print("Skip pulling of image because everything is executed within github.") return + if image_pull_code != 0: + if client is None: + client=self.__docker_client() + try: + client.images.pull(image) + image_pull_code = 0 + except: + image_pull_code = 1 + if image_pull_code != 0: raise ValueError(f'Image could not be successfully pulled. Got return code {image_pull_code}. (expected 0.)') @@ -91,8 +100,9 @@ def ensure_image_available_locally(self, image, client=None): def extract_entrypoint(self, image): image_name = image - self.ensure_image_available_locally(image_name) - image = self.__docker_client().images.get(image_name) + docker_client = self.__docker_client() + self.ensure_image_available_locally(image_name, docker_client) + image = docker_client.images.get(image_name) ret = image.attrs['Config']['Entrypoint'] if not ret: return None @@ -114,11 +124,28 @@ def make_command_absolute(self, image_name, command): else: return command.replace(executable, (self.docker_image_work_dir(image_name) + '/' + executable).replace('//', '/').replace('/./', '/')) + def __docker_linux_sockets(self): + ret = [os.path.expanduser("~/.docker/desktop/docker.sock"), "/run/podman/podman.sock"] + + try: + ret += ["/run/user/" + str(os.getuid()) + "/podman/podman.sock"] + except: + pass + + return ret + + def __docker_client(self): try: environ = os.environ.copy() - if sys.platform == "linux" and os.path.exists(os.path.expanduser("~/.docker/desktop/docker.sock")): - environ["DOCKER_HOST"] = "unix:///" + os.path.expanduser("~/.docker/desktop/docker.sock") + if sys.platform == "linux" and "DOCKER_HOST" not in environ: + for docker_socket in self.__docker_linux_sockets(): + if os.path.exists(docker_socket): + environ["DOCKER_HOST"] = "unix://" + docker_socket + + if "DOCKER_HOST" in environ: + logging.info("Set DOCKER_HOST to '" + environ["DOCKER_HOST"] + "'.") + client = docker.from_env(environment=environ) assert len(client.images.list()) >= 0