prepare usage of podman with tests #647

python-client/Makefile

@@ -8,6 +8,7 @@ build-pypi-package: run-tests
 run-tests:
 	docker run -u root --rm -v /var/run/docker.sock:/var/run/docker.sock -v ${PWD}:/app -w /app --entrypoint pytest webis/tira:python-client-dev-0.0.5
 	docker run -u root --rm -v /var/run/docker.sock:/var/run/docker.sock -v ${PWD}:/app -w /app --entrypoint pytest webis/tira:python-client-dev-0.0.5-python3.7
+	docker run -u root --rm -v /var/run/docker.sock:/run/user/0/podman/podman.sock -v ${PWD}:/app -w /app --entrypoint pytest webis/tira:python-client-dev-0.0.5-python3.7
 
 run-tests-legacy:
 	../application/venv/bin/coverage run --data-file=tests/test-coverage/.coverage ../application/venv/bin/pytest \

python-client/tests/docker_integration_test.py

@@ -30,7 +30,7 @@ def test_extraction_of_entrypoint(self):
         expected = '/docker-entrypoint.sh'
 
         tira = Client()
-        actual = tira.extract_entrypoint(image='bash:alpine3.16')
+        actual = tira.extract_entrypoint(image='docker.io/bash:alpine3.16')
 
         assert actual == expected
 

python-client/tests/jupyter_notebook_pipeline_construction_test.py

@@ -2,9 +2,13 @@
 from pathlib import Path
 import unittest
 from subprocess import check_output
+from tira.local_execution_integration import LocalExecutionIntegration as Client
 
 TEST_DIR = Path(__file__).parent.resolve()
 
+def build_docker_image(image):
+    Client()._LocalExecutionIntegration__docker_client().images.build(path='.', dockerfile='tests/resources/'+ image, tag=image)
+
 class JupyterNotebookPipelineConstructionTest(unittest.TestCase):
     def test_no_notebook_is_extracted_for_none_command(self):
         command = None
@@ -262,7 +266,7 @@ def test_no_previous_stage_for_bash_file(self):
 
     def test_integration_against_custom_docker_image_01(self):
         image = 'dockerfile_bash_script_absolute'
-        check_output(['docker', 'build', '-t', image, '-f', f'tests/resources/{image}', '.'])
+        build_docker_image(image)
 
         expected = ['ir-benchmarks/tira-ir-starter/Index (tira-ir-starter-pyterrier)']
         actual = extract_previous_stages_from_docker_image(image, '/usr/bin/retrieve-with-pyterrier-bash.sh')
@@ -271,7 +275,7 @@ def test_integration_against_custom_docker_image_01(self):
 
     def test_integration_against_custom_docker_image_02(self):
         image = 'dockerfile_bash_script_absolute'
-        check_output(['docker', 'build', '-t', image, '-f', f'tests/resources/{image}', '.'])
+        build_docker_image(image)
 
         expected = ['ir-benchmarks/tira-ir-starter/Index (tira-ir-starter-pyterrier)']
         actual = extract_previous_stages_from_docker_image(image)
@@ -280,7 +284,7 @@ def test_integration_against_custom_docker_image_02(self):
 
     def test_integration_against_custom_docker_image_03(self):
         image = 'dockerfile_bash_script_absolute'
-        check_output(['docker', 'build', '-t', image, '-f', f'tests/resources/{image}', '.'])
+        build_docker_image(image)
 
         expected = []
         actual = extract_previous_stages_from_docker_image(image, '/etc/hostname')
@@ -289,7 +293,7 @@ def test_integration_against_custom_docker_image_03(self):
 
     def test_integration_against_custom_docker_image_04(self):
         image = 'jupyter_script_relative'
-        check_output(['docker', 'build', '-t', image, '-f', f'tests/resources/{image}', '.'])
+        build_docker_image(image)
 
         expected = ['ir-benchmarks/tira-ir-starter/Index (tira-ir-starter-pyterrier)']
         actual = extract_previous_stages_from_docker_image(image)
@@ -298,7 +302,7 @@ def test_integration_against_custom_docker_image_04(self):
 
     def test_integration_against_custom_docker_image_05(self):
         image = 'jupyter_script_relative'
-        check_output(['docker', 'build', '-t', image, '-f', f'tests/resources/{image}', '.'])
+        build_docker_image(image)
 
         expected = ['ir-benchmarks/tira-ir-starter/Index (tira-ir-starter-pyterrier)']
         actual = extract_previous_stages_from_docker_image(image, 'python3 /usr/bin/retrieve-with-pyterrier-index.py')
@@ -307,7 +311,7 @@ def test_integration_against_custom_docker_image_05(self):
 
     def test_integration_against_custom_docker_image_06(self):
         image = 'jupyter_script_relative'
-        check_output(['docker', 'build', '-t', image, '-f', f'tests/resources/{image}', '.'])
+        build_docker_image(image)
 
         expected = ['ir-benchmarks/tira-ir-starter/Index (tira-ir-starter-pyterrier)']
         actual = extract_previous_stages_from_docker_image(image, 'python3 /usr/bin/retrieve-with-pyterrier-index.py; sleep3')
@@ -316,7 +320,7 @@ def test_integration_against_custom_docker_image_06(self):
 
     def test_integration_against_custom_docker_image_07(self):
         image = 'jupyter_script_relative'
-        check_output(['docker', 'build', '-t', image, '-f', f'tests/resources/{image}', '.'])
+        build_docker_image(image)
 
         expected = ['ir-benchmarks/tira-ir-starter/Index (tira-ir-starter-pyterrier)']
         actual = extract_previous_stages_from_docker_image(image, 'python3 /usr/bin/retrieve-with-pyterrier-index.py&&sleep3')
@@ -325,7 +329,7 @@ def test_integration_against_custom_docker_image_07(self):
 
     def test_integration_against_custom_docker_image_08(self):
         image = 'jupyter_script_relative'
-        check_output(['docker', 'build', '-t', image, '-f', f'tests/resources/{image}', '.'])
+        build_docker_image(image)
 
         expected = ['ir-benchmarks/tira-ir-starter/Index (tira-ir-starter-pyterrier)']
         actual = extract_previous_stages_from_docker_image(image, 'python3 "/usr/bin/retrieve-with-pyterrier-index.py"&&sleep3')
@@ -335,7 +339,7 @@ def test_integration_against_custom_docker_image_08(self):
 
     def test_integration_against_custom_docker_image_09(self):
         image = 'jupyter_script_relative'
-        check_output(['docker', 'build', '-t', image, '-f', f'tests/resources/{image}', '.'])
+        build_docker_image(image)
 
         expected = ['ir-benchmarks/tira-ir-starter/Index (tira-ir-starter-pyterrier)']
         actual = extract_previous_stages_from_docker_image(image, "python3 '/usr/bin/retrieve-with-pyterrier-index.py'&&sleep3")
@@ -364,4 +368,4 @@ def test_pyterrier_from_run_output_01(self):
         expected = 'ir-benchmarks/tira-ir-starter/BM25 (tira-ir-starter-pyterrier)'
         actual = parse_extraction_of_tira_approach(python_line)
 
-        self.assertEqual(expected, actual)
+        self.assertEqual(expected, actual)

python-client/tira/local_execution_integration.py

@@ -84,15 +84,25 @@ def ensure_image_available_locally(self, image, client=None):
             print("Skip pulling of image because everything is executed within github.")
             return
 
+        if image_pull_code != 0:
+            if client is None:
+                client=self.__docker_client()
+            try:
+                client.images.pull(image)
+                image_pull_code = 0
+            except:
+                image_pull_code = 1
+
         if image_pull_code != 0:
             raise ValueError(f'Image could not be successfully pulled. Got return code {image_pull_code}. (expected 0.)')
 
         print('\n\n Image pulled successfully.\n\nI will now run the software.\n\n')
 
     def extract_entrypoint(self, image):
         image_name = image
-        self.ensure_image_available_locally(image_name)
-        image = self.__docker_client().images.get(image_name)
+        docker_client = self.__docker_client()
+        self.ensure_image_available_locally(image_name, docker_client)
+        image = docker_client.images.get(image_name)
         ret = image.attrs['Config']['Entrypoint']
         if not ret:
             return None
@@ -114,11 +124,28 @@ def make_command_absolute(self, image_name, command):
         else:
             return command.replace(executable, (self.docker_image_work_dir(image_name) + '/' + executable).replace('//', '/').replace('/./', '/'))
 
+    def __docker_linux_sockets(self):
+        ret = [os.path.expanduser("~/.docker/desktop/docker.sock"), "/run/podman/podman.sock"]
+
+        try:
+            ret += ["/run/user/" + str(os.getuid()) + "/podman/podman.sock"]
+        except:
+            pass
+
+        return ret
+
+
     def __docker_client(self):
         try:
             environ = os.environ.copy()
-            if sys.platform == "linux" and os.path.exists(os.path.expanduser("~/.docker/desktop/docker.sock")):
-                environ["DOCKER_HOST"] = "unix:///" + os.path.expanduser("~/.docker/desktop/docker.sock")
+            if sys.platform == "linux" and "DOCKER_HOST" not in environ:
+                for docker_socket in self.__docker_linux_sockets():
+                    if os.path.exists(docker_socket):
+                        environ["DOCKER_HOST"] = "unix://" + docker_socket
+
+                if "DOCKER_HOST" in environ:
+                    logging.info("Set DOCKER_HOST to '" + environ["DOCKER_HOST"] + "'.")
+
             client = docker.from_env(environment=environ)
 
             assert len(client.images.list()) >= 0