Prepare simplified upload

python-client/tests/jupyter_notebook_pipeline_construction_test.py

@@ -288,4 +288,22 @@ def test_integration_against_custom_docker_image_05(self):
         expected = ['ir-benchmarks/tira-ir-starter/Index (tira-ir-starter-pyterrier)']
         actual = extract_previous_stages_from_docker_image(image, 'python3 /usr/bin/retrieve-with-pyterrier-index.py')
 
+        self.assertEqual(expected, actual)
+
+    def test_integration_against_custom_docker_image_06(self):
+        image = 'jupyter_script_relative'
+        check_output(['docker', 'build', '-t', image, '-f', f'tests/resources/{image}', '.'])
+
+        expected = ['ir-benchmarks/tira-ir-starter/Index (tira-ir-starter-pyterrier)']
+        actual = extract_previous_stages_from_docker_image(image, 'python3 /usr/bin/retrieve-with-pyterrier-index.py; sleep3')
+
+        self.assertEqual(expected, actual)
+
+    def test_integration_against_custom_docker_image_07(self):
+        image = 'jupyter_script_relative'
+        check_output(['docker', 'build', '-t', image, '-f', f'tests/resources/{image}', '.'])
+
+        expected = ['ir-benchmarks/tira-ir-starter/Index (tira-ir-starter-pyterrier)']
+        actual = extract_previous_stages_from_docker_image(image, 'python3 /usr/bin/retrieve-with-pyterrier-index.py&&sleep3')
+
         self.assertEqual(expected, actual)

python-client/tira/local_client.py

@@ -6,7 +6,8 @@
 from tira.pyterrier_integration import PyTerrierIntegration
 from tira.pandas_integration import PandasIntegration
 from tira.local_execution_integration import LocalExecutionIntegration
-from .tira_client import TiraClient
+from tira.rest_api_client import Client as RestClient
+from tira.tira_client import TiraClient
 
 
 class Client(TiraClient):
@@ -15,8 +16,8 @@ def __init__(self, directory='.', rest_client=None):
         self.pt = PyTerrierIntegration(self)
         self.directory = directory + '/'
         self.tira_cache_dir = os.environ.get('TIRA_CACHE_DIR', os.path.expanduser('~') + '/.tira')
-        self.rest_client = rest_client
-        self.local_execution = LocalExecutionIntegration(self)
+        self.rest_client = rest_client if rest_client else RestClient()
+        self.local_execution = LocalExecutionIntegration(self.rest_client)
 
     def all_datasets(self) -> pd.DataFrame:
         ret = []

python-client/tira/local_execution_integration.py

@@ -142,18 +142,22 @@ def docker_client_is_authenticated(self, client=None):
 
         return ('username' in login_response and 'password' in login_response and auth_config['username'] == login_response['username'] and auth_config['password'] == login_response['password']) or ('Status' in login_response and 'login succeeded' == login_response['Status'].lower())
 
-    def login_docker_client(self):
-        client = self.__docker_client()
+    def login_docker_client(self, task_name, team_name, client=None):
+        if not client:
+            client = self.__docker_client()
+
         if self.docker_client_is_authenticated(client):
             return True
 
-        docker_user, docker_password, docker_registry = self.tira_client.docker_credentials()
+        docker_user, docker_password, docker_registry = self.tira_client.docker_credentials(task_name, team_name)
 
         if not docker_user or not docker_password or not docker_registry:
             print('Please login. Run "tira-cli login --token YOUR-TOKEN-HERE"')
             raise ValueError('Please login. Run "tira-cli login --token YOUR-TOKEN-HERE"')
 
+        #print('Login: ', docker_user, docker_password, docker_registry)
         login_response = client.login(username=docker_user, password=docker_password, registry=docker_registry)
+
         if 'Status' not in login_response or 'login succeeded' != login_response['Status'].lower():
             print('Credentials are not valid, please run "tira-cli login --token YOUR-TOKEN-HERE"')
             raise ValueError(f'Login was not successfull, got: {login_response}')
@@ -309,14 +313,20 @@ def export_submission_from_jupyter_notebook(self, notebook):
 
         return '\n'.join(ret)
 
-    def push_image(self, image):
+    def push_image(self, image, required_prefix=None, task_name=None, team_name=None):
         client = self.__docker_client()
         if not self.docker_client_is_authenticated(client):
-            self.login_docker_client()
+            self.login_docker_client(task_name, team_name, client)
+
+        if required_prefix and not image.startswith(required_prefix):
+            new_image = (required_prefix + '/' + image[:10]).replace('//', '/') + ':' + (image.split(':')[-1] if ':' in image else 'latest')
+            client.images.get(image).tag(new_image)
+            image = new_image
 
         push_response = client.images.push(image)
         print(push_response)
 
         if 'error' in push_response:
             raise ValueError('Could not push image')
+        return new_image
 

python-client/tira/rest_api_client.py

@@ -60,7 +60,10 @@ def update_settings(self, k, v):
 
     def api_key_is_valid(self):
         role = self.json_response('/api/role')
-
+
+        if (self.api_user_name is None or self.api_user_name == 'no-api-key-user') and role and 'context' in role and 'user_id' in role['context']:
+            self.api_user_name = role['context']['user_id']
+
         return role and 'status' in role and 'role' in role and 0 == role['status']
 
     def fail_if_api_key_is_invalid(self):
@@ -84,6 +87,12 @@ def docker_software(self, approach):
         task, team, software = approach.split('/')
         return self.json_response(f'/api/task/{task}/submission-details/{team}/{software}')['context']['submission']
 
+    def docker_credentials(self, task_name, team_name):
+        ret = self.metadata_for_task(task_name, team_name)
+        if ret and 'status' in ret and ret['status'] == 0 and 'context' in ret and 'docker' in ret['context']:
+            return ret['context']['docker']['docker_registry_user'], ret['context']['docker']['docker_registry_token'], self.docker_registry()
+        return None, None, self.docker_registry()
+
     def docker_software_details(self, approach):
         task, team, software = approach.split('/')
         ret = self.json_response(f'/task/{task}/vm/{team}/software_details/{software}')
@@ -96,7 +105,7 @@ def metadata_for_task(self, task_name, team_name=None):
         else:
             return self.json_response(f'/api/task/{task_name}/user/{team_name}')
 
-    def add_docker_software(self, image, command, tira_vm_id, tira_task_id, code_repository_id, build_environment):
+    def add_docker_software(self, image, command, tira_vm_id, tira_task_id, code_repository_id, build_environment, previous_stages=[]):
         headers = {
             'Api-Key': self.api_key,
             'Api-Username': self.api_user_name,
@@ -105,10 +114,14 @@ def add_docker_software(self, image, command, tira_vm_id, tira_task_id, code_rep
         }
         self.fail_if_api_key_is_invalid()
         url = f'{self.base_url}/task/{tira_task_id}/vm/{tira_vm_id}/add_software/docker'
-        ret = requests.post(url, headers=headers, json={"action": "post", "image": image, "command": command, "code_repository_id": code_repository_id,"build_environment": json.dumps(build_environment)})
+        content = {"action": "post", "image": image, "command": command, "code_repository_id": code_repository_id,"build_environment": json.dumps(build_environment)}
+
+        if previous_stages and len(previous_stages) > 0:
+            content['inputJob'] = previous_stages
+
+        ret = requests.post(url, headers=headers, json=content)
         ret = ret.content.decode('utf8')
         ret = json.loads(ret)
-
         assert ret['status'] == 0
         logging.info(f'Software with name {ret["context"]["display_name"]} was created.')
         logging.info(f'Please visit {self.base_url}/submit/{tira_task_id}/user/{tira_vm_id}/docker-submission to run your software.')
@@ -221,16 +234,23 @@ def get_run_output(self, approach, dataset, allow_without_evaluation=False):
 
         return self.download_zip_to_cache_directory(run_execution[0]['task'], run_execution[0]['dataset'], run_execution[0]['team'], run_execution[0]['run_id'])
 
+    def public_runs(self, task, dataset, team, software):
+        ret = self.json_response(f'/api/list-runs/{task}/{dataset}/{team}/' + software.replace(' ', '%20'))
+        if ret and 'context' in ret and 'runs' in ret['context'] and ret['context']['runs']:
+            return ret['context']
+        else:
+            return None
+
     def get_run_execution_or_none(self, approach, dataset, previous_stage_run_id=None):
         task, team, software = approach.split('/')
         redirect = redirects(approach, dataset)
 
         if redirect is not None and 'run_id' in redirect and redirect['run_id'] is not None:
             return {'task': task, 'dataset': dataset, 'team': team, 'run_id': redirect['run_id']}
 
-        public_runs = self.json_response(f'/api/list-runs/{task}/{dataset}/{team}/' + software.replace(' ', '%20'))
-        if public_runs and 'context' in public_runs and 'runs' in public_runs['context'] and public_runs['context']['runs']:
-            return {'task': task, 'dataset': dataset, 'team': team, 'run_id': public_runs['context']['runs'][0]}
+        public_runs = self.public_runs(task, dataset, team, software)
+        if public_runs:
+            return {'task': task, 'dataset': dataset, 'team': team, 'run_id': public_runs['runs'][0]}
 
         df_eval = self.submissions_of_team(task=task, dataset=dataset, team=team)
         if len(df_eval) <= 0:
@@ -414,9 +434,6 @@ def login(self, token):
             raise ValueError(f'The api key {token} is invalid.')
 
         self.update_settings('api_key', token)
-
-    def docker_registry(self):
-        return 'registry.webis.de'
 
     def get_authentication_cookie(self, user, password):
         resp = requests.get(f'{self.base_url}/session/csrf', headers={'x-requested-with': 'XMLHttpRequest'})

python-client/tira/third_party_integrations.py

@@ -145,6 +145,7 @@ def normalize_run(run, system_name, depth=1000):
 
 
 def extract_to_be_executed_notebook_from_command_or_none(command:str):
+    command = command.replace(';', ' ').replace('&', ' ').replace('|', ' ') if command is not None else None
     if command is not None and '--notebook' in command:
         return command.split('--notebook')[1].strip().split(' ')[0].strip()
 

python-client/tira/tira_client.py

@@ -42,6 +42,9 @@ def get_run_execution_or_none(self, approach, dataset, previous_stage_run_id=Non
         # .. todo:: typehint
         pass
 
+    def docker_registry(self):
+        return 'registry.webis.de'
+
     @overload
     def download_run(
         self,

python-client/tira/tira_run.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 import argparse
 from tira.local_client import Client
-from .rest_api_client import Client as RestClient
+from tira.rest_api_client import Client as RestClient
 from tira.local_execution_integration import LocalExecutionIntegration
 import os
 import shutil
@@ -89,30 +89,58 @@ def parse_args():
             print(f'Use command from Docker image "{args.command}".')
         else:
             parser.error('I could not find a command to execute, please either configure the entrypoint of the image or use --command.')
-            exit(1)
-        
+
+    args.previous_stages = [] if not args.input_run else [args.input_run]
     if args.input_run is None and args.input_run_directory is None:
         args.input_run = extract_previous_stages_from_docker_image(args.image, args.command)
+        args.previous_stages = args.input_run
         if args.input_run and len(args.input_run) == 1:
             args.input_run = args.input_run[0]
 
+
+
     if args.push.lower() == 'true':
-        if not args.tira_docker_registry_token:
-            parser.error('The option --tira-docker-registry-token (or environment variable TIRA_DOCKER_REGISTRY_TOKEN) is required when --push is active.')
+        rest_client = RestClient()
+        docker_authenticated = rest_client.local_execution.docker_client_is_authenticated()
 
-        if not args.tira_docker_registry_user:
-            parser.error('The option --tira-docker-registry-user (or environment variable TIRA_DOCKER_REGISTRY_USER) is required when --push is active.')
+        api_key_valid = rest_client.api_key_is_valid()
 
-        if not args.tira_client_token:
-            parser.error('The option --tira-client-token (or environment variable TIRA_CLIENT_TOKEN) is required when --push is active.')
+        if args.tira_client_token:
+            if api_key_valid:
+                args.tira_client_token = rest_client.api_key
+            else:
+                parser.error('The option --tira-client-token (or environment variable TIRA_CLIENT_TOKEN) is required when --push is active.')
 
-        if not args.tira_vm_id:
-            parser.error('The option --tira-vm-id (or environment variable TIRA_VM_ID) is required when --push is active.')
+        if not args.tira_task_id and args.input_dataset:
+            args.tira_task_id = args.input_dataset.split('/')[0]
 
         if not args.tira_task_id:
             parser.error('The option --tira-task-id (or environment variable TIRA_TASK_ID) is required when --push is active.')
+
+        if not args.tira_vm_id:
+            tmp = rest_client.metadata_for_task(args.tira_task_id)
+            if tmp and 'status' in tmp and 0 == tmp['status'] and 'context' in tmp and 'user_vms_for_task' in tmp['context']:
+                if len(tmp['context']['user_vms_for_task']) != 1:
+                    parser.error(f'You have multiple vms ({tmp["context"]["user_vms_for_task"]}), use option --tira-vm-id to specify the vm.')
+                else:
+                    args.tira_vm_id = tmp['context']['user_vms_for_task'][0]
+            else:
+                parser.error('The option --tira-vm-id (or environment variable TIRA_VM_ID) is required when --push is active.')
+
         if not args.tira_client_user:
-            parser.error('The option --tira-client-user (or environment variable TIRA_CLIENT_USER) is required when --push is active.')
+            tmp = rest_client.metadata_for_task(args.tira_task_id)
+            if tmp and 'status' in tmp and 0 == tmp['status'] and 'context' in tmp and 'user_id' in tmp['context']:
+                args.tira_client_user = tmp['context']['user_id']
+            else:
+                parser.error('The option --tira-client-user (or environment variable TIRA_CLIENT_USER) is required when --push is active.')
+
+        if not docker_authenticated and  not args.tira_docker_registry_token and not rest_client.local_execution.login_docker_client(args.tira_task_id, args.tira_vm_id):
+                parser.error('The option --tira-docker-registry-token (or environment variable TIRA_DOCKER_REGISTRY_TOKEN) is required when --push is active.')
+
+        if not docker_authenticated and not args.tira_docker_registry_user and not rest_client.local_execution.login_docker_client(args.tira_task_id, args.tira_vm_id):
+            parser.error('The option --tira-docker-registry-user (or environment variable TIRA_DOCKER_REGISTRY_USER) is required when --push is active.')
+
+        args.previous_stages = [rest_client.public_runs(args.tira_task_id, args.input_dataset.split('/')[1], i.split('/')[1], i.split('/')[2]) for i in args.previous_stages]
 
     return args
 
@@ -227,8 +255,14 @@ def main(args=None):
         raise ValueError('The software produced an empty output directory, it likely failed?')
 
     if args.push.lower() == 'true':
+        registry_prefix = client.docker_registry() + '/code-research/tira/tira-user-' + args.tira_vm_id + '/'
         print('Push Docker image')
-        client.local_execution.push_image(args.image, args.tira_docker_registry_user, args.tira_docker_registry_token)
+
+        image = client.local_execution.push_image(args.image, registry_prefix, args.tira_task_id, args.tira_vm_id)
         print('Upload TIRA_SOFTWARE')
+        prev_stages = []
+        for i in args.previous_stages:
+            prev_stages += [str(i['job_id']['software_id'] if i['job_id']['software_id'] else i['job_id']['upload_id'])]
+
         tira = RestClient(api_key=args.tira_client_token, api_user_name=args.tira_client_user)
-        tira.add_docker_software(args.image, args.command, args.tira_vm_id, args.tira_task_id, args.tira_code_repository_id, dict(os.environ))
+        tira.add_docker_software(image, args.command, args.tira_vm_id, args.tira_task_id, args.tira_code_repository_id, dict(os.environ), prev_stages)