Merge branch 'master' into DEVX-217-cleanup-repo

Author: sainivedh

VERSION

@@ -1 +1 @@
-9.10.0
+9.10.2

clarifai/client/app.py

@@ -83,7 +83,7 @@ def list_datasets(self, page_no: int = None,
     for dataset_info in all_datasets_info:
       if 'version' in list(dataset_info.keys()):
         del dataset_info['version']['metrics']
-      yield Dataset(**dataset_info)
+      yield Dataset(base_url=self.base, **dataset_info)
 
   def list_models(self,
                   filter_by: Dict[str, Any] = {},
@@ -124,7 +124,7 @@ def list_models(self,
       if only_in_app:
         if model_info['app_id'] != self.id:
           continue
-      yield Model(**model_info)
+      yield Model(base_url=self.base, **model_info)
 
   def list_workflows(self,
                      filter_by: Dict[str, Any] = {},
@@ -163,7 +163,7 @@ def list_workflows(self,
       if only_in_app:
         if workflow_info['app_id'] != self.id:
           continue
-      yield Workflow(**workflow_info)
+      yield Workflow(base_url=self.base, **workflow_info)
 
   def list_modules(self,
                    filter_by: Dict[str, Any] = {},
@@ -202,7 +202,7 @@ def list_modules(self,
       if only_in_app:
         if module_info['app_id'] != self.id:
           continue
-      yield Module(**module_info)
+      yield Module(base_url=self.base, **module_info)
 
   def list_installed_module_versions(self,
                                      filter_by: Dict[str, Any] = {},
@@ -237,7 +237,8 @@ def list_installed_module_versions(self,
     for imv_info in all_imv_infos:
       del imv_info['deploy_url']
       del imv_info['installed_module_version_id']  # TODO: remove this after the backend fix
-      yield Module(module_id=imv_info['module_version']['module_id'], **imv_info)
+      yield Module(
+          module_id=imv_info['module_version']['module_id'], base_url=self.base, **imv_info)
 
   def list_concepts(self, page_no: int = None,
                     per_page: int = None) -> Generator[Concept, None, None]:
@@ -302,7 +303,7 @@ def create_dataset(self, dataset_id: str, **kwargs) -> Dataset:
     if response.status.code != status_code_pb2.SUCCESS:
       raise Exception(response.status)
     self.logger.info("\nDataset created\n%s", response.status)
-    kwargs.update({'app_id': self.id, 'user_id': self.user_id})
+    kwargs.update({'app_id': self.id, 'user_id': self.user_id, 'base_url': self.base})
 
     return Dataset(dataset_id=dataset_id, **kwargs)
 
@@ -330,7 +331,8 @@ def create_model(self, model_id: str, **kwargs) -> Model:
     kwargs.update({
         'app_id': self.id,
         'user_id': self.user_id,
-        'model_type_id': response.model.model_type_id
+        'model_type_id': response.model.model_type_id,
+        'base_url': self.base
     })
 
     return Model(model_id=model_id, **kwargs)
@@ -423,6 +425,7 @@ def create_workflow(self,
       display_workflow_tree(dict_response["workflows"][0]["nodes"])
     kwargs = self.process_response_keys(dict_response[list(dict_response.keys())[1]][0],
                                         "workflow")
+    kwargs.update({'base_url': self.base})
 
     return Workflow(**kwargs)
 
@@ -450,7 +453,7 @@ def create_module(self, module_id: str, description: str, **kwargs) -> Module:
     if response.status.code != status_code_pb2.SUCCESS:
       raise Exception(response.status)
     self.logger.info("\nModule created\n%s", response.status)
-    kwargs.update({'app_id': self.id, 'user_id': self.user_id})
+    kwargs.update({'app_id': self.id, 'user_id': self.user_id, 'base_url': self.base})
 
     return Module(module_id=module_id, **kwargs)
 
@@ -477,6 +480,7 @@ def dataset(self, dataset_id: str, **kwargs) -> Dataset:
     kwargs = self.process_response_keys(dict_response[list(dict_response.keys())[1]],
                                         list(dict_response.keys())[1])
     kwargs['version'] = response.dataset.version if response.dataset.version else None
+    kwargs.update({'base_url': self.base})
     return Dataset(**kwargs)
 
   def model(self, model_id: str, model_version_id: str = "", **kwargs) -> Model:
@@ -512,6 +516,7 @@ def model(self, model_id: str, model_version_id: str = "", **kwargs) -> Model:
     kwargs = self.process_response_keys(dict_response['model'], 'model')
     kwargs[
         'model_version'] = response.model.model_version if response.model.model_version else None
+    kwargs.update({'base_url': self.base})
 
     return Model(**kwargs)
 
@@ -537,6 +542,7 @@ def workflow(self, workflow_id: str, **kwargs) -> Workflow:
     dict_response = MessageToDict(response, preserving_proto_field_name=True)
     kwargs = self.process_response_keys(dict_response[list(dict_response.keys())[1]],
                                         list(dict_response.keys())[1])
+    kwargs.update({'base_url': self.base})
 
     return Workflow(**kwargs)
 
@@ -563,6 +569,7 @@ def module(self, module_id: str, module_version_id: str = "", **kwargs) -> Modul
       raise Exception(response.status)
     dict_response = MessageToDict(response, preserving_proto_field_name=True)
     kwargs = self.process_response_keys(dict_response['module'], 'module')
+    kwargs.update({'base_url': self.base})
 
     return Module(**kwargs)
 
@@ -572,7 +579,7 @@ def inputs(self,):
     Returns:
         Inputs: An input object.
     """
-    return Inputs(self.user_id, self.id)
+    return Inputs(self.user_id, self.id, base_url=self.base)
 
   def delete_dataset(self, dataset_id: str) -> None:
     """Deletes an dataset for the user.

clarifai/client/dataset.py

@@ -93,7 +93,8 @@ def create_version(self, **kwargs) -> 'Dataset':
         'dataset_id': self.id,
         'app_id': self.app_id,
         'user_id': self.user_id,
-        'version': response.dataset_versions[0]
+        'version': response.dataset_versions[0],
+        'base_url': self.base
     })
     return Dataset(**kwargs)
 
@@ -155,7 +156,8 @@ def list_versions(self, page_no: int = None,
           'dataset_id': self.id,
           'app_id': self.app_id,
           'user_id': self.user_id,
-          'version': resources_pb2.DatasetVersion(**dataset_version_info)
+          'version': resources_pb2.DatasetVersion(**dataset_version_info),
+          'base_url': self.base
       }
       yield Dataset(**kwargs)
 

clarifai/client/model.py

@@ -329,7 +329,7 @@ def create_version(self, **kwargs) -> 'Model':
     dict_response = MessageToDict(response, preserving_proto_field_name=True)
     kwargs = self.process_response_keys(dict_response['model'], 'model')
 
-    return Model(**kwargs)
+    return Model(base_url=self.base, **kwargs)
 
   def list_versions(self, page_no: int = None,
                     per_page: int = None) -> Generator['Model', None, None]:
@@ -371,7 +371,10 @@ def list_versions(self, page_no: int = None,
         del model_version_info['train_info']['dataset']['version']['metrics']
       except KeyError:
         pass
-      yield Model(model_id=self.id, **dict(self.kwargs, model_version=model_version_info))
+      yield Model(
+          model_id=self.id,
+          base_url=self.base,
+          **dict(self.kwargs, model_version=model_version_info))
 
   def predict(self, inputs: List[Input], inference_params: Dict = {}, output_config: Dict = {}):
     """Predicts the model based on the given inputs.

clarifai/client/module.py

@@ -76,7 +76,10 @@ def list_versions(self, page_no: int = None,
     for module_version_info in all_module_versions_info:
       module_version_info['id'] = module_version_info['module_version_id']
       del module_version_info['module_version_id']
-      yield Module(module_id=self.id, **dict(self.kwargs, module_version=module_version_info))
+      yield Module(
+          module_id=self.id,
+          base_url=self.base,
+          **dict(self.kwargs, module_version=module_version_info))
 
   def __getattr__(self, name):
     return getattr(self.module_info, name)

clarifai/client/search.py

@@ -1,3 +1,4 @@
+from math import ceil
 from typing import Any, Callable, Dict, Generator
 
 from clarifai_grpc.grpc.api import resources_pb2, service_pb2
@@ -41,11 +42,12 @@ def __init__(self,
     self.app_id = app_id
     self.metric_distance = dict(cosine="COSINE_DISTANCE", euclidean="EUCLIDEAN_DISTANCE")[metric]
     self.data_proto = resources_pb2.Data()
+    self.top_k = top_k
 
     self.inputs = Inputs(user_id=self.user_id, app_id=self.app_id)
     self.rank_filter_schema = get_schema()
     BaseClient.__init__(self, user_id=self.user_id, app_id=self.app_id, base=base_url)
-    Lister.__init__(self, page_size=top_k)
+    Lister.__init__(self, page_size=1000)
 
   def _get_annot_proto(self, **kwargs):
     """Get an Annotation proto message based on keyword arguments.
@@ -120,7 +122,7 @@ def _get_input_proto(self, **kwargs):
             self.data_proto.video.CopyFrom(resources_pb2.Video())
         self.input_proto.data.CopyFrom(self.data_proto)
       elif key == "input_dataset_ids":
-        self.input_proto.dataset_ids = value
+        self.input_proto.dataset_ids.extend(value)
       elif key == "input_status_code":
         self.input_proto.status.code = value
       else:
@@ -156,18 +158,31 @@ def list_all_pages_generator(
         Yields:
             response_dict: The next item in the listing.
         """
+    max_pages = ceil(self.top_k / self.default_page_size)
+    total_hits = 0
     page = 1
-    request_data['pagination'] = service_pb2.Pagination(page=page, per_page=self.default_page_size)
-    while True:
-      request_data['pagination'].page = page
+    while (page <= max_pages):
+      if (page == max_pages):
+        per_page = self.top_k - total_hits
+      else:
+        per_page = self.default_page_size
+      request_data['pagination'] = service_pb2.Pagination(page=page, per_page=per_page)
       response = self._grpc_request(endpoint, proto_message(**request_data))
       dict_response = MessageToDict(response, preserving_proto_field_name=True)
       if response.status.code != status_code_pb2.SUCCESS:
-        raise Exception(f"Listing failed with response {response!r}")
+        if "page * perPage cannot exceed" in str(response.status.details):
+          msg = (f"Your top_k is set to {self.top_k}. "
+                 f"The current pagination settings exceed the limit. Please reach out to "
+                 f"[email protected] to request an increase for your use case.\n"
+                 f"req_id: {response.status.req_id}")
+          raise UserError(msg)
+        else:
+          raise Exception(f"Listing failed with response {response!r}")
 
       if 'hits' not in list(dict_response.keys()):
         break
       page += 1
+      total_hits += per_page
       yield response
 
   def query(self, ranks=[{}], filters=[{}]):
@@ -188,7 +203,7 @@ def query(self, ranks=[{}], filters=[{}]):
 
         Vector search over inputs
         >>> from clarifai.client.search import Search
-        >>> search = Search(user_id='user_id', app_id='app_id', top_k=10, metric='cosine')
+        >>> search = Search(user_id='user_id', app_id='app_id', top_k=1, metric='cosine')
         >>> res = search.query(ranks=[{'image_url': 'https://samples.clarifai.com/dog.tiff'}])
 
     Note: For more detailed search examples, please refer to [examples](https://github.com/Clarifai/examples/tree/main/search).

clarifai/client/user.py

@@ -57,7 +57,7 @@ def list_apps(self, filter_by: Dict[str, Any] = {}, page_no: int = None,
         per_page=per_page,
         page_no=page_no)
     for app_info in all_apps_info:
-      yield App(**app_info)
+      yield App(base_url=self.base, **app_info)
 
   def list_runners(self, filter_by: Dict[str, Any] = {}, page_no: int = None,
                    per_page: int = None) -> Generator[Runner, None, None]:
@@ -89,7 +89,7 @@ def list_runners(self, filter_by: Dict[str, Any] = {}, page_no: int = None,
         page_no=page_no)
 
     for runner_info in all_runners_info:
-      yield Runner(check_runner_exists=False, **runner_info)
+      yield Runner(check_runner_exists=False, base_url=self.base, **runner_info)
 
   def create_app(self, app_id: str, base_workflow: str = 'Empty', **kwargs) -> App:
     """Creates an app for the user.
@@ -115,7 +115,7 @@ def create_app(self, app_id: str, base_workflow: str = 'Empty', **kwargs) -> App
     if response.status.code != status_code_pb2.SUCCESS:
       raise Exception(response.status)
     self.logger.info("\nApp created\n%s", response.status)
-    kwargs.update({'user_id': self.id})
+    kwargs.update({'user_id': self.id, 'base_url': self.base})
     return App(app_id=app_id, **kwargs)
 
   def create_runner(self, runner_id: str, labels: List[str], description: str) -> Runner:
@@ -151,7 +151,8 @@ def create_runner(self, runner_id: str, labels: List[str], description: str) ->
         user_id=self.id,
         labels=labels,
         description=description,
-        check_runner_exists=False)
+        check_runner_exists=False,
+        base_url=self.base)
 
   def app(self, app_id: str, **kwargs) -> App:
     """Returns an App object for the specified app ID.
@@ -174,6 +175,7 @@ def app(self, app_id: str, **kwargs) -> App:
       raise Exception(response.status)
 
     kwargs['user_id'] = self.id
+    kwargs.update({'base_url': self.base})
     return App(app_id=app_id, **kwargs)
 
   def runner(self, runner_id: str) -> Runner:
@@ -202,7 +204,7 @@ def runner(self, runner_id: str) -> Runner:
     kwargs = self.process_response_keys(dict_response[list(dict_response.keys())[1]],
                                         list(dict_response.keys())[1])
 
-    return Runner(check_runner_exists=False, **kwargs)
+    return Runner(check_runner_exists=False, base_url=self.base, **kwargs)
 
   def delete_app(self, app_id: str) -> None:
     """Deletes an app for the user.

clarifai/client/workflow.py

@@ -184,7 +184,10 @@ def list_versions(self, page_no: int = None,
     for workflow_version_info in all_workflow_versions_info:
       workflow_version_info['id'] = workflow_version_info['workflow_version_id']
       del workflow_version_info['workflow_version_id']
-      yield Workflow(workflow_id=self.id, **dict(self.kwargs, version=workflow_version_info))
+      yield Workflow(
+          workflow_id=self.id,
+          base_url=self.base,
+          **dict(self.kwargs, version=workflow_version_info))
 
   def export(self, out_path: str):
     """Exports the workflow to a yaml file.

clarifai/versions.py

@@ -1,6 +1,6 @@
 import os
 
-CLIENT_VERSION = "9.10.0"
+CLIENT_VERSION = "9.10.2"
 OS_VER = os.sys.platform
 PYTHON_VERSION = '.'.join(
     map(str, [os.sys.version_info.major, os.sys.version_info.minor, os.sys.version_info.micro]))

requirements.txt

@@ -1,12 +1,12 @@
 clarifai-grpc==9.10.0
-pandas==1.3.5
-numpy==1.22.0
+pandas>=1.3.5
+numpy>=1.22.0
 tqdm>=4.65.0
 omegaconf==2.2.3
-pycocotools==2.0.6
 opencv-python==4.7.0.68
 tritonclient==2.34.0
-rich==13.4.2
+rich>=13.4.2
 pytest==7.4.1
-PyYAML==6.0.1
+PyYAML>=6.0.1
 schema==0.7.5
+Pillow>=9.5.0

setup.py

@@ -34,6 +34,9 @@
     license="Apache 2.0",
     python_requires='>=3.8',
     install_requires=install_requires,
+    extras_require={
+        'all': ["pycocotools==2.0.6"],
+    },
     entry_points={
         "console_scripts": [
             "clarifai-model-upload-init = clarifai.models.model_serving.cli.repository:model_upload_init",

tests/test_app.py

@@ -75,7 +75,7 @@ def test_create_app(self):
 
   def test_create_search(self, create_app):
     search = create_app.search()
-    assert search.default_page_size == DEFAULT_TOP_K and search.metric_distance == "COSINE_DISTANCE"
+    assert search.top_k == DEFAULT_TOP_K and search.metric_distance == "COSINE_DISTANCE"
 
   def test_create_dataset(self, create_app):
     dataset = create_app.create_dataset(CREATE_DATASET_ID)