From 383d079b398f65f154bb26ffd0ab5cbb869ddfb2 Mon Sep 17 00:00:00 2001 From: Sai Nivedh Date: Wed, 22 Nov 2023 19:33:46 +0530 Subject: [PATCH 1/5] wip --- clarifai/utils/misc.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/clarifai/utils/misc.py b/clarifai/utils/misc.py index a36ba86b..259edc4d 100644 --- a/clarifai/utils/misc.py +++ b/clarifai/utils/misc.py @@ -1,4 +1,5 @@ -from typing import List +import os +from typing import List, Any, Dict, Optional class Chunker: @@ -31,3 +32,23 @@ def __next__(self): return 0.01 * (2**(self.count + 4)) else: return 0.01 * (2**10) # 10 seconds + + +def get_from_dict_or_env(data: Dict[str, Any], key: str, env_key: str) -> str: + """Get a value from a dictionary or an environment variable.""" + if key in data and data[key]: + return data[key] + else: + return get_from_env(key, env_key) + + +def get_from_env(key: str, env_key: str) -> str: + """Get a value from a dictionary or an environment variable.""" + if env_key in os.environ and os.environ[env_key]: + return os.environ[env_key] + else: + raise ValueError( + f"Did not find {key}, please add an environment variable" + f" `{env_key}` which contains it, or pass" + f" `{key}` as a named parameter." + ) \ No newline at end of file From 0b8ebb0b6839af1dc580e7ad13851a1e5cfa1286 Mon Sep 17 00:00:00 2001 From: Sai Nivedh Date: Wed, 22 Nov 2023 23:06:28 +0530 Subject: [PATCH 2/5] wip --- clarifai/client/base.py | 6 +++--- clarifai/client/user.py | 4 +++- clarifai/utils/misc.py | 3 ++- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/clarifai/client/base.py b/clarifai/client/base.py index 6f37883f..8704b821 100644 --- a/clarifai/client/base.py +++ b/clarifai/client/base.py @@ -31,9 +31,9 @@ class BaseClient: """ def __init__(self, **kwargs): - pat = os.environ.get('CLARIFAI_PAT', "") - if pat == "": - raise UserError("CLARIFAI_PAT must be set as env vars") + # pat = os.environ.get('CLARIFAI_PAT', "") + # if pat == "": + # raise UserError("CLARIFAI_PAT must be set as env vars") self.auth_helper = ClarifaiAuthHelper(**kwargs, pat=pat, validate=False) self.STUB = create_stub(self.auth_helper) self.metadata = self.auth_helper.metadata diff --git a/clarifai/client/user.py b/clarifai/client/user.py index 605c2f60..b07033b9 100644 --- a/clarifai/client/user.py +++ b/clarifai/client/user.py @@ -10,6 +10,7 @@ from clarifai.client.runner import Runner from clarifai.errors import UserError from clarifai.utils.logging import get_logger +from clarifai.utils.misc import get_from_dict_or_env class User(Lister, BaseClient): @@ -26,7 +27,8 @@ def __init__(self, user_id: str = "", base_url: str = "https://api.clarifai.com" self.kwargs = {**kwargs, 'id': user_id} self.user_info = resources_pb2.User(**self.kwargs) self.logger = get_logger(logger_level="INFO", name=__name__) - BaseClient.__init__(self, user_id=self.id, app_id="", base=base_url) + BaseClient.__init__(self, user_id=self.id, app_id="", + base=base_url, pat=get_from_dict_or_env(**kwargs, key="pat", env_key="CLARIFAI_PAT")) Lister.__init__(self) def list_apps(self, filter_by: Dict[str, Any] = {}, page_no: int = None, diff --git a/clarifai/utils/misc.py b/clarifai/utils/misc.py index 259edc4d..961c87a3 100644 --- a/clarifai/utils/misc.py +++ b/clarifai/utils/misc.py @@ -1,5 +1,6 @@ import os from typing import List, Any, Dict, Optional +from clarifai.errors import UserError class Chunker: @@ -47,7 +48,7 @@ def get_from_env(key: str, env_key: str) -> str: if env_key in os.environ and os.environ[env_key]: return os.environ[env_key] else: - raise ValueError( + raise UserError( f"Did not find {key}, please add an environment variable" f" `{env_key}` which contains it, or pass" f" `{key}` as a named parameter." From 802ea09c61aab9d00cd26192ed5aba220d974d82 Mon Sep 17 00:00:00 2001 From: Sai Nivedh Date: Thu, 23 Nov 2023 09:39:14 +0530 Subject: [PATCH 3/5] support pat as arg --- clarifai/client/app.py | 48 ++++++++++++++++++++++++------------- clarifai/client/base.py | 12 +++++----- clarifai/client/dataset.py | 10 +++++--- clarifai/client/input.py | 3 ++- clarifai/client/model.py | 7 ++++-- clarifai/client/module.py | 5 +++- clarifai/client/runner.py | 4 +++- clarifai/client/search.py | 6 +++-- clarifai/client/user.py | 24 +++++++++++-------- clarifai/client/workflow.py | 4 +++- clarifai/utils/misc.py | 19 +++++++-------- tests/test_app.py | 12 ++++++---- 12 files changed, 96 insertions(+), 58 deletions(-) diff --git a/clarifai/client/app.py b/clarifai/client/app.py index 723d849f..c7fc740d 100644 --- a/clarifai/client/app.py +++ b/clarifai/client/app.py @@ -31,6 +31,7 @@ def __init__(self, url: str = "", app_id: str = "", base_url: str = "https://api.clarifai.com", + pat: str = "", **kwargs): """Initializes an App object. @@ -38,6 +39,7 @@ def __init__(self, url (str): The URL to initialize the app object. app_id (str): The App ID for the App to interact with. base_url (str): Base API url. Default "https://api.clarifai.com" + pat (str): A personal access token for authentication. Can be set as env var CLARIFAI_PAT **kwargs: Additional keyword arguments to be passed to the App. - name (str): The name of the app. - description (str): The description of the app. @@ -50,7 +52,7 @@ def __init__(self, self.kwargs = {**kwargs, 'id': app_id} self.app_info = resources_pb2.App(**self.kwargs) self.logger = get_logger(logger_level="INFO", name=__name__) - BaseClient.__init__(self, user_id=self.user_id, app_id=self.id, base=base_url) + BaseClient.__init__(self, user_id=self.user_id, app_id=self.id, base=base_url, pat=pat) Lister.__init__(self) def list_datasets(self, page_no: int = None, @@ -83,7 +85,7 @@ def list_datasets(self, page_no: int = None, for dataset_info in all_datasets_info: if 'version' in list(dataset_info.keys()): del dataset_info['version']['metrics'] - yield Dataset(base_url=self.base, **dataset_info) + yield Dataset(base_url=self.base, pat=self.pat, **dataset_info) def list_models(self, filter_by: Dict[str, Any] = {}, @@ -124,7 +126,7 @@ def list_models(self, if only_in_app: if model_info['app_id'] != self.id: continue - yield Model(base_url=self.base, **model_info) + yield Model(base_url=self.base, pat=self.pat, **model_info) def list_workflows(self, filter_by: Dict[str, Any] = {}, @@ -163,7 +165,7 @@ def list_workflows(self, if only_in_app: if workflow_info['app_id'] != self.id: continue - yield Workflow(base_url=self.base, **workflow_info) + yield Workflow(base_url=self.base, pat=self.pat, **workflow_info) def list_modules(self, filter_by: Dict[str, Any] = {}, @@ -202,7 +204,7 @@ def list_modules(self, if only_in_app: if module_info['app_id'] != self.id: continue - yield Module(base_url=self.base, **module_info) + yield Module(base_url=self.base, pat=self.pat, **module_info) def list_installed_module_versions(self, filter_by: Dict[str, Any] = {}, @@ -238,7 +240,10 @@ def list_installed_module_versions(self, del imv_info['deploy_url'] del imv_info['installed_module_version_id'] # TODO: remove this after the backend fix yield Module( - module_id=imv_info['module_version']['module_id'], base_url=self.base, **imv_info) + module_id=imv_info['module_version']['module_id'], + base_url=self.base, + pat=self.pat, + **imv_info) def list_concepts(self, page_no: int = None, per_page: int = None) -> Generator[Concept, None, None]: @@ -303,7 +308,12 @@ def create_dataset(self, dataset_id: str, **kwargs) -> Dataset: if response.status.code != status_code_pb2.SUCCESS: raise Exception(response.status) self.logger.info("\nDataset created\n%s", response.status) - kwargs.update({'app_id': self.id, 'user_id': self.user_id, 'base_url': self.base}) + kwargs.update({ + 'app_id': self.id, + 'user_id': self.user_id, + 'base_url': self.base, + 'pat': self.pat + }) return Dataset(dataset_id=dataset_id, **kwargs) @@ -332,7 +342,8 @@ def create_model(self, model_id: str, **kwargs) -> Model: 'app_id': self.id, 'user_id': self.user_id, 'model_type_id': response.model.model_type_id, - 'base_url': self.base + 'base_url': self.base, + 'pat': self.pat }) return Model(model_id=model_id, **kwargs) @@ -425,7 +436,7 @@ def create_workflow(self, display_workflow_tree(dict_response["workflows"][0]["nodes"]) kwargs = self.process_response_keys(dict_response[list(dict_response.keys())[1]][0], "workflow") - kwargs.update({'base_url': self.base}) + kwargs.update({'base_url': self.base, 'pat': self.pat}) return Workflow(**kwargs) @@ -453,7 +464,12 @@ def create_module(self, module_id: str, description: str, **kwargs) -> Module: if response.status.code != status_code_pb2.SUCCESS: raise Exception(response.status) self.logger.info("\nModule created\n%s", response.status) - kwargs.update({'app_id': self.id, 'user_id': self.user_id, 'base_url': self.base}) + kwargs.update({ + 'app_id': self.id, + 'user_id': self.user_id, + 'base_url': self.base, + 'pat': self.pat + }) return Module(module_id=module_id, **kwargs) @@ -480,7 +496,7 @@ def dataset(self, dataset_id: str, **kwargs) -> Dataset: kwargs = self.process_response_keys(dict_response[list(dict_response.keys())[1]], list(dict_response.keys())[1]) kwargs['version'] = response.dataset.version if response.dataset.version else None - kwargs.update({'base_url': self.base}) + kwargs.update({'base_url': self.base, 'pat': self.pat}) return Dataset(**kwargs) def model(self, model_id: str, model_version_id: str = "", **kwargs) -> Model: @@ -516,7 +532,7 @@ def model(self, model_id: str, model_version_id: str = "", **kwargs) -> Model: kwargs = self.process_response_keys(dict_response['model'], 'model') kwargs[ 'model_version'] = response.model.model_version if response.model.model_version else None - kwargs.update({'base_url': self.base}) + kwargs.update({'base_url': self.base, 'pat': self.pat}) return Model(**kwargs) @@ -542,7 +558,7 @@ def workflow(self, workflow_id: str, **kwargs) -> Workflow: dict_response = MessageToDict(response, preserving_proto_field_name=True) kwargs = self.process_response_keys(dict_response[list(dict_response.keys())[1]], list(dict_response.keys())[1]) - kwargs.update({'base_url': self.base}) + kwargs.update({'base_url': self.base, 'pat': self.pat}) return Workflow(**kwargs) @@ -569,7 +585,7 @@ def module(self, module_id: str, module_version_id: str = "", **kwargs) -> Modul raise Exception(response.status) dict_response = MessageToDict(response, preserving_proto_field_name=True) kwargs = self.process_response_keys(dict_response['module'], 'module') - kwargs.update({'base_url': self.base}) + kwargs.update({'base_url': self.base, 'pat': self.pat}) return Module(**kwargs) @@ -579,7 +595,7 @@ def inputs(self,): Returns: Inputs: An input object. """ - return Inputs(self.user_id, self.id, base_url=self.base) + return Inputs(self.user_id, self.id, base_url=self.base, pat=self.pat) def delete_dataset(self, dataset_id: str) -> None: """Deletes an dataset for the user. @@ -670,7 +686,7 @@ def search(self, **kwargs) -> Model: """ user_id = kwargs.get("user_id", self.user_app_id.user_id) app_id = kwargs.get("app_id", self.user_app_id.app_id) - return Search(user_id=user_id, app_id=app_id, base_url=self.base, **kwargs) + return Search(user_id=user_id, app_id=app_id, base_url=self.base, pat=self.pat, **kwargs) def __getattr__(self, name): return getattr(self.app_info, name) diff --git a/clarifai/client/base.py b/clarifai/client/base.py index 8704b821..4ca4dc4d 100644 --- a/clarifai/client/base.py +++ b/clarifai/client/base.py @@ -1,4 +1,3 @@ -import os from datetime import datetime from typing import Any, Callable @@ -8,7 +7,8 @@ from clarifai.client.auth import create_stub from clarifai.client.auth.helper import ClarifaiAuthHelper -from clarifai.errors import ApiError, UserError +from clarifai.errors import ApiError +from clarifai.utils.misc import get_from_dict_or_env class BaseClient: @@ -31,12 +31,12 @@ class BaseClient: """ def __init__(self, **kwargs): - # pat = os.environ.get('CLARIFAI_PAT', "") - # if pat == "": - # raise UserError("CLARIFAI_PAT must be set as env vars") - self.auth_helper = ClarifaiAuthHelper(**kwargs, pat=pat, validate=False) + pat = get_from_dict_or_env(key="pat", env_key="CLARIFAI_PAT", **kwargs) + kwargs.update({'pat': pat}) + self.auth_helper = ClarifaiAuthHelper(**kwargs, validate=False) self.STUB = create_stub(self.auth_helper) self.metadata = self.auth_helper.metadata + self.pat = self.auth_helper.pat self.user_app_id = self.auth_helper.get_user_app_id_proto() self.base = self.auth_helper.base diff --git a/clarifai/client/dataset.py b/clarifai/client/dataset.py index 7187d30f..aa977c2c 100644 --- a/clarifai/client/dataset.py +++ b/clarifai/client/dataset.py @@ -37,6 +37,7 @@ def __init__(self, url: str = "", dataset_id: str = "", base_url: str = "https://api.clarifai.com", + pat: str = "", **kwargs): """Initializes a Dataset object. @@ -44,6 +45,7 @@ def __init__(self, url (str): The URL to initialize the dataset object. dataset_id (str): The Dataset ID within the App to interact with. base_url (str): Base API url. Default "https://api.clarifai.com" + pat (str): A personal access token for authentication. Can be set as env var CLARIFAI_PAT **kwargs: Additional keyword arguments to be passed to the Dataset. """ if url != "" and dataset_id != "": @@ -61,7 +63,7 @@ def __init__(self, self.task = None # Upload dataset type self.input_object = Inputs(user_id=self.user_id, app_id=self.app_id) self.logger = get_logger(logger_level="INFO") - BaseClient.__init__(self, user_id=self.user_id, app_id=self.app_id, base=base_url) + BaseClient.__init__(self, user_id=self.user_id, app_id=self.app_id, base=base_url, pat=pat) Lister.__init__(self) def create_version(self, **kwargs) -> 'Dataset': @@ -94,7 +96,8 @@ def create_version(self, **kwargs) -> 'Dataset': 'app_id': self.app_id, 'user_id': self.user_id, 'version': response.dataset_versions[0], - 'base_url': self.base + 'base_url': self.base, + 'pat': self.pat }) return Dataset(**kwargs) @@ -157,7 +160,8 @@ def list_versions(self, page_no: int = None, 'app_id': self.app_id, 'user_id': self.user_id, 'version': resources_pb2.DatasetVersion(**dataset_version_info), - 'base_url': self.base + 'base_url': self.base, + 'pat': self.pat } yield Dataset(**kwargs) diff --git a/clarifai/client/input.py b/clarifai/client/input.py index db2be082..b71baa94 100644 --- a/clarifai/client/input.py +++ b/clarifai/client/input.py @@ -29,6 +29,7 @@ def __init__(self, app_id: str = "", logger_level: str = "INFO", base_url: str = "https://api.clarifai.com", + pat: str = "", **kwargs): """Initializes an Input object. @@ -43,7 +44,7 @@ def __init__(self, self.kwargs = {**kwargs} self.input_info = resources_pb2.Input(**self.kwargs) self.logger = get_logger(logger_level=logger_level, name=__name__) - BaseClient.__init__(self, user_id=self.user_id, app_id=self.app_id, base=base_url) + BaseClient.__init__(self, user_id=self.user_id, app_id=self.app_id, base=base_url, pat=pat) Lister.__init__(self) def _get_proto(self, diff --git a/clarifai/client/model.py b/clarifai/client/model.py index 163a80fb..2e47774f 100644 --- a/clarifai/client/model.py +++ b/clarifai/client/model.py @@ -31,6 +31,7 @@ def __init__(self, model_id: str = "", model_version: Dict = {'id': ""}, base_url: str = "https://api.clarifai.com", + pat: str = "", **kwargs): """Initializes a Model object. @@ -39,6 +40,7 @@ def __init__(self, model_id (str): The Model ID to interact with. model_version (dict): The Model Version to interact with. base_url (str): Base API url. Default "https://api.clarifai.com" + pat (str): A personal access token for authentication. Can be set as env var CLARIFAI_PAT **kwargs: Additional keyword arguments to be passed to the Model. """ if url != "" and model_id != "": @@ -53,7 +55,7 @@ def __init__(self, self.model_info = resources_pb2.Model(**self.kwargs) self.logger = get_logger(logger_level="INFO") self.training_params = {} - BaseClient.__init__(self, user_id=self.user_id, app_id=self.app_id, base=base_url) + BaseClient.__init__(self, user_id=self.user_id, app_id=self.app_id, base=base_url, pat=pat) Lister.__init__(self) def list_training_templates(self) -> List[str]: @@ -329,7 +331,7 @@ def create_version(self, **kwargs) -> 'Model': dict_response = MessageToDict(response, preserving_proto_field_name=True) kwargs = self.process_response_keys(dict_response['model'], 'model') - return Model(base_url=self.base, **kwargs) + return Model(base_url=self.base, pat=self.pat, **kwargs) def list_versions(self, page_no: int = None, per_page: int = None) -> Generator['Model', None, None]: @@ -374,6 +376,7 @@ def list_versions(self, page_no: int = None, yield Model( model_id=self.id, base_url=self.base, + pat=self.pat, **dict(self.kwargs, model_version=model_version_info)) def predict(self, inputs: List[Input], inference_params: Dict = {}, output_config: Dict = {}): diff --git a/clarifai/client/module.py b/clarifai/client/module.py index 7f2c6b66..937dbcd4 100644 --- a/clarifai/client/module.py +++ b/clarifai/client/module.py @@ -17,6 +17,7 @@ def __init__(self, module_id: str = "", module_version: Dict = {'id': ""}, base_url: str = "https://api.clarifai.com", + pat: str = "", **kwargs): """Initializes a Module object. @@ -25,6 +26,7 @@ def __init__(self, module_id (str): The Module ID to interact with. module_version (dict): The Module Version to interact with. base_url (str): Base API url. Default "https://api.clarifai.com" + pat (str): A personal access token for authentication. Can be set as env var CLARIFAI_PAT **kwargs: Additional keyword arguments to be passed to the Module. """ if url != "" and module_id != "": @@ -39,7 +41,7 @@ def __init__(self, self.kwargs = {**kwargs, 'id': module_id, 'module_version': module_version} self.module_info = resources_pb2.Module(**self.kwargs) self.logger = get_logger(logger_level="INFO") - BaseClient.__init__(self, user_id=self.user_id, app_id=self.app_id, base=base_url) + BaseClient.__init__(self, user_id=self.user_id, app_id=self.app_id, base=base_url, pat=pat) Lister.__init__(self) def list_versions(self, page_no: int = None, @@ -79,6 +81,7 @@ def list_versions(self, page_no: int = None, yield Module( module_id=self.id, base_url=self.base, + pat=self.pat, **dict(self.kwargs, module_version=module_version_info)) def __getattr__(self, name): diff --git a/clarifai/client/runner.py b/clarifai/client/runner.py index 45f8862c..54178a1a 100644 --- a/clarifai/client/runner.py +++ b/clarifai/client/runner.py @@ -35,12 +35,14 @@ def __init__(self, user_id: str = "", check_runner_exists: bool = True, base_url: str = "https://api.clarifai.com", + pat: str = "", **kwargs) -> None: """ Args: runner_id (str): the id of the runner to use. Create the runner in the Clarifai API first user_id (str): Clarifai User ID base_url (str): Base API url. Default "https://api.clarifai.com" + pat (str): A personal access token for authentication. Can be set as env var CLARIFAI_PAT """ user_id = user_id or os.environ.get("CLARIFAI_USER_ID", "") @@ -52,7 +54,7 @@ def __init__(self, self.logger = get_logger("INFO", __name__) self.kwargs = {**kwargs, 'id': runner_id, 'user_id': user_id} self.runner_info = resources_pb2.Runner(**self.kwargs) - BaseClient.__init__(self, user_id=self.user_id, app_id="", base=base_url) + BaseClient.__init__(self, user_id=self.user_id, app_id="", base=base_url, pat=pat) # Check that the runner exists. if check_runner_exists: diff --git a/clarifai/client/search.py b/clarifai/client/search.py index c000137e..5798afed 100644 --- a/clarifai/client/search.py +++ b/clarifai/client/search.py @@ -22,7 +22,8 @@ def __init__(self, app_id, top_k: int = DEFAULT_TOP_K, metric: str = DEFAULT_SEARCH_METRIC, - base_url: str = "https://api.clarifai.com"): + base_url: str = "https://api.clarifai.com", + pat: str = ""): """Initialize the Search object. Args: @@ -31,6 +32,7 @@ def __init__(self, top_k (int, optional): Top K results to retrieve. Defaults to 10. metric (str, optional): Similarity metric (either 'cosine' or 'euclidean'). Defaults to 'cosine'. base_url (str, optional): Base API url. Defaults to "https://api.clarifai.com". + pat (str, optional): A personal access token for authentication. Can be set as env var CLARIFAI_PAT Raises: UserError: If the metric is not 'cosine' or 'euclidean'. @@ -46,7 +48,7 @@ def __init__(self, self.inputs = Inputs(user_id=self.user_id, app_id=self.app_id) self.rank_filter_schema = get_schema() - BaseClient.__init__(self, user_id=self.user_id, app_id=self.app_id, base=base_url) + BaseClient.__init__(self, user_id=self.user_id, app_id=self.app_id, base=base_url, pat=pat) Lister.__init__(self, page_size=1000) def _get_annot_proto(self, **kwargs): diff --git a/clarifai/client/user.py b/clarifai/client/user.py index b07033b9..68041aaf 100644 --- a/clarifai/client/user.py +++ b/clarifai/client/user.py @@ -10,25 +10,28 @@ from clarifai.client.runner import Runner from clarifai.errors import UserError from clarifai.utils.logging import get_logger -from clarifai.utils.misc import get_from_dict_or_env class User(Lister, BaseClient): """User is a class that provides access to Clarifai API endpoints related to user information.""" - def __init__(self, user_id: str = "", base_url: str = "https://api.clarifai.com", **kwargs): + def __init__(self, + user_id: str = "", + base_url: str = "https://api.clarifai.com", + pat: str = "", + **kwargs): """Initializes an User object. Args: user_id (str): The user ID for the user to interact with. base_url (str): Base API url. Default "https://api.clarifai.com" + pat (str): A personal access token for authentication. Can be set as env var CLARIFAI_PAT **kwargs: Additional keyword arguments to be passed to the User. """ self.kwargs = {**kwargs, 'id': user_id} self.user_info = resources_pb2.User(**self.kwargs) self.logger = get_logger(logger_level="INFO", name=__name__) - BaseClient.__init__(self, user_id=self.id, app_id="", - base=base_url, pat=get_from_dict_or_env(**kwargs, key="pat", env_key="CLARIFAI_PAT")) + BaseClient.__init__(self, user_id=self.id, app_id="", base=base_url, pat=pat) Lister.__init__(self) def list_apps(self, filter_by: Dict[str, Any] = {}, page_no: int = None, @@ -59,7 +62,7 @@ def list_apps(self, filter_by: Dict[str, Any] = {}, page_no: int = None, per_page=per_page, page_no=page_no) for app_info in all_apps_info: - yield App(base_url=self.base, **app_info) + yield App(base_url=self.base, pat=self.pat, **app_info) def list_runners(self, filter_by: Dict[str, Any] = {}, page_no: int = None, per_page: int = None) -> Generator[Runner, None, None]: @@ -91,7 +94,7 @@ def list_runners(self, filter_by: Dict[str, Any] = {}, page_no: int = None, page_no=page_no) for runner_info in all_runners_info: - yield Runner(check_runner_exists=False, base_url=self.base, **runner_info) + yield Runner(check_runner_exists=False, base_url=self.base, pat=self.pat, **runner_info) def create_app(self, app_id: str, base_workflow: str = 'Empty', **kwargs) -> App: """Creates an app for the user. @@ -117,7 +120,7 @@ def create_app(self, app_id: str, base_workflow: str = 'Empty', **kwargs) -> App if response.status.code != status_code_pb2.SUCCESS: raise Exception(response.status) self.logger.info("\nApp created\n%s", response.status) - kwargs.update({'user_id': self.id, 'base_url': self.base}) + kwargs.update({'user_id': self.id, 'base_url': self.base, 'pat': self.pat}) return App(app_id=app_id, **kwargs) def create_runner(self, runner_id: str, labels: List[str], description: str) -> Runner: @@ -154,7 +157,8 @@ def create_runner(self, runner_id: str, labels: List[str], description: str) -> labels=labels, description=description, check_runner_exists=False, - base_url=self.base) + base_url=self.base, + pat=self.pat) def app(self, app_id: str, **kwargs) -> App: """Returns an App object for the specified app ID. @@ -177,7 +181,7 @@ def app(self, app_id: str, **kwargs) -> App: raise Exception(response.status) kwargs['user_id'] = self.id - kwargs.update({'base_url': self.base}) + kwargs.update({'base_url': self.base, 'pat': self.pat}) return App(app_id=app_id, **kwargs) def runner(self, runner_id: str) -> Runner: @@ -206,7 +210,7 @@ def runner(self, runner_id: str) -> Runner: kwargs = self.process_response_keys(dict_response[list(dict_response.keys())[1]], list(dict_response.keys())[1]) - return Runner(check_runner_exists=False, base_url=self.base, **kwargs) + return Runner(check_runner_exists=False, base_url=self.base, pat=self.pat, **kwargs) def delete_app(self, app_id: str) -> None: """Deletes an app for the user. diff --git a/clarifai/client/workflow.py b/clarifai/client/workflow.py index 5f635a45..60211a86 100644 --- a/clarifai/client/workflow.py +++ b/clarifai/client/workflow.py @@ -23,6 +23,7 @@ def __init__(self, workflow_version: Dict = {'id': ""}, output_config: Dict = {'min_value': 0}, base_url: str = "https://api.clarifai.com", + pat: str = "", **kwargs): """Initializes a Workflow object. @@ -51,7 +52,7 @@ def __init__(self, self.output_config = output_config self.workflow_info = resources_pb2.Workflow(**self.kwargs) self.logger = get_logger(logger_level="INFO") - BaseClient.__init__(self, user_id=self.user_id, app_id=self.app_id, base=base_url) + BaseClient.__init__(self, user_id=self.user_id, app_id=self.app_id, base=base_url, pat=pat) Lister.__init__(self) def predict(self, inputs: List[Input]): @@ -187,6 +188,7 @@ def list_versions(self, page_no: int = None, yield Workflow( workflow_id=self.id, base_url=self.base, + pat=self.pat, **dict(self.kwargs, version=workflow_version_info)) def export(self, out_path: str): diff --git a/clarifai/utils/misc.py b/clarifai/utils/misc.py index 961c87a3..3ffe3dd0 100644 --- a/clarifai/utils/misc.py +++ b/clarifai/utils/misc.py @@ -1,5 +1,6 @@ import os -from typing import List, Any, Dict, Optional +from typing import List + from clarifai.errors import UserError @@ -35,21 +36,19 @@ def __next__(self): return 0.01 * (2**10) # 10 seconds -def get_from_dict_or_env(data: Dict[str, Any], key: str, env_key: str) -> str: +def get_from_dict_or_env(key: str, env_key: str, **data) -> str: """Get a value from a dictionary or an environment variable.""" if key in data and data[key]: - return data[key] + return data[key] else: - return get_from_env(key, env_key) + return get_from_env(key, env_key) def get_from_env(key: str, env_key: str) -> str: """Get a value from a dictionary or an environment variable.""" if env_key in os.environ and os.environ[env_key]: - return os.environ[env_key] + return os.environ[env_key] else: - raise UserError( - f"Did not find {key}, please add an environment variable" - f" `{env_key}` which contains it, or pass" - f" `{key}` as a named parameter." - ) \ No newline at end of file + raise UserError(f"Did not find `{key}`, please add an environment variable" + f" `{env_key}` which contains it, or pass" + f" `{key}` as a named parameter.") diff --git a/tests/test_app.py b/tests/test_app.py index 3f2dc5b5..14e494d1 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -21,20 +21,22 @@ CREATE_MODULE_ID = f"ci_test_module_{NOW}" CREATE_RUNNER_ID = f"ci_test_runner_{NOW}" +CLARIFAI_PAT = os.environ["CLARIFAI_PAT"] + @pytest.fixture def create_app(): - return App(user_id=CREATE_APP_USER_ID, app_id=CREATE_APP_ID) + return App(user_id=CREATE_APP_USER_ID, app_id=CREATE_APP_ID, pat=CLARIFAI_PAT) @pytest.fixture def app(): - return App(user_id=MAIN_APP_USER_ID, app_id=MAIN_APP_ID) + return App(user_id=MAIN_APP_USER_ID, app_id=MAIN_APP_ID, pat=CLARIFAI_PAT) @pytest.fixture def client(): - return User(user_id=MAIN_APP_USER_ID) + return User(user_id=MAIN_APP_USER_ID, pat=CLARIFAI_PAT) class TestApp: @@ -70,7 +72,7 @@ def test_get_workflow(self, client): assert workflow.id == General_Workflow_ID and workflow.app_id == MAIN_APP_ID and workflow.user_id == MAIN_APP_USER_ID def test_create_app(self): - app = User(user_id=CREATE_APP_USER_ID).create_app(app_id=CREATE_APP_ID) + app = User(user_id=CREATE_APP_USER_ID, pat=CLARIFAI_PAT).create_app(app_id=CREATE_APP_ID) assert app.id == CREATE_APP_ID and app.user_id == CREATE_APP_USER_ID def test_create_search(self, create_app): @@ -90,7 +92,7 @@ def test_create_module(self, create_app): assert module.id == CREATE_MODULE_ID and module.app_id == CREATE_APP_ID and module.user_id == CREATE_APP_USER_ID def test_create_runner(self, client): - client = User(user_id=CREATE_APP_USER_ID) + client = User(user_id=CREATE_APP_USER_ID, pat=CLARIFAI_PAT) runner = client.create_runner( CREATE_RUNNER_ID, labels=["ci runner"], description="CI test runner") assert runner.id == CREATE_RUNNER_ID and runner.user_id == CREATE_APP_USER_ID From 6c61586a4c75066fc8c74a09d55d26eee5a165dd Mon Sep 17 00:00:00 2001 From: Sai Nivedh Date: Thu, 23 Nov 2023 13:21:45 +0530 Subject: [PATCH 4/5] use staticmethods in inputs and necessary changes --- clarifai/client/input.py | 84 ++++++++++++++----------------- clarifai/client/model.py | 16 +++--- clarifai/client/workflow.py | 16 +++--- clarifai/datasets/upload/image.py | 15 ++---- clarifai/datasets/upload/text.py | 3 +- tests/test_model_predict.py | 10 ++-- tests/workflow/test_predict.py | 6 ++- 7 files changed, 71 insertions(+), 79 deletions(-) diff --git a/clarifai/client/input.py b/clarifai/client/input.py index b71baa94..1e9b7682 100644 --- a/clarifai/client/input.py +++ b/clarifai/client/input.py @@ -47,8 +47,8 @@ def __init__(self, BaseClient.__init__(self, user_id=self.user_id, app_id=self.app_id, base=base_url, pat=pat) Lister.__init__(self) - def _get_proto(self, - input_id: str, + @staticmethod + def _get_proto(input_id: str, dataset_id: Union[str, None], imagepb: Image = None, video_pb: Video = None, @@ -107,8 +107,8 @@ def _get_proto(self, concepts=concepts, metadata=metadata)) - def get_input_from_url(self, - input_id: str, + @staticmethod + def get_input_from_url(input_id: str, image_url: str = None, video_url: str = None, audio_url: str = None, @@ -130,8 +130,7 @@ def get_input_from_url(self, Example: >>> from clarifai.client.input import Inputs - >>> input_obj = Inputs() - >>> input_proto = input_obj.get_input_from_url(input_id = 'demo', image_url='https://samples.clarifai.com/metro-north.jpg') + >>> input_proto = Inputs.get_input_from_url(input_id = 'demo', image_url='https://samples.clarifai.com/metro-north.jpg') """ if not any((image_url, video_url, audio_url, text_url)): raise ValueError( @@ -140,7 +139,7 @@ def get_input_from_url(self, video_pb = resources_pb2.Video(url=video_url) if video_url else None audio_pb = resources_pb2.Audio(url=audio_url) if audio_url else None text_pb = resources_pb2.Text(url=text_url) if text_url else None - return self._get_proto( + return Inputs._get_proto( input_id=input_id, dataset_id=dataset_id, imagepb=image_pb, @@ -149,8 +148,8 @@ def get_input_from_url(self, text_pb=text_pb, **kwargs) - def get_input_from_file(self, - input_id: str, + @staticmethod + def get_input_from_file(input_id: str, image_file: str = None, video_file: str = None, audio_file: str = None, @@ -172,8 +171,7 @@ def get_input_from_file(self, Example: >>> from clarifai.client.input import Inputs - >>> input_obj = Inputs() - >>> input_proto = input_obj.get_input_from_file(input_id = 'demo', video_file='file_path') + >>> input_proto = Inputs.get_input_from_file(input_id = 'demo', video_file='file_path') """ if not any((image_file, video_file, audio_file, text_file)): raise ValueError( @@ -182,7 +180,7 @@ def get_input_from_file(self, video_pb = resources_pb2.Video(base64=open(video_file, 'rb').read()) if video_file else None audio_pb = resources_pb2.Audio(base64=open(audio_file, 'rb').read()) if audio_file else None text_pb = resources_pb2.Text(raw=open(text_file, 'rb').read()) if text_file else None - return self._get_proto( + return Inputs._get_proto( input_id=input_id, dataset_id=dataset_id, imagepb=image_pb, @@ -191,8 +189,8 @@ def get_input_from_file(self, text_pb=text_pb, **kwargs) - def get_input_from_bytes(self, - input_id: str, + @staticmethod + def get_input_from_bytes(input_id: str, image_bytes: bytes = None, video_bytes: bytes = None, audio_bytes: bytes = None, @@ -214,10 +212,9 @@ def get_input_from_bytes(self, Example: >>> from clarifai.client.input import Inputs - >>> input_obj = Inputs() >>> image = open('demo.jpg', 'rb').read() >>> video = open('demo.mp4', 'rb').read() - >>> input_proto = input_obj.get_input_from_bytes(input_id = 'demo',image_bytes =image, video_bytes=video) + >>> input_proto = Inputs.get_input_from_bytes(input_id = 'demo',image_bytes =image, video_bytes=video) """ if not any((image_bytes, video_bytes, audio_bytes, text_bytes)): raise ValueError( @@ -226,7 +223,7 @@ def get_input_from_bytes(self, video_pb = resources_pb2.Video(base64=video_bytes) if video_bytes else None audio_pb = resources_pb2.Audio(base64=audio_bytes) if audio_bytes else None text_pb = resources_pb2.Text(raw=text_bytes) if text_bytes else None - return self._get_proto( + return Inputs._get_proto( input_id=input_id, dataset_id=dataset_id, imagepb=image_pb, @@ -235,9 +232,8 @@ def get_input_from_bytes(self, text_pb=text_pb, **kwargs) - def get_image_inputs_from_folder(self, - folder_path: str, - dataset_id: str = None, + @staticmethod + def get_image_inputs_from_folder(folder_path: str, dataset_id: str = None, labels: bool = False) -> List[Input]: #image specific """Create input protos for image data type from folder. @@ -249,8 +245,7 @@ def get_image_inputs_from_folder(self, Example: >>> from clarifai.client.input import Inputs - >>> input_obj = Inputs() - >>> input_protos = input_obj.get_image_inputs_from_folder(folder_path='demo_folder') + >>> input_protos = Inputs.get_image_inputs_from_folder(folder_path='demo_folder') """ input_protos = [] labels = [folder_path.split('/')[-1]] if labels else None @@ -260,11 +255,12 @@ def get_image_inputs_from_folder(self, input_id = filename.split('.')[0] image_pb = resources_pb2.Image(base64=open(os.path.join(folder_path, filename), 'rb').read()) input_protos.append( - self._get_proto( + Inputs._get_proto( input_id=input_id, dataset_id=dataset_id, imagepb=image_pb, labels=labels)) return input_protos - def get_text_input(self, input_id: str, raw_text: str, dataset_id: str = None, + @staticmethod + def get_text_input(input_id: str, raw_text: str, dataset_id: str = None, **kwargs) -> Text: #text specific """Create input proto for text data type from rawtext. @@ -279,14 +275,13 @@ def get_text_input(self, input_id: str, raw_text: str, dataset_id: str = None, Example: >>> from clarifai.client.input import Inputs - >>> input_obj = Inputs() - >>> input_protos = input_obj.get_text_input(input_id = 'demo', raw_text = 'This is a test') + >>> input_protos = Inputs.get_text_input(input_id = 'demo', raw_text = 'This is a test') """ text_pb = resources_pb2.Text(raw=raw_text) - return self._get_proto(input_id=input_id, dataset_id=dataset_id, text_pb=text_pb, **kwargs) + return Inputs._get_proto(input_id=input_id, dataset_id=dataset_id, text_pb=text_pb, **kwargs) - def get_inputs_from_csv(self, - csv_path: str, + @staticmethod + def get_inputs_from_csv(csv_path: str, input_type: str = 'text', csv_type: str = 'raw', dataset_id: str = None, @@ -305,8 +300,7 @@ def get_inputs_from_csv(self, Example: >>> from clarifai.client.input import Inputs - >>> input_obj = Inputs() - >>> input_protos = input_obj.get_inputs_from_csv(csv_path='filepath', input_type='text', csv_type='raw') + >>> input_protos = Inputs.get_inputs_from_csv(csv_path='filepath', input_type='text', csv_type='raw') """ input_protos = [] with open(csv_path) as _file: @@ -357,7 +351,7 @@ def get_inputs_from_csv(self, if csv_type == 'raw': input_protos.append( - self.get_text_input( + Inputs.get_text_input( input_id=input_id, raw_text=text, dataset_id=dataset_id, @@ -366,7 +360,7 @@ def get_inputs_from_csv(self, geo_info=geo_info)) elif csv_type == 'url': input_protos.append( - self.get_input_from_url( + Inputs.get_input_from_url( input_id=input_id, image_url=image, text_url=text, @@ -378,7 +372,7 @@ def get_inputs_from_csv(self, geo_info=geo_info)) else: input_protos.append( - self.get_input_from_file( + Inputs.get_input_from_file( input_id=input_id, image_file=image, text_file=text, @@ -391,9 +385,8 @@ def get_inputs_from_csv(self, return input_protos - def get_text_inputs_from_folder(self, - folder_path: str, - dataset_id: str = None, + @staticmethod + def get_text_inputs_from_folder(folder_path: str, dataset_id: str = None, labels: bool = False) -> List[Text]: #text specific """Create input protos for text data type from folder. @@ -405,8 +398,7 @@ def get_text_inputs_from_folder(self, Example: >>> from clarifai.client.input import Inputs - >>> input_obj = Inputs() - >>> input_protos = input_obj.get_text_inputs_from_folder(folder_path='demo_folder') + >>> input_protos = Inputs.get_text_inputs_from_folder(folder_path='demo_folder') """ input_protos = [] labels = [folder_path.split('/')[-1]] if labels else None @@ -416,11 +408,12 @@ def get_text_inputs_from_folder(self, input_id = filename.split('.')[0] text_pb = resources_pb2.Text(raw=open(os.path.join(folder_path, filename), 'rb').read()) input_protos.append( - self._get_proto( + Inputs._get_proto( input_id=input_id, dataset_id=dataset_id, text_pb=text_pb, labels=labels)) return input_protos - def get_annotation_proto(self, input_id: str, label: str, annotations: List) -> Annotation: + @staticmethod + def get_annotation_proto(input_id: str, label: str, annotations: List) -> Annotation: """Create an annotation proto for each bounding box, label input pair. Args: @@ -433,8 +426,7 @@ def get_annotation_proto(self, input_id: str, label: str, annotations: List) -> Example: >>> from clarifai.client.input import Inputs - >>> input_obj = Inputs() - >>> input_obj.get_annotation_proto(input_id='demo', label='demo', annotations=[x_min, y_min, x_max, y_max]) + >>> Inputs.get_annotation_proto(input_id='demo', label='demo', annotations=[x_min, y_min, x_max, y_max]) """ if not isinstance(annotations, list): raise UserError("annotations must be a list of bbox cooridnates") @@ -459,7 +451,8 @@ def get_annotation_proto(self, input_id: str, label: str, annotations: List) -> return input_annot_proto - def get_mask_proto(self, input_id: str, label: str, polygons: List[List[float]]) -> Annotation: + @staticmethod + def get_mask_proto(input_id: str, label: str, polygons: List[List[float]]) -> Annotation: """Create an annotation proto for each polygon box, label input pair. Args: @@ -472,8 +465,7 @@ def get_mask_proto(self, input_id: str, label: str, polygons: List[List[float]]) Example: >>> from clarifai.client.input import Inputs - >>> input_obj = Inputs() - >>> input_obj.get_mask_proto(input_id='demo', label='demo', polygons=[[[x,y],...,[x,y]],...]) + >>> Inputs.get_mask_proto(input_id='demo', label='demo', polygons=[[[x,y],...,[x,y]],...]) """ if not isinstance(polygons, list): raise UserError("polygons must be a list of points") diff --git a/clarifai/client/model.py b/clarifai/client/model.py index 2e47774f..ba313c83 100644 --- a/clarifai/client/model.py +++ b/clarifai/client/model.py @@ -478,13 +478,13 @@ def predict_by_bytes(self, raise UserError('Invalid bytes.') if input_type == "image": - input_proto = Inputs().get_input_from_bytes("", image_bytes=input_bytes) + input_proto = Inputs.get_input_from_bytes("", image_bytes=input_bytes) elif input_type == "text": - input_proto = Inputs().get_input_from_bytes("", text_bytes=input_bytes) + input_proto = Inputs.get_input_from_bytes("", text_bytes=input_bytes) elif input_type == "video": - input_proto = Inputs().get_input_from_bytes("", video_bytes=input_bytes) + input_proto = Inputs.get_input_from_bytes("", video_bytes=input_bytes) elif input_type == "audio": - input_proto = Inputs().get_input_from_bytes("", audio_bytes=input_bytes) + input_proto = Inputs.get_input_from_bytes("", audio_bytes=input_bytes) return self.predict( inputs=[input_proto], inference_params=inference_params, output_config=output_config) @@ -517,13 +517,13 @@ def predict_by_url(self, f"Got input type {input_type} but expected one of image, text, video, audio.") if input_type == "image": - input_proto = Inputs().get_input_from_url("", image_url=url) + input_proto = Inputs.get_input_from_url("", image_url=url) elif input_type == "text": - input_proto = Inputs().get_input_from_url("", text_url=url) + input_proto = Inputs.get_input_from_url("", text_url=url) elif input_type == "video": - input_proto = Inputs().get_input_from_url("", video_url=url) + input_proto = Inputs.get_input_from_url("", video_url=url) elif input_type == "audio": - input_proto = Inputs().get_input_from_url("", audio_url=url) + input_proto = Inputs.get_input_from_url("", audio_url=url) return self.predict( inputs=[input_proto], inference_params=inference_params, output_config=output_config) diff --git a/clarifai/client/workflow.py b/clarifai/client/workflow.py index 60211a86..209011b6 100644 --- a/clarifai/client/workflow.py +++ b/clarifai/client/workflow.py @@ -113,13 +113,13 @@ def predict_by_bytes(self, input_bytes: bytes, input_type: str): raise UserError('Invalid bytes.') if input_type == "image": - input_proto = Inputs().get_input_from_bytes("", image_bytes=input_bytes) + input_proto = Inputs.get_input_from_bytes("", image_bytes=input_bytes) elif input_type == "text": - input_proto = Inputs().get_input_from_bytes("", text_bytes=input_bytes) + input_proto = Inputs.get_input_from_bytes("", text_bytes=input_bytes) elif input_type == "video": - input_proto = Inputs().get_input_from_bytes("", video_bytes=input_bytes) + input_proto = Inputs.get_input_from_bytes("", video_bytes=input_bytes) elif input_type == "audio": - input_proto = Inputs().get_input_from_bytes("", audio_bytes=input_bytes) + input_proto = Inputs.get_input_from_bytes("", audio_bytes=input_bytes) return self.predict(inputs=[input_proto]) @@ -141,13 +141,13 @@ def predict_by_url(self, url: str, input_type: str): raise UserError('Invalid input type it should be image, text, video or audio.') if input_type == "image": - input_proto = Inputs().get_input_from_url("", image_url=url) + input_proto = Inputs.get_input_from_url("", image_url=url) elif input_type == "text": - input_proto = Inputs().get_input_from_url("", text_url=url) + input_proto = Inputs.get_input_from_url("", text_url=url) elif input_type == "video": - input_proto = Inputs().get_input_from_url("", video_url=url) + input_proto = Inputs.get_input_from_url("", video_url=url) elif input_type == "audio": - input_proto = Inputs().get_input_from_url("", audio_url=url) + input_proto = Inputs.get_input_from_url("", audio_url=url) return self.predict(inputs=[input_proto]) diff --git a/clarifai/datasets/upload/image.py b/clarifai/datasets/upload/image.py index 0c67253d..e6f04088 100644 --- a/clarifai/datasets/upload/image.py +++ b/clarifai/datasets/upload/image.py @@ -12,7 +12,6 @@ class VisualClassificationDataset(ClarifaiDataset): def __init__(self, data_generator: Type[ClarifaiDataLoader], dataset_id: str) -> None: - self.input_object = Inputs() super().__init__(data_generator, dataset_id) def _extract_protos(self, batch_input_ids: List[str] @@ -41,7 +40,7 @@ def process_data_item(id): self.all_input_ids[id] = input_id input_protos.append( - self.input_object.get_input_from_file( + Inputs.get_input_from_file( input_id=input_id, image_file=image_path, dataset_id=self.dataset_id, @@ -61,7 +60,6 @@ class VisualDetectionDataset(ClarifaiDataset): """Visual detection dataset proto class.""" def __init__(self, data_generator: Type[ClarifaiDataLoader], dataset_id: str) -> None: - self.input_object = Inputs() super().__init__(data_generator, dataset_id) def _extract_protos(self, batch_input_ids: List[int] @@ -90,7 +88,7 @@ def process_data_item(id): self.all_input_ids[id] = input_id input_protos.append( - self.input_object.get_input_from_file( + Inputs.get_input_from_file( input_id=input_id, image_file=image, dataset_id=self.dataset_id, @@ -100,8 +98,7 @@ def process_data_item(id): # one id could have more than one bbox and label for i in range(len(bboxes)): annotation_protos.append( - self.input_object.get_annotation_proto( - input_id=input_id, label=labels[i], annotations=bboxes[i])) + Inputs.get_annotation_proto(input_id=input_id, label=labels[i], annotations=bboxes[i])) with ThreadPoolExecutor(max_workers=4) as executor: futures = [executor.submit(process_data_item, id) for id in batch_input_ids] @@ -115,7 +112,6 @@ class VisualSegmentationDataset(ClarifaiDataset): """Visual segmentation dataset proto class.""" def __init__(self, data_generator: Type[ClarifaiDataLoader], dataset_id: str) -> None: - self.input_object = Inputs() super().__init__(data_generator, dataset_id) def _extract_protos(self, batch_input_ids: List[str] @@ -144,7 +140,7 @@ def process_data_item(id): self.all_input_ids[id] = input_id input_protos.append( - self.input_object.get_input_from_file( + Inputs.get_input_from_file( input_id=input_id, image_file=image, dataset_id=self.dataset_id, @@ -156,8 +152,7 @@ def process_data_item(id): for i, _polygon in enumerate(_polygons): try: annotation_protos.append( - self.input_object.get_mask_proto( - input_id=input_id, label=labels[i], polygons=_polygon)) + Inputs.get_mask_proto(input_id=input_id, label=labels[i], polygons=_polygon)) except IndexError: continue diff --git a/clarifai/datasets/upload/text.py b/clarifai/datasets/upload/text.py index 40272e56..21ebfe5a 100644 --- a/clarifai/datasets/upload/text.py +++ b/clarifai/datasets/upload/text.py @@ -13,7 +13,6 @@ class TextClassificationDataset(ClarifaiDataset): """Upload text classification datasets to clarifai datasets""" def __init__(self, data_generator: Type[ClarifaiDataLoader], dataset_id: str) -> None: - self.input_object = Inputs() super().__init__(data_generator, dataset_id) def _extract_protos(self, batch_input_ids: List[int] @@ -41,7 +40,7 @@ def process_data_item(id): self.all_input_ids[id] = input_id input_protos.append( - self.input_object.get_text_input( + Inputs.get_text_input( input_id=input_id, raw_text=text, dataset_id=self.dataset_id, diff --git a/tests/test_model_predict.py b/tests/test_model_predict.py index 7da58c3c..5b0d2303 100644 --- a/tests/test_model_predict.py +++ b/tests/test_model_predict.py @@ -20,10 +20,13 @@ RAW_TEXT = "Hi my name is Jim." RAW_TEXT_BYTES = b"Hi my name is Jim." +CLARIFAI_PAT = os.environ["CLARIFAI_PAT"] + @pytest.fixture def model(): - return Model(user_id=MAIN_APP_USER_ID, app_id=MAIN_APP_ID, model_id=GENERAL_MODEL_ID) + return Model( + user_id=MAIN_APP_USER_ID, app_id=MAIN_APP_ID, model_id=GENERAL_MODEL_ID, pat=CLARIFAI_PAT) def validate_concepts_length(response): @@ -110,7 +113,7 @@ def test_predict_video_url_with_custom_sample_ms(): app_id=MAIN_APP_ID, model_id=GENERAL_MODEL_ID, ) - video_proto = Inputs().get_input_from_url("", video_url=BEER_VIDEO_URL) + video_proto = Inputs.get_input_from_url("", video_url=BEER_VIDEO_URL) response = model_with_custom_sample_ms.predict([video_proto], output_config=dict(sample_ms=2000)) # The expected time per frame is the middle between the start and the end of the frame # (in milliseconds). @@ -127,8 +130,7 @@ def test_text_embed_predict_with_raw_text(): clip_embed_model = Model( user_id=MAIN_APP_USER_ID, app_id=MAIN_APP_ID, model_id=CLIP_EMBED_MODEL_ID) - input_text_proto = Inputs().get_input_from_bytes( - "", text_bytes=RAW_TEXT.encode(encoding='UTF-8')) + input_text_proto = Inputs.get_input_from_bytes("", text_bytes=RAW_TEXT.encode(encoding='UTF-8')) response = clip_embed_model.predict([input_text_proto]) assert response.outputs[0].data.embeddings[0].num_dimensions == clip_dim diff --git a/tests/workflow/test_predict.py b/tests/workflow/test_predict.py index 29d4c738..e0ed042a 100644 --- a/tests/workflow/test_predict.py +++ b/tests/workflow/test_predict.py @@ -1,3 +1,4 @@ +import os import pytest from clarifai_grpc.grpc.api import resources_pb2 @@ -12,6 +13,8 @@ MAIN_APP_USER_ID = "clarifai" WORKFLOW_ID = "General" +CLARIFAI_PAT = os.environ["CLARIFAI_PAT"] + @pytest.fixture def workflow(): @@ -19,7 +22,8 @@ def workflow(): user_id=MAIN_APP_USER_ID, app_id=MAIN_APP_ID, workflow_id=WORKFLOW_ID, - output_config=resources_pb2.OutputConfig(max_concepts=3)) + output_config=resources_pb2.OutputConfig(max_concepts=3), + pat=CLARIFAI_PAT) def test_workflow_predict_image_url(workflow): From 1cd291653b7d59f57707aa3ef32c8dc3748e2684 Mon Sep 17 00:00:00 2001 From: Sai Nivedh Date: Thu, 23 Nov 2023 15:13:14 +0530 Subject: [PATCH 5/5] change defaults to None --- clarifai/client/app.py | 10 +++++----- clarifai/client/dataset.py | 10 +++++----- clarifai/client/input.py | 6 +++--- clarifai/client/model.py | 12 ++++++------ clarifai/client/module.py | 12 ++++++------ clarifai/client/runner.py | 6 +++--- clarifai/client/search.py | 2 +- clarifai/client/user.py | 4 ++-- clarifai/client/workflow.py | 12 ++++++------ 9 files changed, 37 insertions(+), 37 deletions(-) diff --git a/clarifai/client/app.py b/clarifai/client/app.py index c7fc740d..62198111 100644 --- a/clarifai/client/app.py +++ b/clarifai/client/app.py @@ -28,10 +28,10 @@ class App(Lister, BaseClient): """App is a class that provides access to Clarifai API endpoints related to App information.""" def __init__(self, - url: str = "", - app_id: str = "", + url: str = None, + app_id: str = None, base_url: str = "https://api.clarifai.com", - pat: str = "", + pat: str = None, **kwargs): """Initializes an App object. @@ -44,9 +44,9 @@ def __init__(self, - name (str): The name of the app. - description (str): The description of the app. """ - if url != "" and app_id != "": + if url and app_id: raise UserError("You can only specify one of url or app_id.") - if url != "": + if url: user_id, app_id, _, _, _ = ClarifaiUrlHelper.split_clarifai_url(url) kwargs = {'user_id': user_id} self.kwargs = {**kwargs, 'id': app_id} diff --git a/clarifai/client/dataset.py b/clarifai/client/dataset.py index aa977c2c..2f2e9428 100644 --- a/clarifai/client/dataset.py +++ b/clarifai/client/dataset.py @@ -34,10 +34,10 @@ class Dataset(Lister, BaseClient): """Dataset is a class that provides access to Clarifai API endpoints related to Dataset information.""" def __init__(self, - url: str = "", - dataset_id: str = "", + url: str = None, + dataset_id: str = None, base_url: str = "https://api.clarifai.com", - pat: str = "", + pat: str = None, **kwargs): """Initializes a Dataset object. @@ -48,9 +48,9 @@ def __init__(self, pat (str): A personal access token for authentication. Can be set as env var CLARIFAI_PAT **kwargs: Additional keyword arguments to be passed to the Dataset. """ - if url != "" and dataset_id != "": + if url and dataset_id: raise UserError("You can only specify one of url or dataset_id.") - if url != "": + if url: user_id, app_id, _, dataset_id, _ = ClarifaiUrlHelper.split_clarifai_url(url) kwargs = {'user_id': user_id, 'app_id': app_id} self.kwargs = {**kwargs, 'id': dataset_id} diff --git a/clarifai/client/input.py b/clarifai/client/input.py index 1e9b7682..96c0feb6 100644 --- a/clarifai/client/input.py +++ b/clarifai/client/input.py @@ -25,11 +25,11 @@ class Inputs(Lister, BaseClient): """Inputs is a class that provides access to Clarifai API endpoints related to Input information.""" def __init__(self, - user_id: str = "", - app_id: str = "", + user_id: str = None, + app_id: str = None, logger_level: str = "INFO", base_url: str = "https://api.clarifai.com", - pat: str = "", + pat: str = None, **kwargs): """Initializes an Input object. diff --git a/clarifai/client/model.py b/clarifai/client/model.py index ba313c83..ea2a23e2 100644 --- a/clarifai/client/model.py +++ b/clarifai/client/model.py @@ -27,11 +27,11 @@ class Model(Lister, BaseClient): """Model is a class that provides access to Clarifai API endpoints related to Model information.""" def __init__(self, - url: str = "", - model_id: str = "", + url: str = None, + model_id: str = None, model_version: Dict = {'id': ""}, base_url: str = "https://api.clarifai.com", - pat: str = "", + pat: str = None, **kwargs): """Initializes a Model object. @@ -43,11 +43,11 @@ def __init__(self, pat (str): A personal access token for authentication. Can be set as env var CLARIFAI_PAT **kwargs: Additional keyword arguments to be passed to the Model. """ - if url != "" and model_id != "": + if url and model_id: raise UserError("You can only specify one of url or model_id.") - if url == "" and model_id == "": + if not url and not model_id: raise UserError("You must specify one of url or model_id.") - if url != "": + if url: user_id, app_id, _, model_id, model_version_id = ClarifaiUrlHelper.split_clarifai_url(url) model_version = {'id': model_version_id} kwargs = {'user_id': user_id, 'app_id': app_id} diff --git a/clarifai/client/module.py b/clarifai/client/module.py index 937dbcd4..05eb18fc 100644 --- a/clarifai/client/module.py +++ b/clarifai/client/module.py @@ -13,11 +13,11 @@ class Module(Lister, BaseClient): """Module is a class that provides access to Clarifai API endpoints related to Module information.""" def __init__(self, - url: str = "", - module_id: str = "", + url: str = None, + module_id: str = None, module_version: Dict = {'id': ""}, base_url: str = "https://api.clarifai.com", - pat: str = "", + pat: str = None, **kwargs): """Initializes a Module object. @@ -29,11 +29,11 @@ def __init__(self, pat (str): A personal access token for authentication. Can be set as env var CLARIFAI_PAT **kwargs: Additional keyword arguments to be passed to the Module. """ - if url != "" and module_id != "": + if url and module_id: raise UserError("You can only specify one of url or module_id.") - if url == "" and module_id == "": + if not url and not module_id: raise UserError("You must specify one of url or module_id.") - if url != "": + if url: user_id, app_id, module_id, module_version_id = ClarifaiUrlHelper.split_module_ui_url(url) module_version = {'id': module_version_id} kwargs = {'user_id': user_id, 'app_id': app_id} diff --git a/clarifai/client/runner.py b/clarifai/client/runner.py index 54178a1a..4e037e7a 100644 --- a/clarifai/client/runner.py +++ b/clarifai/client/runner.py @@ -32,10 +32,10 @@ class Runner(BaseClient): def __init__(self, runner_id: str, - user_id: str = "", + user_id: str = None, check_runner_exists: bool = True, base_url: str = "https://api.clarifai.com", - pat: str = "", + pat: str = None, **kwargs) -> None: """ Args: @@ -46,7 +46,7 @@ def __init__(self, """ user_id = user_id or os.environ.get("CLARIFAI_USER_ID", "") - if user_id == "": + if not user_id: raise UserError( "Set CLARIFAI_USER_ID as environment variables or pass user_id as input arguments") diff --git a/clarifai/client/search.py b/clarifai/client/search.py index 5798afed..dcbcbe18 100644 --- a/clarifai/client/search.py +++ b/clarifai/client/search.py @@ -23,7 +23,7 @@ def __init__(self, top_k: int = DEFAULT_TOP_K, metric: str = DEFAULT_SEARCH_METRIC, base_url: str = "https://api.clarifai.com", - pat: str = ""): + pat: str = None): """Initialize the Search object. Args: diff --git a/clarifai/client/user.py b/clarifai/client/user.py index 68041aaf..7980eda8 100644 --- a/clarifai/client/user.py +++ b/clarifai/client/user.py @@ -16,9 +16,9 @@ class User(Lister, BaseClient): """User is a class that provides access to Clarifai API endpoints related to user information.""" def __init__(self, - user_id: str = "", + user_id: str = None, base_url: str = "https://api.clarifai.com", - pat: str = "", + pat: str = None, **kwargs): """Initializes an User object. diff --git a/clarifai/client/workflow.py b/clarifai/client/workflow.py index 209011b6..aa565735 100644 --- a/clarifai/client/workflow.py +++ b/clarifai/client/workflow.py @@ -18,12 +18,12 @@ class Workflow(Lister, BaseClient): """Workflow is a class that provides access to Clarifai API endpoints related to Workflow information.""" def __init__(self, - url: str = "", - workflow_id: str = "", + url: str = None, + workflow_id: str = None, workflow_version: Dict = {'id': ""}, output_config: Dict = {'min_value': 0}, base_url: str = "https://api.clarifai.com", - pat: str = "", + pat: str = None, **kwargs): """Initializes a Workflow object. @@ -39,11 +39,11 @@ def __init__(self, base_url (str): Base API url. Default "https://api.clarifai.com" **kwargs: Additional keyword arguments to be passed to the Workflow. """ - if url != "" and workflow_id != "": + if url and workflow_id: raise UserError("You can only specify one of url or workflow_id.") - if url == "" and workflow_id == "": + if not url and not workflow_id: raise UserError("You must specify one of url or workflow_id.") - if url != "": + if url: user_id, app_id, _, workflow_id, workflow_version_id = ClarifaiUrlHelper.split_clarifai_url( url) workflow_version = {'id': workflow_version_id}