diff --git a/CHANGELOG.md b/CHANGELOG.md index 743c6513d..6dfba5878 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,9 @@ - ### Changed -- +- [AWS Lambda] Eliminate the need for access and secret keys in the configuration +- [AWS Batch] Eliminate the need for access and secret keys in the configuration +- [AWS S3] Eliminate the need for access and secret keys in the configuration ### Fixed - [AWS Lambda] Fixed runtime deletion with "lithops runtime delete" diff --git a/docs/index.rst b/docs/index.rst index bbb113285..b800069a5 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,12 +1,6 @@ What is Lithops? **************** -.. image:: source/images/lithops_logo_readme.png - :alt: Lithops - :align: center - -| - **Lithops is a Python multi-cloud serverless computing framework. It allows to run unmodified local python code at massive scale in the main serverless computing platforms.** Lithops delivers the user’s code into the cloud without requiring knowledge of how it is deployed and run. @@ -28,6 +22,18 @@ analytics, to name a few. Lithops abstracts away the underlying cloud-specific APIs for accessing storage and provides an intuitive and easy to use interface to process high volumes of data. +Use any Cloud +************* +**Lithops provides an extensible backend architecture that is designed to work with different compute and storage services available on Cloud providers and on-premise backends.** + +In this sense, you can code your application in Python and run it unmodified wherever your data is located at: IBM Cloud, AWS, Azure, Google Cloud and Alibaba Aliyun... + +.. image:: source/images/multicloud.jpg + :alt: Available backends + :align: center + +| + Quick Start *********** @@ -50,17 +56,6 @@ You're ready to execute a simple example! fut = fexec.call_async(hello, 'World') print(fut.result()) -Use any Cloud -************* -**Lithops provides an extensible backend architecture that is designed to work with different compute and storage services available on Cloud providers and on-premise backends.** - -In this sense, you can code your application in Python and run it unmodified wherever your data is located at: IBM Cloud, AWS, Azure, Google Cloud and Alibaba Aliyun... - -.. image:: source/images/multicloud.jpg - :alt: Available backends - :align: center - -| Additional resources ******************** diff --git a/docs/source/compute_backends.rst b/docs/source/compute_backends.rst index e05991fe1..f2eeb81da 100644 --- a/docs/source/compute_backends.rst +++ b/docs/source/compute_backends.rst @@ -19,7 +19,6 @@ Compute Backends compute_config/oracle_functions.md compute_config/aliyun_functions.md compute_config/openwhisk.md - compute_config/ibm_cf.md **Serverless (CaaS) Backends:** diff --git a/docs/source/compute_config/aws_batch.md b/docs/source/compute_config/aws_batch.md index f6b90945b..d0a904ab2 100644 --- a/docs/source/compute_config/aws_batch.md +++ b/docs/source/compute_config/aws_batch.md @@ -51,8 +51,8 @@ aws_batch: |Group|Key|Default|Mandatory|Additional info| |---|---|---|---|---| |aws | region | |yes | AWS region name. For example `us-east-1` | -|aws | access_key_id | |yes | Account access key to AWS services. To find them, navigate to *My Security Credentials* and click *Create Access Key* if you don't already have one. | -|aws | secret_access_key | |yes | Account secret access key to AWS services. To find them, navigate to *My Security Credentials* and click *Create Access Key* if you don't already have one. | +|aws | access_key_id | |no | Account access key to AWS services. To find them, navigate to *My Security Credentials* and click *Create Access Key* if you don't already have one. | +|aws | secret_access_key | |no | Account secret access key to AWS services. To find them, navigate to *My Security Credentials* and click *Create Access Key* if you don't already have one. | |aws | session_token | |no | Session token for temporary AWS credentials | |aws | account_id | |no | *This field will be used if present to retrieve the account ID instead of using AWS STS. The account ID is used to format full image names for container runtimes. | diff --git a/docs/source/compute_config/aws_lambda.md b/docs/source/compute_config/aws_lambda.md index f1b5cc0ce..f752fbd84 100644 --- a/docs/source/compute_config/aws_lambda.md +++ b/docs/source/compute_config/aws_lambda.md @@ -59,8 +59,8 @@ aws_lambda: |Group|Key|Default|Mandatory|Additional info| |---|---|---|---|---| |aws | region | |yes | AWS Region. For example `us-east-1` | -|aws | access_key_id | |yes | Account access key to AWS services. To find them, navigate to *My Security Credentials* and click *Create Access Key* if you don't already have one. | -|aws | secret_access_key | |yes | Account secret access key to AWS services. To find them, navigate to *My Security Credentials* and click *Create Access Key* if you don't already have one. | +|aws | access_key_id | |no | Account access key to AWS services. To find them, navigate to *My Security Credentials* and click *Create Access Key* if you don't already have one. | +|aws | secret_access_key | |no | Account secret access key to AWS services. To find them, navigate to *My Security Credentials* and click *Create Access Key* if you don't already have one. | |aws | session_token | |no | Session token for temporary AWS credentials | |aws | account_id | |no | *This field will be used if present to retrieve the account ID instead of using AWS STS. The account ID is used to format full image names for container runtimes. | diff --git a/docs/source/compute_config/azure_vms.md b/docs/source/compute_config/azure_vms.md index c737b4861..e49253f8c 100644 --- a/docs/source/compute_config/azure_vms.md +++ b/docs/source/compute_config/azure_vms.md @@ -1,4 +1,4 @@ -# Azure Virtual Machines (Beta) +# Azure Virtual Machines The Azure Virtual Machines client of Lithops can provide a truely serverless user experience on top of Azure VMs where Lithops creates new Virtual Machines (VMs) dynamically in runtime and scale Lithops jobs against them. Alternatively Lithops can start and stop an existing VM instances. diff --git a/docs/source/compute_config/oracle_functions.md b/docs/source/compute_config/oracle_functions.md index 6df94137b..336b1dd46 100644 --- a/docs/source/compute_config/oracle_functions.md +++ b/docs/source/compute_config/oracle_functions.md @@ -1,4 +1,4 @@ -# Oracle Functions (beta) +# Oracle Functions Lithops with *Oracle Functions* as serverless compute backend. diff --git a/docs/source/storage_config/aws_s3.md b/docs/source/storage_config/aws_s3.md index 21f89f609..393de1a18 100644 --- a/docs/source/storage_config/aws_s3.md +++ b/docs/source/storage_config/aws_s3.md @@ -37,8 +37,8 @@ Lithops with AWS S3 as storage backend. |Group|Key|Default|Mandatory|Additional info| |---|---|---|---|---| |aws | region | |yes | AWS Region. For example `us-east-1` | -|aws | access_key_id | |yes | Account access key to AWS services. To find them, navigate to *My Security Credentials* and click *Create Access Key* if you don't already have one. | -|aws | secret_access_key | |yes | Account secret access key to AWS services. To find them, navigate to *My Security Credentials* and click *Create Access Key* if you don't already have one. | +|aws | access_key_id | |no | Account access key to AWS services. To find them, navigate to *My Security Credentials* and click *Create Access Key* if you don't already have one. | +|aws | secret_access_key | |no | Account secret access key to AWS services. To find them, navigate to *My Security Credentials* and click *Create Access Key* if you don't already have one. | |aws | session_token | |no | Session token for temporary AWS credentials | ### Summary of configuration keys for AWS S3: diff --git a/lithops/serverless/backends/aws_batch/aws_batch.py b/lithops/serverless/backends/aws_batch/aws_batch.py index d2b192a87..1e1789733 100644 --- a/lithops/serverless/backends/aws_batch/aws_batch.py +++ b/lithops/serverless/backends/aws_batch/aws_batch.py @@ -45,34 +45,44 @@ def __init__(self, aws_batch_config, internal_storage): self.name = 'aws_batch' self.type = utils.BackendType.BATCH.value self.aws_batch_config = aws_batch_config - - self.user_key = aws_batch_config['access_key_id'][-4:] - self.package = f'lithops_v{__version__.replace(".", "-")}_{self.user_key}' - self.region_name = aws_batch_config['region'] + self.region = aws_batch_config['region'] + self.namespace = aws_batch_config.get('namespace') self._env_type = self.aws_batch_config['env_type'] self._queue_name = f'{self.package}_{self._env_type.replace("_", "-")}_queue' self._compute_env_name = f'{self.package}_{self._env_type.replace("_", "-")}_env' logger.debug('Creating Boto3 AWS Session and Batch Client') - self.aws_session = boto3.Session(aws_access_key_id=aws_batch_config['access_key_id'], - aws_secret_access_key=aws_batch_config['secret_access_key'], - aws_session_token=aws_batch_config.get('session_token'), - region_name=self.region_name) - self.batch_client = self.aws_session.client('batch', region_name=self.region_name) + self.aws_session = boto3.Session( + aws_access_key_id=aws_batch_config.get('access_key_id'), + aws_secret_access_key=aws_batch_config.get('secret_access_key'), + aws_session_token=aws_batch_config.get('session_token'), + region_name=self.region + ) + self.batch_client = self.aws_session.client('batch', region_name=self.region) self.internal_storage = internal_storage if 'account_id' in self.aws_batch_config: self.account_id = self.aws_batch_config['account_id'] else: - sts_client = self.aws_session.client('sts', region_name=self.region_name) + sts_client = self.aws_session.client('sts', region_name=self.region) self.account_id = sts_client.get_caller_identity()["Account"] - self.ecr_client = self.aws_session.client('ecr', region_name=self.region_name) + sts_client = self.aws_session.client('sts', region_name=self.region) + caller_id = sts_client.get_caller_identity() + + if ":" in caller_id["UserId"]: # SSO user + self.user_key = caller_id["UserId"].split(":")[1] + else: # IAM user + self.user_key = caller_id["UserId"][-4:].lower() + + self.ecr_client = self.aws_session.client('ecr', region_name=self.region) + package = f'lithops_v{__version__.replace(".", "")}_{self.user_key}' + self.package = f"{package}_{self.namespace}" if self.namespace else package msg = COMPUTE_CLI_MSG.format('AWS Batch') - logger.info("{} - Region: {}".format(msg, self.region_name)) + logger.info(f"{msg} - Region: {self.region}") def _get_default_runtime_image_name(self): python_version = utils.CURRENT_PY_VERSION.replace('.', '') @@ -81,7 +91,7 @@ def _get_default_runtime_image_name(self): def _get_full_image_name(self, runtime_name): full_image_name = runtime_name if ':' in runtime_name else f'{runtime_name}:latest' - registry = f'{self.account_id}.dkr.ecr.{self.region_name}.amazonaws.com' + registry = f'{self.account_id}.dkr.ecr.{self.region}.amazonaws.com' full_image_name = '/'.join([registry, self.package.replace('-', '.'), full_image_name]).lower() repo_name = full_image_name.split('/', 1)[1:].pop().split(':')[0] return full_image_name, registry, repo_name @@ -585,7 +595,7 @@ def invoke(self, runtime_name, runtime_memory, payload): def get_runtime_key(self, runtime_name, runtime_memory, version=__version__): jobdef_name = self._format_jobdef_name(runtime_name, runtime_memory, version) - runtime_key = os.path.join(self.name, version, self.region_name, jobdef_name) + runtime_key = os.path.join(self.name, version, self.region, jobdef_name) return runtime_key def get_runtime_info(self): diff --git a/lithops/serverless/backends/aws_batch/config.py b/lithops/serverless/backends/aws_batch/config.py index de2892810..c63e5291b 100644 --- a/lithops/serverless/backends/aws_batch/config.py +++ b/lithops/serverless/backends/aws_batch/config.py @@ -76,9 +76,6 @@ def load_config(config_data): if 'aws' not in config_data: raise Exception("'aws' section is mandatory in the configuration") - if not {'access_key_id', 'secret_access_key'}.issubset(set(config_data['aws'])): - raise Exception("'access_key_id' and 'secret_access_key' are mandatory under the 'aws' section of the configuration") - if not config_data['aws_batch']: raise Exception("'aws_batch' section is mandatory in the configuration") diff --git a/lithops/serverless/backends/aws_lambda/aws_lambda.py b/lithops/serverless/backends/aws_lambda/aws_lambda.py index d89c789a3..647858d1a 100644 --- a/lithops/serverless/backends/aws_lambda/aws_lambda.py +++ b/lithops/serverless/backends/aws_lambda/aws_lambda.py @@ -57,21 +57,21 @@ def __init__(self, lambda_config, internal_storage): self.lambda_config = lambda_config self.internal_storage = internal_storage self.user_agent = lambda_config['user_agent'] - self.region_name = lambda_config['region'] + self.region = lambda_config['region'] self.role_arn = lambda_config['execution_role'] self.namespace = lambda_config.get('namespace') logger.debug('Creating Boto3 AWS Session and Lambda Client') self.aws_session = boto3.Session( - aws_access_key_id=lambda_config['access_key_id'], - aws_secret_access_key=lambda_config['secret_access_key'], + aws_access_key_id=lambda_config.get('access_key_id'), + aws_secret_access_key=lambda_config.get('secret_access_key'), aws_session_token=lambda_config.get('session_token'), - region_name=self.region_name + region_name=self.region ) self.lambda_client = self.aws_session.client( - 'lambda', region_name=self.region_name, + 'lambda', region_name=self.region, config=botocore.client.Config( user_agent_extra=self.user_agent ) @@ -79,15 +79,15 @@ def __init__(self, lambda_config, internal_storage): self.credentials = self.aws_session.get_credentials() self.session = URLLib3Session() - self.host = f'lambda.{self.region_name}.amazonaws.com' + self.host = f'lambda.{self.region}.amazonaws.com' if 'account_id' in self.lambda_config: self.account_id = self.lambda_config['account_id'] else: - sts_client = self.aws_session.client('sts', region_name=self.region_name) + sts_client = self.aws_session.client('sts', region_name=self.region) self.account_id = sts_client.get_caller_identity()["Account"] - sts_client = self.aws_session.client('sts', region_name=self.region_name) + sts_client = self.aws_session.client('sts', region_name=self.region) caller_id = sts_client.get_caller_identity() if ":" in caller_id["UserId"]: # SSO user @@ -95,15 +95,15 @@ def __init__(self, lambda_config, internal_storage): else: # IAM user self.user_key = caller_id["UserId"][-4:].lower() - self.ecr_client = self.aws_session.client('ecr', region_name=self.region_name) + self.ecr_client = self.aws_session.client('ecr', region_name=self.region) package = f'lithops_v{__version__.replace(".", "")}_{self.user_key}' self.package = f"{package}_{self.namespace}" if self.namespace else package msg = COMPUTE_CLI_MSG.format('AWS Lambda') if self.namespace: - logger.info(f"{msg} - Region: {self.region_name} - Namespace: {self.namespace}") + logger.info(f"{msg} - Region: {self.region} - Namespace: {self.namespace}") else: - logger.info(f"{msg} - Region: {self.region_name}") + logger.info(f"{msg} - Region: {self.region}") def _format_function_name(self, runtime_name, runtime_memory, version=__version__): name = f'{runtime_name}-{runtime_memory}-{version}' @@ -357,7 +357,7 @@ def build_runtime(self, runtime_name, runtime_file, extra_args=[]): finally: os.remove(LITHOPS_FUNCTION_ZIP) - registry = f'{self.account_id}.dkr.ecr.{self.region_name}.amazonaws.com' + registry = f'{self.account_id}.dkr.ecr.{self.region}.amazonaws.com' res = self.ecr_client.get_authorization_token() if res['ResponseMetadata']['HTTPStatusCode'] != 200: @@ -474,7 +474,7 @@ def _deploy_container_runtime(self, runtime_name, memory, timeout): except botocore.exceptions.ClientError: raise Exception(f'Runtime "{runtime_name}" is not deployed to ECR') - registry = f'{self.account_id}.dkr.ecr.{self.region_name}.amazonaws.com' + registry = f'{self.account_id}.dkr.ecr.{self.region}.amazonaws.com' image_uri = f'{registry}/{repo_name}@{image_digest}' env_vars = {t['name']: t['value'] for t in self.lambda_config['env_vars']} @@ -628,7 +628,7 @@ def invoke(self, runtime_name, runtime_memory, payload): headers = {'Host': self.host, 'X-Amz-Invocation-Type': 'Event', 'User-Agent': self.user_agent} url = f'https://{self.host}/2015-03-31/functions/{function_name}/invocations' request = AWSRequest(method="POST", url=url, data=json.dumps(payload, default=str), headers=headers) - SigV4Auth(self.credentials, "lambda", self.region_name).add_auth(request) + SigV4Auth(self.credentials, "lambda", self.region).add_auth(request) invoked = False while not invoked: @@ -674,7 +674,7 @@ def get_runtime_key(self, runtime_name, runtime_memory, version=__version__): in order to know which runtimes are installed and which not. """ action_name = self._format_function_name(runtime_name, runtime_memory, version) - runtime_key = os.path.join(self.name, version, self.region_name, action_name) + runtime_key = os.path.join(self.name, version, self.region, action_name) return runtime_key diff --git a/lithops/serverless/backends/aws_lambda/config.py b/lithops/serverless/backends/aws_lambda/config.py index 68027e6f6..78ad143b4 100644 --- a/lithops/serverless/backends/aws_lambda/config.py +++ b/lithops/serverless/backends/aws_lambda/config.py @@ -70,9 +70,6 @@ def load_config(config_data): if 'aws' not in config_data: raise Exception("'aws' section is mandatory in the configuration") - if not {'access_key_id', 'secret_access_key'}.issubset(set(config_data['aws'])): - raise Exception("'access_key_id' and 'secret_access_key' are mandatory under the 'aws' section of the configuration") - if not config_data['aws_lambda']: raise Exception("'aws_lambda' section is mandatory in the configuration") diff --git a/lithops/storage/backends/aliyun_oss/aliyun_oss.py b/lithops/storage/backends/aliyun_oss/aliyun_oss.py index 4d00c370c..584e413cc 100644 --- a/lithops/storage/backends/aliyun_oss/aliyun_oss.py +++ b/lithops/storage/backends/aliyun_oss/aliyun_oss.py @@ -59,6 +59,15 @@ def _connect_bucket(self, bucket_name): def get_client(self): return self + def generate_bucket_name(self): + """ + Generates a unique bucket name + """ + key = self.config['access_key_id'] + self.config['storage_bucket'] = f'lithops-{self.region}-{key[:6].lower()}' + + return self.config['storage_bucket'] + def create_bucket(self, bucket_name): """ Create a bucket if it doesn't exist diff --git a/lithops/storage/backends/aliyun_oss/config.py b/lithops/storage/backends/aliyun_oss/config.py index 55b407499..73720aa2f 100644 --- a/lithops/storage/backends/aliyun_oss/config.py +++ b/lithops/storage/backends/aliyun_oss/config.py @@ -15,7 +15,6 @@ # import copy -import hashlib CONNECTION_POOL_SIZE = 300 @@ -48,9 +47,3 @@ def load_config(config_data=None): region = config_data['aliyun_oss']['region'] config_data['aliyun_oss']['public_endpoint'] = PUBLIC_ENDPOINT.format(region) config_data['aliyun_oss']['internal_endpoint'] = INTERNAL_ENDPOINT.format(region) - - if 'storage_bucket' not in config_data['aliyun_oss']: - ossc = config_data['aliyun_oss'] - key = ossc['access_key_id'] - endpoint = hashlib.sha1(ossc['public_endpoint'].encode()).hexdigest()[:6] - config_data['aliyun_oss']['storage_bucket'] = f'lithops-{endpoint}-{key[:6].lower()}' diff --git a/lithops/storage/backends/aws_s3/aws_s3.py b/lithops/storage/backends/aws_s3/aws_s3.py index 0955b2dc3..d034652b4 100644 --- a/lithops/storage/backends/aws_s3/aws_s3.py +++ b/lithops/storage/backends/aws_s3/aws_s3.py @@ -17,7 +17,6 @@ import os import logging import boto3 -from botocore import UNSIGNED from botocore.config import Config import botocore @@ -34,46 +33,57 @@ class S3Backend: def __init__(self, s3_config): - logger.debug("Creating S3 client") + logger.debug("Creating Boto3 AWS Session and S3 Client") self.config = s3_config self.user_agent = s3_config['user_agent'] - self.region_name = s3_config.get('region') - self.access_key_id = s3_config.get('access_key_id') - self.secret_access_key = s3_config.get('secret_access_key') - self.session_token = s3_config.get('session_token') - - if self.access_key_id and self.secret_access_key: - client_config = Config( - max_pool_connections=128, - user_agent_extra=self.user_agent, - connect_timeout=CONN_READ_TIMEOUT, - read_timeout=CONN_READ_TIMEOUT, - retries={'max_attempts': OBJ_REQ_RETRIES} - ) - self.s3_client = boto3.client( - 's3', aws_access_key_id=self.access_key_id, - aws_secret_access_key=self.secret_access_key, - aws_session_token=self.session_token, - config=client_config, - region_name=self.region_name - ) - else: - client_config = Config( - signature_version=UNSIGNED, - user_agent_extra=self.user_agent - ) - self.s3_client = boto3.client('s3', config=client_config) + self.region = s3_config.get('region') + + self.aws_session = boto3.Session( + aws_access_key_id=s3_config.get('access_key_id'), + aws_secret_access_key=s3_config.get('secret_access_key'), + aws_session_token=s3_config.get('session_token'), + region_name=self.region + ) + + s3_client_config = Config( + max_pool_connections=128, + user_agent_extra=self.user_agent, + connect_timeout=CONN_READ_TIMEOUT, + read_timeout=CONN_READ_TIMEOUT, + retries={'max_attempts': OBJ_REQ_RETRIES} + ) + + self.s3_client = self.aws_session.client( + 's3', config=s3_client_config, + region_name=self.region + ) msg = STORAGE_CLI_MSG.format('S3') - logger.info(f"{msg} - Region: {self.region_name}") + logger.info(f"{msg} - Region: {self.region}") def get_client(self): - ''' + """ Get boto3 client. :return: boto3 client - ''' + """ return self.s3_client + def generate_bucket_name(self): + """ + Generates a unique bucket name + """ + sts_client = self.aws_session.client('sts', region_name=self.region) + caller_id = sts_client.get_caller_identity() + + if ":" in caller_id["UserId"]: # SSO user + user_key = caller_id["UserId"].split(":")[1] + else: # IAM user + user_key = caller_id["UserId"][-4:].lower() + + self.config['storage_bucket'] = f'lithops-{self.region}-{user_key}' + + return self.config['storage_bucket'] + def create_bucket(self, bucket_name): """ Create a bucket if it doesn't exist @@ -84,19 +94,19 @@ def create_bucket(self, bucket_name): if e.response['ResponseMetadata']['HTTPStatusCode'] == 404: logger.debug(f"Could not find the bucket {bucket_name} in the AWS S3 storage backend") logger.debug(f"Creating new bucket {bucket_name} in the AWS S3 storage backend") - bucket_config = {'LocationConstraint': self.region_name} + bucket_config = {'LocationConstraint': self.region} self.s3_client.create_bucket(Bucket=bucket_name, CreateBucketConfiguration=bucket_config) else: raise e def put_object(self, bucket_name, key, data): - ''' + """ Put an object in COS. Override the object if the key already exists. :param key: key of the object. :param data: data of the object :type data: str/bytes :return: None - ''' + """ try: res = self.s3_client.put_object(Bucket=bucket_name, Key=key, Body=data) status = 'OK' if res['ResponseMetadata']['HTTPStatusCode'] == 200 else 'Error' @@ -111,12 +121,12 @@ def put_object(self, bucket_name, key, data): raise e def get_object(self, bucket_name, key, stream=False, extra_get_args={}): - ''' + """ Get object from COS with a key. Throws StorageNoSuchKeyError if the given key does not exist. :param key: key of the object :return: Data of the object :rtype: str/bytes - ''' + """ try: r = self.s3_client.get_object(Bucket=bucket_name, Key=key, **extra_get_args) if stream: @@ -173,12 +183,12 @@ def download_file(self, bucket, key, file_name=None, extra_args={}, config=None) return True def head_object(self, bucket_name, key): - ''' + """ Head object from COS with a key. Throws StorageNoSuchKeyError if the given key does not exist. :param key: key of the object :return: Data of the object :rtype: str/bytes - ''' + """ try: metadata = self.s3_client.head_object(Bucket=bucket_name, Key=key) return metadata['ResponseMetadata']['HTTPHeaders'] @@ -189,19 +199,19 @@ def head_object(self, bucket_name, key): raise e def delete_object(self, bucket_name, key): - ''' + """ Delete an object from storage. :param bucket: bucket name :param key: data key - ''' + """ return self.s3_client.delete_object(Bucket=bucket_name, Key=key) def delete_objects(self, bucket_name, key_list): - ''' + """ Delete a list of objects from storage. :param bucket: bucket name :param key_list: list of keys - ''' + """ result = [] max_keys_num = 1000 for i in range(0, len(key_list), max_keys_num): @@ -211,12 +221,12 @@ def delete_objects(self, bucket_name, key_list): return result def head_bucket(self, bucket_name): - ''' + """ Head bucket from COS with a name. Throws StorageNoSuchKeyError if the given bucket does not exist. :param bucket_name: name of the bucket :return: Metadata of the bucket :rtype: str/bytes - ''' + """ try: return self.s3_client.head_bucket(Bucket=bucket_name) except botocore.exceptions.ClientError as e: @@ -226,13 +236,13 @@ def head_bucket(self, bucket_name): raise e def list_objects(self, bucket_name, prefix=None, match_pattern=None): - ''' + """ Return a list of objects for the given bucket and prefix. :param bucket_name: Name of the bucket. :param prefix: Prefix to filter object names. :return: List of objects in bucket that match the given prefix. :rtype: list of str - ''' + """ try: prefix = '' if prefix is None else prefix paginator = self.s3_client.get_paginator('list_objects_v2') @@ -253,13 +263,13 @@ def list_objects(self, bucket_name, prefix=None, match_pattern=None): raise e def list_keys(self, bucket_name, prefix=None): - ''' + """ Return a list of keys for the given prefix. :param bucket_name: Name of the bucket. :param prefix: Prefix to filter object names. :return: List of keys in bucket that match the given prefix. :rtype: list of str - ''' + """ try: prefix = '' if prefix is None else prefix paginator = self.s3_client.get_paginator('list_objects_v2') diff --git a/lithops/storage/backends/aws_s3/config.py b/lithops/storage/backends/aws_s3/config.py index 60585a827..f72b9750a 100644 --- a/lithops/storage/backends/aws_s3/config.py +++ b/lithops/storage/backends/aws_s3/config.py @@ -22,9 +22,6 @@ def load_config(config_data): if 'aws' in config_data: - if not {'access_key_id', 'secret_access_key'}.issubset(set(config_data['aws'])): - raise Exception("'access_key_id' and 'secret_access_key' are mandatory under the 'aws' section of the configuration") - if 'aws_s3' not in config_data: config_data['aws_s3'] = {} @@ -37,8 +34,3 @@ def load_config(config_data): if 'region' not in config_data['aws_s3']: raise Exception("'region' is mandatory under 'aws_s3' or 'aws' section of the configuration") - - if 'storage_bucket' not in config_data['aws_s3']: - key = config_data['aws_s3']['access_key_id'] - region = config_data['aws_s3']['region'] - config_data['aws_s3']['storage_bucket'] = f'lithops-{region}-{key[:6].lower()}' diff --git a/lithops/storage/backends/azure_storage/azure_storage.py b/lithops/storage/backends/azure_storage/azure_storage.py index b2407eccc..123be7db7 100644 --- a/lithops/storage/backends/azure_storage/azure_storage.py +++ b/lithops/storage/backends/azure_storage/azure_storage.py @@ -15,6 +15,7 @@ # import os +import hashlib import shutil import logging from io import BytesIO @@ -30,10 +31,14 @@ class AzureBlobStorageBackend: def __init__(self, azure_blob_config): logger.debug("Creating Azure Blob Storage client") + self.config = azure_blob_config self.storage_account_name = azure_blob_config['storage_account_name'] self.blob_service_url = 'https://{}.blob.core.windows.net'.format(self.storage_account_name) - self.blob_client = BlobServiceClient(account_url=self.blob_service_url, - credential=azure_blob_config['storage_account_key']) + + self.blob_client = BlobServiceClient( + account_url=self.blob_service_url, + credential=azure_blob_config['storage_account_key'] + ) msg = STORAGE_CLI_MSG.format('Azure Blob') logger.info("{}".format(msg)) @@ -46,6 +51,16 @@ def get_client(self): """ return self.blob_client + def generate_bucket_name(self): + """ + Generates a unique bucket name + """ + key = self.config['storage_account_key'] + account = hashlib.sha1(self.config['storage_account_name'].encode()).hexdigest()[:6] + self.config['storage_bucket'] = f'lithops-{account}-{key[:6].lower()}' + + return self.config['storage_bucket'] + def create_bucket(self, bucket_name): """ Create a bucket if it doesn't exist diff --git a/lithops/storage/backends/azure_storage/config.py b/lithops/storage/backends/azure_storage/config.py index 62cff7761..abb1ca949 100644 --- a/lithops/storage/backends/azure_storage/config.py +++ b/lithops/storage/backends/azure_storage/config.py @@ -14,8 +14,6 @@ # limitations under the License. # -import hashlib - REQ_PARAMS = ('storage_account_name', 'storage_account_key') @@ -29,9 +27,3 @@ def load_config(config_data=None): if param not in config_data['azure_storage']: msg = f"'{param}' is mandatory under 'azure_storage' section of the configuration" raise Exception(msg) - - if 'storage_bucket' not in config_data['azure_storage']: - azsc = config_data['azure_storage'] - key = azsc['storage_account_key'] - account = hashlib.sha1(azsc['storage_account_name'].encode()).hexdigest()[:6] - config_data['azure_storage']['storage_bucket'] = f'lithops-{account}-{key[:6].lower()}' diff --git a/lithops/storage/backends/ceph/ceph.py b/lithops/storage/backends/ceph/ceph.py index fd6fe057e..8f5cbba13 100644 --- a/lithops/storage/backends/ceph/ceph.py +++ b/lithops/storage/backends/ceph/ceph.py @@ -17,6 +17,7 @@ import os import logging import boto3 +import hashlib import botocore from lithops.storage.utils import StorageNoSuchKeyError from lithops.utils import sizeof_fmt @@ -37,9 +38,9 @@ def __init__(self, ceph_config): logger.debug("Creating Ceph client") self.config = ceph_config user_agent = ceph_config['user_agent'] - service_endpoint = ceph_config['endpoint'] + self.service_endpoint = ceph_config['endpoint'] - logger.debug(f"Setting Ceph endpoint to {service_endpoint}") + logger.debug(f"Setting Ceph endpoint to {self.service_endpoint}") client_config = botocore.client.Config( max_pool_connections=128, @@ -54,11 +55,11 @@ def __init__(self, ceph_config): aws_secret_access_key=ceph_config['secret_access_key'], aws_session_token=ceph_config.get('session_token'), config=client_config, - endpoint_url=service_endpoint + endpoint_url=self.service_endpoint ) msg = STORAGE_CLI_MSG.format('Ceph') - logger.info(f"{msg} - Endpoint: {service_endpoint}") + logger.info(f"{msg} - Endpoint: {self.service_endpoint}") def get_client(self): """ @@ -67,6 +68,16 @@ def get_client(self): """ return self.s3_client + def generate_bucket_name(self): + """ + Generates a unique bucket name + """ + key = self.config['access_key_id'] + endpoint = hashlib.sha1(self.service_endpoint.encode()).hexdigest()[:6] + self.config['storage_bucket'] = f'lithops-{endpoint}-{key[:6].lower()}' + + return self.config['storage_bucket'] + def create_bucket(self, bucket_name): """ Create a bucket if it doesn't exist diff --git a/lithops/storage/backends/ceph/config.py b/lithops/storage/backends/ceph/config.py index 810ddc8c1..36bf6a48d 100644 --- a/lithops/storage/backends/ceph/config.py +++ b/lithops/storage/backends/ceph/config.py @@ -15,9 +15,6 @@ # -import hashlib - - REQ_PARAMS = ('endpoint', 'secret_access_key', 'access_key_id') @@ -32,8 +29,3 @@ def load_config(config_data): if not config_data['ceph']['endpoint'].startswith('http'): raise Exception('Ceph endpoint must start with http:// or https://') - - if 'storage_bucket' not in config_data['ceph']: - key = config_data['ceph']['access_key_id'] - endpoint = hashlib.sha1(config_data['ceph']['endpoint'].encode()).hexdigest()[:6] - config_data['ceph']['storage_bucket'] = f'lithops-{endpoint}-{key[:6].lower()}' diff --git a/lithops/storage/backends/gcp_storage/config.py b/lithops/storage/backends/gcp_storage/config.py index 432e51451..ad8453056 100644 --- a/lithops/storage/backends/gcp_storage/config.py +++ b/lithops/storage/backends/gcp_storage/config.py @@ -15,7 +15,6 @@ # import copy -import hashlib import os @@ -39,9 +38,3 @@ def load_config(config_data=None): if 'region' not in config_data['gcp_storage']: raise Exception("'region' parameter is mandatory under 'gcp_storage' or 'gcp' section of the configuration") - - if 'storage_bucket' not in config_data['gcp_storage']: - gcps = config_data['gcp_storage'] - region = gcps['region'] - key = hashlib.sha1(gcps['credentials_path'].encode()).hexdigest()[:6] - config_data['gcp_storage']['storage_bucket'] = f'lithops-{region}-{key[:6].lower()}' diff --git a/lithops/storage/backends/gcp_storage/gcp_storage.py b/lithops/storage/backends/gcp_storage/gcp_storage.py index b82d049af..086995b72 100644 --- a/lithops/storage/backends/gcp_storage/gcp_storage.py +++ b/lithops/storage/backends/gcp_storage/gcp_storage.py @@ -18,6 +18,7 @@ import os import shutil import time +import hashlib import logging from requests.exceptions import SSLError as TooManyConnectionsError from io import BytesIO @@ -35,6 +36,7 @@ class GCPStorageBackend: def __init__(self, gcp_storage_config): logger.debug("Creating GCP Storage client") + self.config = gcp_storage_config self.credentials_path = gcp_storage_config.get('credentials_path') self.region = gcp_storage_config['region'] @@ -51,6 +53,15 @@ def __init__(self, gcp_storage_config): def get_client(self): return self.client + def generate_bucket_name(self): + """ + Generates a unique bucket name + """ + key = hashlib.sha1(self.credentials_path.encode()).hexdigest()[:6] + self.config['storage_bucket'] = f'lithops-{self.region}-{key[:6].lower()}' + + return self.config['storage_bucket'] + def exists_bucket(self, bucket_name): try: self.client.get_bucket(bucket_name, timeout=TIMEOUT) diff --git a/lithops/storage/backends/ibm_cos/config.py b/lithops/storage/backends/ibm_cos/config.py index 1a65242cb..c629d2fc2 100644 --- a/lithops/storage/backends/ibm_cos/config.py +++ b/lithops/storage/backends/ibm_cos/config.py @@ -78,18 +78,3 @@ def load_config(config_data): if 'region' not in config_data['ibm_cos']: endpoint = config_data['ibm_cos']['endpoint'] config_data['ibm_cos']['region'] = endpoint.split('//')[1].split('.')[1] - - if 'access_key' in config_data['ibm_cos']: - config_data['ibm_cos']['access_key_id'] = config_data['ibm_cos'].pop('access_key') - if 'secret_key' in config_data['ibm_cos']: - config_data['ibm_cos']['secret_access_key'] = config_data['ibm_cos'].pop('secret_key') - - if 'storage_bucket' not in config_data['ibm_cos']: - if not {'access_key_id', 'secret_access_key'}.issubset(config_data['ibm_cos']): - msg = "'storage_bucket' parameter not found in config. " - msg += "You must provide HMAC Credentials if you want the bucket to be automatically created" - raise Exception(msg) - cosc = config_data['ibm_cos'] - key = cosc.get('access_key_id') or cosc.get('api_key') or cosc.get('iam_api_key') - region = config_data['ibm_cos']['region'] - config_data['ibm_cos']['storage_bucket'] = f'lithops-{region}-{key[:6].lower()}' diff --git a/lithops/storage/backends/ibm_cos/ibm_cos.py b/lithops/storage/backends/ibm_cos/ibm_cos.py index c6465daf5..e62f25c89 100644 --- a/lithops/storage/backends/ibm_cos/ibm_cos.py +++ b/lithops/storage/backends/ibm_cos/ibm_cos.py @@ -116,6 +116,19 @@ def get_client(self): """ return self.cos_client + def generate_bucket_name(self): + """ + Generates a unique bucket name + """ + if not {'access_key_id', 'secret_access_key'}.issubset(self.config): + msg = "'storage_bucket' parameter not found in config. You must provide HMAC " + msg += "Credentials if you want the bucket to be automatically created" + raise Exception(msg) + key = self.config.get('access_key_id') or self.api_key or self.iam_api_key + self.config['storage_bucket'] = f"lithops-{self.region}-{key[:6].lower()}" + + return self.config['storage_bucket'] + def create_bucket(self, bucket_name): """ Create a bucket if it doesn't exist diff --git a/lithops/storage/backends/minio/config.py b/lithops/storage/backends/minio/config.py index eff74165e..1f3a3f7fd 100644 --- a/lithops/storage/backends/minio/config.py +++ b/lithops/storage/backends/minio/config.py @@ -15,9 +15,6 @@ # -import hashlib - - REQ_PARAMS = ('endpoint', 'secret_access_key', 'access_key_id') @@ -32,8 +29,3 @@ def load_config(config_data): if not config_data['minio']['endpoint'].startswith('http'): raise Exception('MinIO endpoint must start with http:// or https://') - - if 'storage_bucket' not in config_data['minio']: - key = config_data['minio']['access_key_id'] - endpoint = hashlib.sha1(config_data['minio']['endpoint'].encode()).hexdigest()[:6] - config_data['minio']['storage_bucket'] = f'lithops-{endpoint}-{key[:6].lower()}' diff --git a/lithops/storage/backends/minio/minio.py b/lithops/storage/backends/minio/minio.py index a23ec7c12..b2b14acb1 100644 --- a/lithops/storage/backends/minio/minio.py +++ b/lithops/storage/backends/minio/minio.py @@ -15,6 +15,7 @@ # import os +import hashlib import logging import boto3 import botocore @@ -37,9 +38,9 @@ def __init__(self, minio_config): logger.debug("Creating MinIO client") self.config = minio_config user_agent = minio_config['user_agent'] - service_endpoint = minio_config['endpoint'] + self.service_endpoint = minio_config['endpoint'] - logger.debug(f"Setting MinIO endpoint to {service_endpoint}") + logger.debug(f"Setting MinIO endpoint to {self.service_endpoint}") client_config = botocore.client.Config( max_pool_connections=128, @@ -54,11 +55,11 @@ def __init__(self, minio_config): aws_secret_access_key=minio_config['secret_access_key'], aws_session_token=minio_config.get('session_token'), config=client_config, - endpoint_url=service_endpoint + endpoint_url=self.service_endpoint ) msg = STORAGE_CLI_MSG.format('MinIO') - logger.info(f"{msg} - Endpoint: {service_endpoint}") + logger.info(f"{msg} - Endpoint: {self.service_endpoint}") def get_client(self): """ @@ -67,6 +68,16 @@ def get_client(self): """ return self.s3_client + def generate_bucket_name(self): + """ + Generates a unique bucket name + """ + key = self.config['access_key_id'] + endpoint = hashlib.sha1(self.service_endpoint.encode()).hexdigest()[:6] + self.config['storage_bucket'] = f'lithops-{endpoint}-{key[:6].lower()}' + + return self.config['storage_bucket'] + def create_bucket(self, bucket_name): """ Create a bucket if it doesn't exist diff --git a/lithops/storage/backends/oracle_oss/config.py b/lithops/storage/backends/oracle_oss/config.py index 7db1cecf9..f6319e84e 100644 --- a/lithops/storage/backends/oracle_oss/config.py +++ b/lithops/storage/backends/oracle_oss/config.py @@ -37,8 +37,3 @@ def load_config(config_data=None): temp = copy.deepcopy(config_data['oracle_oss']) config_data['oracle_oss'].update(config_data['oracle']) config_data['oracle_oss'].update(temp) - - if 'storage_bucket' not in config_data['oracle_oss']: - user = config_data['oracle_oss']['user'] - region = config_data['oracle_oss']['region'] - config_data['oracle_oss']['storage_bucket'] = f'lithops-{region}-{user[-8:-1].lower()}' diff --git a/lithops/storage/backends/oracle_oss/oracle_oss.py b/lithops/storage/backends/oracle_oss/oracle_oss.py index c0577509c..f5ec60b59 100644 --- a/lithops/storage/backends/oracle_oss/oracle_oss.py +++ b/lithops/storage/backends/oracle_oss/oracle_oss.py @@ -56,6 +56,15 @@ def _init_storage_client(self): def get_client(self): return self + def generate_bucket_name(self): + """ + Generates a unique bucket name + """ + user = self.config['user'] + self.config['storage_bucket'] = f'lithops-{self.region}-{user[-8:-1].lower()}' + + return self.config['storage_bucket'] + def create_bucket(self, bucket_name): """ Create a bucket if it doesn't exist diff --git a/lithops/storage/backends/swift/swift.py b/lithops/storage/backends/swift/swift.py index 580106ccd..bd64b3677 100644 --- a/lithops/storage/backends/swift/swift.py +++ b/lithops/storage/backends/swift/swift.py @@ -57,6 +57,14 @@ def __init__(self, swift_config): msg = STORAGE_CLI_MSG.format('OpenStack Swift') logger.info("{} - Region: {}".format(msg, self.region)) + def generate_bucket_name(self): + """ + Generates a unique bucket name + """ + self.config['storage_bucket'] = f'lithops-{self.region}-{self.user_id[:6].lower()}' + + return self.config['storage_bucket'] + def generate_swift_token(self): """ Generates new token for accessing to Swift. @@ -89,7 +97,7 @@ def generate_swift_token(self): message = json.loads(r.text)['error']['message'] raise Exception("{} - {} - {}".format(r.status_code, r.reason, message)) - def put_object(self, container_name, key, data): + def put_object(self, bucket_name, key, data): """ Put an object in Swift. Override the object if the key already exists. :param key: key of the object. @@ -97,7 +105,7 @@ def put_object(self, container_name, key, data): :type data: str/bytes :return: None """ - url = '/'.join([self.endpoint, container_name, key]) + url = '/'.join([self.endpoint, bucket_name, key]) try: res = self.session.put(url, data=data) status = 'OK' if res.status_code == 201 else 'Error' @@ -108,16 +116,14 @@ def put_object(self, container_name, key, data): except Exception as e: print(e) - def get_object(self, container_name, key, stream=False, extra_get_args={}): + def get_object(self, bucket_name, key, stream=False, extra_get_args={}): """ Get object from Swift with a key. Throws StorageNoSuchKeyError if the given key does not exist. :param key: key of the object :return: Data of the object :rtype: str/bytes """ - if not container_name: - container_name = self.storage_container - url = '/'.join([self.endpoint, container_name, key]) + url = '/'.join([self.endpoint, bucket_name, key]) headers = {'X-Auth-Token': self.token} headers.update(extra_get_args) try: @@ -129,14 +135,14 @@ def get_object(self, container_name, key, stream=False, extra_get_args={}): data = res.content return data elif res.status_code == 404: - raise StorageNoSuchKeyError(container_name, key) + raise StorageNoSuchKeyError(bucket_name, key) else: raise Exception('{} - {}'.format(res.status_code, key)) except StorageNoSuchKeyError: - raise StorageNoSuchKeyError(container_name, key) + raise StorageNoSuchKeyError(bucket_name, key) except Exception as e: print(e) - raise StorageNoSuchKeyError(container_name, key) + raise StorageNoSuchKeyError(bucket_name, key) def upload_file(self, file_name, bucket, key=None, extra_args={}, config=None): """Upload a file @@ -184,35 +190,35 @@ def download_file(self, bucket, key, file_name=None, extra_args={}, config=None) return False return True - def head_object(self, container_name, key): + def head_object(self, bucket_name, key): """ Head object from Swift with a key. Throws StorageNoSuchKeyError if the given key does not exist. :param key: key of the object :return: Data of the object :rtype: str/bytes """ - url = '/'.join([self.endpoint, container_name, key]) + url = '/'.join([self.endpoint, bucket_name, key]) try: res = self.session.head(url) if res.status_code == 200: return res.headers elif res.status_code == 404: - raise StorageNoSuchKeyError(container_name, key) + raise StorageNoSuchKeyError(bucket_name, key) else: raise Exception('{} - {}'.format(res.status_code, key)) except Exception: - raise StorageNoSuchKeyError(container_name, key) + raise StorageNoSuchKeyError(bucket_name, key) - def delete_object(self, container_name, key): + def delete_object(self, bucket_name, key): """ Delete an object from Swift. :param bucket: bucket name :param key: data key """ - url = '/'.join([self.endpoint, container_name, key]) + url = '/'.join([self.endpoint, bucket_name, key]) return self.session.delete(url) - def delete_objects(self, container_name, key_list): + def delete_objects(self, bucket_name, key_list): """ Delete a list of objects from Swift. :param bucket: bucket name @@ -223,13 +229,13 @@ def delete_objects(self, container_name, key_list): keys_to_delete = [] for key in key_list: - keys_to_delete.append('/{}/{}'.format(container_name, key)) + keys_to_delete.append('/{}/{}'.format(bucket_name, key)) keys_to_delete = '\n'.join(keys_to_delete) url = '/'.join([self.endpoint, '?bulk-delete']) return self.session.delete(url, data=keys_to_delete, headers=headers) - def list_objects(self, container_name, prefix='', match_pattern=None): + def list_objects(self, bucket_name, prefix='', match_pattern=None): """ Lists the objects in a bucket. Throws StorageNoSuchKeyError if the given bucket does not exist. :param key: key of the object @@ -237,9 +243,9 @@ def list_objects(self, container_name, prefix='', match_pattern=None): :rtype: str/bytes """ if prefix: - url = '/'.join([self.endpoint, container_name, '?format=json&prefix=' + prefix]) + url = '/'.join([self.endpoint, bucket_name, '?format=json&prefix=' + prefix]) else: - url = '/'.join([self.endpoint, container_name, '?format=json']) + url = '/'.join([self.endpoint, bucket_name, '?format=json']) try: res = self.session.get(url) objects = res.json() @@ -249,7 +255,7 @@ def list_objects(self, container_name, prefix='', match_pattern=None): except Exception as e: raise e - def list_keys(self, container_name, prefix): + def list_keys(self, bucket_name, prefix): """ Return a list of keys for the given prefix. :param prefix: Prefix to filter object names. @@ -257,7 +263,7 @@ def list_keys(self, container_name, prefix): :rtype: list of str """ try: - objects = self.list_objects(container_name, prefix) + objects = self.list_objects(bucket_name, prefix) object_keys = [r['name'] for r in objects] return object_keys except Exception as e: diff --git a/lithops/storage/storage.py b/lithops/storage/storage.py index 4200b2e6e..fae66dcca 100644 --- a/lithops/storage/storage.py +++ b/lithops/storage/storage.py @@ -58,7 +58,6 @@ def __init__(self, config=None, backend=None, storage_config=None): self.config = extract_storage_config(storage_config) self.backend = self.config['backend'] - self.bucket = self.config['bucket'] try: module_location = f'lithops.storage.backends.{self.backend}' @@ -70,6 +69,8 @@ def __init__(self, config=None, backend=None, storage_config=None): f"'{self.backend}' storage backend") raise e + self.bucket = self.config['bucket'] or self.storage_handler.generate_bucket_name() + def get_client(self) -> object: """ Retrieves the underlying storage client.