diff --git a/hadoop-hdds/docs/content/recipe/BotoClient.md b/hadoop-hdds/docs/content/recipe/BotoClient.md new file mode 100644 index 000000000000..803d0b33a284 --- /dev/null +++ b/hadoop-hdds/docs/content/recipe/BotoClient.md @@ -0,0 +1,189 @@ +--- +title: Access Ozone object store with Amazon Boto3 client +linktitle: Ozone with Boto3 Client +summary: How to access Ozone object store with Boto3 client? +--- + + +This recipe shows how Ozone object store can be accessed from Boto3 client. Following apis were verified: + + - Create bucket + - List bucket + - Head bucket + - Delete bucket + - Upload file + - Download file + - Delete objects(keys) + - Head object + - Multipart upload + + +## Requirements + +You will need a higher version of Python3 for your Boto3 client as Boto3 installation requirement indicates at here: +https://boto3.amazonaws.com/v1/documentation/api/latest/index.html + +## Obtain resource to Ozone +You may reference Amazon Boto3 documentation regarding the creation of 's3' resources at here: +https://boto3.amazonaws.com/v1/documentation/api/latest/guide/resources.html + + s3 = boto3.resource('s3', + endpoint_url='http://localhost:9878', + aws_access_key_id='testuser/scm@EXAMPLE.COM', + aws_secret_access_key='c261b6ecabf7d37d5f9ded654b1c724adac9bd9f13e247a235e567e8296d2999' + ) + 'endpoint_url' is pointing to Ozone s3 endpoint. + + +## Obtain client to Ozone via session +You many reference Amazon Boto3 documentation regarding session at here: +https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html + + Create a session + session = boto3.session.Session() + + Obtain s3 client to Ozone via session: + + s3_client = session.client( + service_name='s3', + aws_access_key_id='testuser/scm@EXAMPLE.COM', + aws_secret_access_key='c261b6ecabf7d37d5f9ded654b1c724adac9bd9f13e247a235e567e8296d2999', + endpoint_url='http://localhost:9878', + ) + 'endpoint_url' is pointing to Ozone s3 endpoint. + + In our code sample below, we're demonstrating the useage of both s3 and s3_client. + +There are multiple ways to configure Boto3 client credentials if you're connecting to a secured cluster. In these cases, +the above lines of passing 'aws_access_key_id' and 'aws_secret_access_key' when creating Ozone s3 client shall be skipped. + +Please refer to Boto3 documentation for details at here: +https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html + + +### Create a bucket + response = s3_client.create_bucket(Bucket='bucket1') + print(response) + +This will create a bucket 'bucket1' in Ozone volume 's3v'. + +### List buckets + response = s3_client.list_buckets() + print('Existing buckets:') + for bucket in response['Buckets']: + print(f' {bucket["Name"]}') + +This will list all buckets in Ozone volume 's3v'. + +### Head a bucket + response = s3_client.head_bucket(Bucket='bucket1') + print(response) + +This will head bucket 'bucket1' in Ozone volume 's3v'. + +### Delete a bucket + response = s3_client.delete_bucket(Bucket='bucket1') + print(response) + +This will delete bucket 'bucket1' from Ozone volume 's3v'. + +### Upload a file + response = s3.Bucket('bucket1').upload_file('./README.md','README.md') + print(response) + +This will upload 'README.md' to Ozone creates a key 'README.md' in volume 's3v'. + +### Download a file + response = s3.Bucket('bucket1').download_file('README.md', 'download.md') + print(response) + +This will download 'README.md' from Ozone volume 's3v' to local and create a file with name 'download.md'. + +### Head an object + response = s3_client.head_object(Bucket='bucket1', Key='README.md') + print(response) + +This will head object 'README.md' from Ozone volume 's3v' in bucket 'bucket1'. + +### Delete Objects + response = s3_client.delete_objects( + Bucket='bucket1', + Delete={ + 'Objects': [ + { + 'Key': 'README4.md', + }, + { + 'Key': 'README3.md', + }, + ], + 'Quiet': False, + }, + ) + +This will delete objects 'README3.md' and 'README4.md' from Ozone volume 's3v' in bucket 'bucket1'. + +### Multipart upload + response = s3_client.create_multipart_upload(Bucket='bucket1', Key='key1') + print(response) + uid=response['UploadId'] + print(uid) + + response = s3_client.upload_part_copy( + Bucket='bucket1', + CopySource='/bucket1/maven.gz', + Key='key1', + PartNumber=1, + UploadId=str(uid) + ) + print(response) + etag1=response.get('CopyPartResult').get('ETag') + print(etag1) + + response = s3_client.upload_part_copy( + Bucket='bucket1', + CopySource='/bucket1/maven1.gz', + Key='key1', + PartNumber=2, + UploadId=str(uid) + ) + print(response) + etag2=response.get('CopyPartResult').get('ETag') + print(etag2) + + response = s3_client.complete_multipart_upload( + Bucket='bucket1', + Key='key1', + MultipartUpload={ + 'Parts': [ + { + 'ETag': str(etag1), + 'PartNumber': 1, + }, + { + 'ETag': str(etag2), + 'PartNumber': 2, + }, + ], + }, + UploadId=str(uid), + ) + print(response) + +This will use 'maven.gz' and 'maven1.gz' as copy source from Ozone volume 's3v' to create a new object 'key1' +in Ozone volume 's3v'. Please note 'ETag's is required and important for the call. diff --git a/hadoop-ozone/dist/src/main/compose/ozone/test.sh b/hadoop-ozone/dist/src/main/compose/ozone/test.sh index 16f55ca8b00f..cc516f303a7f 100755 --- a/hadoop-ozone/dist/src/main/compose/ozone/test.sh +++ b/hadoop-ozone/dist/src/main/compose/ozone/test.sh @@ -47,6 +47,8 @@ for bucket in link generated; do execute_robot_test scm -v BUCKET:${bucket} -N s3-${bucket} s3 done +execute_robot_test scm -v BUCKET:link -N s3-link botoclient + execute_robot_test scm recon execute_robot_test scm om-ratis diff --git a/hadoop-ozone/dist/src/main/smoketest/botoclient/README.md b/hadoop-ozone/dist/src/main/smoketest/botoclient/README.md new file mode 100644 index 000000000000..b67a52790854 --- /dev/null +++ b/hadoop-ozone/dist/src/main/smoketest/botoclient/README.md @@ -0,0 +1,16 @@ + + +## Ozone S3 Boto3 Client Acceptance Tests + diff --git a/hadoop-ozone/dist/src/main/smoketest/botoclient/boto3.robot b/hadoop-ozone/dist/src/main/smoketest/botoclient/boto3.robot new file mode 100644 index 000000000000..439e530bdd31 --- /dev/null +++ b/hadoop-ozone/dist/src/main/smoketest/botoclient/boto3.robot @@ -0,0 +1,35 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +*** Settings *** +Documentation S3 gateway test with Boto3 Client +Library OperatingSystem +Library String +Library Process +Library BuiltIn +Resource ../commonlib.robot +Resource ../s3/commonawslib.robot +Test Timeout 15 minutes +Suite Setup Setup s3 tests + +*** Variables *** +${ENDPOINT_URL} http://s3g:9878 +${BUCKET} generated + +*** Test Cases *** + +Bobo3 Client Test + ${result} = Execute python3 /opt/hadoop/smoketest/botoclient/boto_client.py + Should contain ${result} Boto3 Client Test PASSED! diff --git a/hadoop-ozone/dist/src/main/smoketest/botoclient/boto_client.py b/hadoop-ozone/dist/src/main/smoketest/botoclient/boto_client.py new file mode 100755 index 000000000000..509353957ee0 --- /dev/null +++ b/hadoop-ozone/dist/src/main/smoketest/botoclient/boto_client.py @@ -0,0 +1,220 @@ +import os +import logging +import json +import unittest +import boto3 +from botocore.client import Config +from botocore.exceptions import ClientError + +class TestBotoClient(unittest.TestCase): + + s3 = None + s3_client = None + setup_done = False + + def setUp(self): + if TestBotoClient.setup_done: + return + + TestBotoClient.setup_done = True + + TestBotoClient.s3 = boto3.resource('s3', + endpoint_url='http://s3g:9878') + + TestBotoClient.s3_client = boto3.session.Session().client( + service_name='s3', + endpoint_url='http://s3g:9878') + + self.s3_client.create_bucket(Bucket='bucket1') + self.s3_client.create_bucket(Bucket='bucket10') + + try: + response = self.s3_client.upload_file('README.md', 'bucket1', 'README3.md') + print(response) + + response = self.s3.Bucket('bucket1').upload_file('README.md','README4.md') + print(response) + + self.s3.Bucket('bucket1').upload_file('README.md','README10.md') + print(response) + except ClientError as e: + logging.error(e) + print(e) + + f = open('multiUpload.gz',"wb") + f.seek(10485760) + f.write(b"\0") + f.close() + self.s3.Bucket('bucket1').upload_file('./multiUpload.gz','multiUpload.1.gz') + self.s3.Bucket('bucket1').upload_file('./multiUpload.gz','multiUpload.2.gz') + + def test_create_bucket(self): + try: + self.assertTrue(self.s3_client is not None) + response = self.s3_client.create_bucket(Bucket='bucket3') + print(response) + except ClientError as e: + print(e) + logging.error(e) + return False + return True + + def test_list_bucket(self): + try: + self.assertTrue(self.s3_client is not None) + response = self.s3_client.list_buckets() + print(response) + except ClientError as e: + print(e) + logging.error(e) + return False + return True + + def test_head_bucket(self): + self.assertTrue(self.s3_client is not None) + try: + response = self.s3_client.head_bucket(Bucket='bucket1') + print(response) + except ClientError as e: + print(e) + logging.error(e) + return False + return True + + def test_bucket_delete(self): + self.assertTrue(self.s3_client is not None) + try: + response = self.s3_client.delete_bucket(Bucket='bucket10') + print(response) + except ClientError as e: + logging.error(e) + return False + return True + + def test_upload_file(self): + self.assertTrue(self.s3 is not None) + try: + response = self.s3.Bucket('bucket1').upload_file('./README.md','README1.md') + print(response) + except ClientError as e: + print(e) + logging.error(e) + return False + return True + + def test_download_file(self): + self.assertTrue(self.s3 is not None) + try: + response = self.s3.Bucket('bucket1').download_file('README10.md', 'download.md') + print(response) + except ClientError as e: + print(e) + logging.error(e) + return False + return True + + def test_delete_objects(self): + self.assertTrue(self.s3_client is not None) + try: + response = self.s3_client.delete_objects( + Bucket='bucket1', + Delete={ + 'Objects': [ + { + 'Key': 'README4.md', + }, + { + 'Key': 'README3.md', + }, + ], + 'Quiet': False, + }, + ) + print(response) + except ClientError as e: + print(e) + logging.error(e) + return False + return True + + def test_head_object(self): + self.assertTrue(self.s3_client is not None) + try: + response = self.s3_client.head_object(Bucket='bucket1', Key='README10.md') + print(response) + except ClientError as e: + print(e) + logging.error(e) + return False + return True + + def test_multi_uploads(self): + self.assertTrue(self.s3_client is not None) + try: + response = self.s3_client.create_multipart_upload(Bucket='bucket1', Key='key1') + print(response) + uid=response['UploadId'] + + response = self.s3_client.upload_part_copy( + Bucket='bucket1', + CopySource='/bucket1/multiUpload.1.gz', + Key='key1', + PartNumber=1, + UploadId=str(uid) + ) + etag1=response.get('CopyPartResult').get('ETag') + + response = self.s3_client.upload_part_copy( + Bucket='bucket1', + CopySource='/bucket1/multiUpload.2.gz', + Key='key1', + PartNumber=2, + UploadId=str(uid) + ) + etag2=response.get('CopyPartResult').get('ETag') + + response = self.s3_client.complete_multipart_upload( + Bucket='bucket1', + Key='key1', + MultipartUpload={ + 'Parts': [ + { + 'ETag': str(etag1), + 'PartNumber': 1, + }, + { + 'ETag': str(etag2), + 'PartNumber': 2, + }, + ], + }, + UploadId=str(uid), + ) + except ClientError as e: + print(e) + logging.error(e) + return False + return True + +if __name__ == '__main__': + #unittest.main() + suite = unittest.TestSuite() + + suite.addTest(TestBotoClient('test_create_bucket')) + suite.addTest(TestBotoClient('test_list_bucket')) + suite.addTest(TestBotoClient('test_head_bucket')) + suite.addTest(TestBotoClient('test_bucket_delete')) + suite.addTest(TestBotoClient('test_upload_file')) + suite.addTest(TestBotoClient('test_download_file')) + suite.addTest(TestBotoClient('test_delete_objects')) + suite.addTest(TestBotoClient('test_head_object')) + suite.addTest(TestBotoClient('test_multi_uploads')) + + result = unittest.TextTestRunner(verbosity=2).run(suite) + + if result.wasSuccessful(): + print("Boto3 Client Test PASSED!") + exit(0) + else: + print("Boto3 Client Test FAILED!") + exit(1)