From 342fa5aa7b2f94bbe41ce867b2d22c2f1f07ba53 Mon Sep 17 00:00:00 2001 From: Rasmus Oscar Welander Date: Tue, 30 Jul 2024 13:55:20 +0200 Subject: [PATCH 1/5] Added main class cs3client, configuration and library setup --- .gitignore | 3 + examples/default.conf | 49 +++++++++++ setup.py | 30 +++++++ src/__init__.py | 0 src/config.py | 189 ++++++++++++++++++++++++++++++++++++++++++ src/cs3client.py | 62 ++++++++++++++ 6 files changed, 333 insertions(+) create mode 100644 examples/default.conf create mode 100644 setup.py create mode 100644 src/__init__.py create mode 100644 src/config.py create mode 100644 src/cs3client.py diff --git a/.gitignore b/.gitignore index 82f9275..34de50d 100644 --- a/.gitignore +++ b/.gitignore @@ -51,6 +51,9 @@ coverage.xml .pytest_cache/ cover/ +# vscode settings +.vscode/ + # Translations *.mo *.pot diff --git a/examples/default.conf b/examples/default.conf new file mode 100644 index 0000000..e5be292 --- /dev/null +++ b/examples/default.conf @@ -0,0 +1,49 @@ +# example configuration file for the CS3client. +# +# Authors: Rasmus Welander, Diogo Castro, Giuseppe Lo Presti +# Emails: rasmus.oscar.welander@cern.ch, diogo.castro@cern.ch, giuseppe.lopresti@cern.ch +# Last updated: 29/07/2024 + +[cs3client] + +# Required +host = localhost:19000 +# Optional, defaults to 4194304 +chunk_size = 4194304 +# Optional, defaults to 10 +grpc_timeout = 10 +# Optional, defaults to 10 +http_timeout = 10 + +# Optional, defaults to True +tus_enabled = False + +# Optional, defaults to True +ssl_enabled = False +# Optional, defaults to True +ssl_verify = False +# Optional, defaults to an empty string +ssl_client_cert = test_client_cert +# Optional, defaults to an empty string +ssl_client_key = test_client_key +# Optional, defaults to an empty string +ssl_ca_cert = test_ca_cert + +# Optinal, defaults to an empty string +auth_client_id = einstein +# Optional, defaults to basic +auth_login_type = basic + +# For the future lock implementation + +# Optional, defaults to False +# This configuration is used to enable/disable the fallback mechanism +# if the locks are not implemented in the storage provider +lock_by_setting_attr = False +# This configuration is used to enable/disable the fallback mechanism +# if the locks are not implemented in the storage provider +# Optional, defaults to False +lock_not_impl = False +# Optional, defaults to 1800 +lock_expiration = 1800 + diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..fd099b2 --- /dev/null +++ b/setup.py @@ -0,0 +1,30 @@ +""" +setup.py + +setup file for the cs3client package. + +Authors: Rasmus Welander, Diogo Castro, Giuseppe Lo Presti. +Emails: rasmus.oscar.welander@cern.ch, diogo.castro@cern.ch, giuseppe.lopresti@cern.ch +Last updated: 26/07/2024 +""" + +from setuptools import setup, find_packages + +setup( + name="cs3client", + version="0.1", + author="Rasmus Welander, Diogo Castro, Giuseppe Lo Presti", + package_dir={"": "src"}, + packages=find_packages(where="src"), + py_modules=["cs3client"], + install_requires=[ + "grpcio>=1.47.0", + "grpcio-tools>=1.47.0", + "pyOpenSSL", + "requests", + "cs3apis>=0.1.dev101", + "PyJWT", + "protobuf", + "cryptography", + ], +) diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/config.py b/src/config.py new file mode 100644 index 0000000..316b204 --- /dev/null +++ b/src/config.py @@ -0,0 +1,189 @@ +""" +config.py + +Authors: Rasmus Welander, Diogo Castro, Giuseppe Lo Presti. +Emails: rasmus.oscar.welander@cern.ch, diogo.castro@cern.ch, giuseppe.lopresti@cern.ch +Last updated: 29/07/2024 +""" + +from configparser import ConfigParser + + +class Config: + """ + A class to read and store the configuration parameters provided to the CS3 client. + """ + + def __init__(self, config: ConfigParser, config_category: str) -> None: + """ + Initializes the Config class with the configuration parameters. + + :param config: Dictionary containing configuration parameters. + :param config_category: The category of the configuration parameters. + """ + self._config_category: str = config_category + self._config: ConfigParser = config + + @property + def host(self) -> str: + """ + The host property returns the host address and port from the configuration. + + :return: host address:port + """ + return self._config.get(self._config_category, "host") + + @property + def chunk_size(self) -> int: + """ + The chunk_size property returns the chunk_size value from the configuration, + fallback to 4194304 if not present. + + :return: The chunk size value. + """ + return self._config.getint(self._config_category, "chunk_size", fallback=4194304) + + @property + def grpc_timeout(self) -> int: + """ + The grpc_timeout property returns the grpc_timeout value from the configuration, + fallback to 10 if not present. + + :return: The grpc timeout value. + """ + return self._config.getint(self._config_category, "grpc_timeout", fallback=10) + + @property + def http_timeout(self) -> int: + """ + The http_timeout property returns the http_timeout value from the configuration, + fallback to 10 if not present. + + :return: The http timeout value. + """ + return self._config.getint(self._config_category, "http_timeout", fallback=10) + + @property + def ssl_enabled(self) -> bool: + """ + The ssl_enabled property returns the ssl_enabled value from the configuration, + fallback to True if not present. + + :return: ssl_enabled value. + """ + return self._config.getboolean(self._config_category, "ssl_enabled", fallback=False) + + @property + def ssl_verify(self) -> bool: + """ + The ssl_verify property returns the ssl_verify value from the configuration, + + :return: ssl_verify + """ + return self._config.getboolean(self._config_category, "ssl_verify", fallback=False) + + @property + def ssl_client_cert(self) -> str: + """ + The ssl_client_cert property returns the ssl_client_cert value from the configuration, + if not present, fallback to an empty string since it is not required if ssl is not enabled. + + :return: ssl_client_cert + """ + return self._config.get(self._config_category, "ssl_client_cert", fallback=None) + + @property + def ssl_client_key(self) -> str: + """ + The ssl_client_key property returns the ssl_client_key value from the configuration, + if not present, fallback to an empty string since it is not required if ssl is not enabled. + + :return: ssl_client_key + """ + return self._config.get(self._config_category, "ssl_client_key", fallback=None) + + @property + def ssl_ca_cert(self) -> str: + """ + The ssl_ca_cert property returns the ssl_ca_cert value from the configuration, + if not present, fallback to an empty string since it is not required if ssl is not enabled. + + :return: ssl_ca_cert + """ + return self._config.get(self._config_category, "ssl_ca_cert", fallback=None) + + @property + def auth_login_type(self) -> str: + """ + The auth_login_type property returns the auth_login_type value from the configuration. + e.g. basic, bearer, oauth, machine. + + :return: auth_login_type + """ + return self._config.get(self._config_category, "auth_login_type", fallback="basic") + + @property + def auth_client_id(self) -> str: + """ + The auth_client_id property returns the auth_client_id value from the configuration, + + :return: auth_client_id + """ + return self._config.get(self._config_category, "auth_client_id", fallback=None) + + @property + def tus_enabled(self) -> bool: + """ + The tus_enabled property returns the tus_enabled value from the configuration, + + :return: tus_enabled + """ + return self._config.getboolean(self._config_category, "tus_enabled", fallback=False) + + # For the lock implementation + @property + def lock_by_setting_attr(self) -> bool: + """ + The lock_by_setting_attr property returns the lock_by_setting_attr value from the configuration, + fallback to False if not present. + + The lock by setting attribute setting should be set if the storage provider does not + implement locking functionality. In which case the client will use the fallback mechanism + by locking resources by setting metadata attributes. If lock_not_impl is set to false and + lock_by_setting_attr is set to true, the client will attempt to lock normally first, + and if that fails, it will attempt to lock by setting metadata attributes. + + + :return: lock_by_setting_attr + """ + return self._config.getboolean(self._config_category, "lock_by_setting_attr", fallback=False) + + # For the lock implementation + @property + def lock_not_impl(self) -> bool: + """ + The lock_not_impl property returns the lock_not_impl value from the configuration, + fallback to False if not present. + + The lock not implemented setting should be set if the storage provider + does not implement locking functionality. In which case the client will use the + fallback mechanism by locking resources by setting metadata attributes if the + lock_by_setting_attr is set to True. + + :return: lock_not_impl + """ + return self._config.getboolean(self._config_category, "lock_not_impl", fallback=False) + + # For the lock implementation + @property + def lock_expiration(self) -> int: + """ + The lock_expiration property returns the lock_expiration value from the configuration, + fallback to 1800 if not present. + + The lock expiration setting is used to determine the time + in seconds before a lock is considered expired. + + :return: lock_expiration + """ + return self._config.getint(self._config_category, "lock_expiration", fallback=1800) diff --git a/src/cs3client.py b/src/cs3client.py new file mode 100644 index 0000000..9da5060 --- /dev/null +++ b/src/cs3client.py @@ -0,0 +1,62 @@ +""" +cs3client.py + +Authors: Rasmus Welander, Diogo Castro, Giuseppe Lo Presti. +Emails: rasmus.oscar.welander@cern.ch, diogo.castro@cern.ch, giuseppe.lopresti@cern.ch +Last updated: 29/07/2024 +""" + +import grpc +import logging +import cs3.gateway.v1beta1.gateway_api_pb2_grpc as cs3gw_grpc + +from configparser import ConfigParser +from auth import Auth +from file import File +from config import Config + + +class CS3Client: + """ + Client class to interact with the CS3 API. + """ + + def __init__(self, config: ConfigParser, config_category: str, log: logging.Logger) -> None: + """ + Initializes the CS3Client class with configuration and logger. + :param config: Dictionary containing configuration parameters. + :param config_category: Category of the configuration parameters. + :param log: Logger instance for logging. + """ + + self._config: Config = Config(config, config_category) + self._log: logging.Logger = log + + try: + self.channel: grpc.Channel = self._create_channel() + grpc.channel_ready_future(self.channel).result(timeout=self._config.grpc_timeout) + except grpc.FutureTimeoutError as e: + log.error(f'msg="Failed to connect to Reva via GRPC" error="{e}"') + raise TimeoutError("Failed to connect to Reva via GRPC") + self._gateway: cs3gw_grpc.GatewayAPIStub = cs3gw_grpc.GatewayAPIStub(self.channel) + self.auth = Auth(self._config, self._log, self._gateway) + self.file = File(self._config, self._log, self._gateway, self.auth) + + def _create_channel(self) -> grpc.Channel: + """ + create_channel creates a gRPC channel to the specified host. + + :return: gRPC channel to the specified host. + """ + + if self._config.ssl_enabled: + + credentials = grpc.ssl_channel_credentials( + root_certificates=self._config.ssl_ca_cert, + private_key=self._config.ssl_client_key, + certificate_chain=self._config.ssl_client_cert, + ) + channel = grpc.secure_channel(self._config.host, credentials) + else: + channel = grpc.insecure_channel(self._config.host) + return channel From d2acac0965a391f1b01b74a12dd8fbf4e6a870b6 Mon Sep 17 00:00:00 2001 From: Rasmus Oscar Welander Date: Tue, 30 Jul 2024 14:07:03 +0200 Subject: [PATCH 2/5] Added authentication and exceptions --- src/auth.py | 137 +++++++++++++++++++++++++++++++++++ src/exceptions/exceptions.py | 55 ++++++++++++++ 2 files changed, 192 insertions(+) create mode 100644 src/auth.py create mode 100644 src/exceptions/exceptions.py diff --git a/src/auth.py b/src/auth.py new file mode 100644 index 0000000..0a81551 --- /dev/null +++ b/src/auth.py @@ -0,0 +1,137 @@ +""" +auth.py + +Authors: Rasmus Welander, Diogo Castro, Giuseppe Lo Presti. +Emails: rasmus.oscar.welander@cern.ch, diogo.castro@cern.ch, giuseppe.lopresti@cern.ch +Last updated: 29/07/2024 +""" + +from typing import List +import grpc +import jwt +import datetime +import logging +import cs3.gateway.v1beta1.gateway_api_pb2 as gw +from cs3.auth.registry.v1beta1.registry_api_pb2 import ListAuthProvidersRequest +from cs3.gateway.v1beta1.gateway_api_pb2_grpc import GatewayAPIStub +from cs3.rpc.v1beta1.code_pb2 import CODE_OK + +from exceptions.exceptions import AuthenticationException, SecretNotSetException +from config import Config + + +class Auth: + """ + Auth class to handle authentication and token validation with CS3 Gateway API. + """ + + def __init__(self, config: Config, log: logging.Logger, gateway: GatewayAPIStub) -> None: + """ + Initializes the Auth class with configuration, logger, and gateway stub, + NOTE that token OR the client secret has to be set when instantiating the auth object. + + :param config: Config object containing the configuration parameters. + :param log: Logger instance for logging. + :param gateway: GatewayAPIStub instance for interacting with CS3 Gateway. + """ + self._gateway: GatewayAPIStub = gateway + self._log: logging.Logger = log + self._config: Config = config + # The user should be able to change the client secret (e.g. token) at runtime + self._client_secret: str | None = None + self._token: str | None = None + + def set_token(self, token: str) -> None: + """ + Should be used if the user wishes to set the reva token directly, instead of letting the client + exchange credentials for the token. NOTE that token OR the client secret has to be set when + instantiating the client object. + + :param token: The reva token. + """ + self._token = token + + def set_client_secret(self, token: str) -> None: + """ + Sets the client secret, exists so that the user can change the client secret (e.g. token, password) at runtime, + without having to create a new Auth object. NOTE that token OR the client secret has to be set when + instantiating the client object. + + :param token: Auth token/password. + """ + self._client_secret = token + + def get_token(self) -> tuple[str, str]: + """ + Attempts to get a valid authentication token. If the token is not valid, a new token is requested + if the client secret is set, if only the token is set then an exception will be thrown stating that + the credentials have expired. + + :return tuple: A tuple containing the header key and the token. + May throw AuthenticationException (token expired, or failed to authenticate) + or SecretNotSetException (neither token or client secret was set). + """ + + if not Auth._check_token(self._token): + # Check that client secret or token is set + if not self._client_secret and not self._token: + self._log.error("Attempted to authenticate, neither client secret or token was set.") + raise SecretNotSetException("") + elif not self._client_secret and self._token: + # Case where ONLY a token is provided but it has expired + self._log.error("The provided token have expired") + raise AuthenticationException("The credentials have expired") + # Create an authentication request + req = gw.AuthenticateRequest( + type=self._config.auth_login_type, + client_id=self._config.auth_client_id, + client_secret=self._client_secret, + ) + # Send the authentication request to the CS3 Gateway + res = self._gateway.Authenticate(req) + + if res.status.code != CODE_OK: + self._log.error( + f"Failed to authenticate user {self._config.auth_client_id}, error: {res.status.message}" + ) + raise AuthenticationException( + f"Failed to authenticate user {self._config.auth_client_id}, error: {res.status.message}" + ) + self._token = res.token + return ("x-access-token", self._token) + + def list_auth_providers(self) -> List[str]: + """ + list authentication providers + + :return: a list of the supported authentication types + May return ConnectionError (Could not connect to host) + """ + try: + res = self._gateway.ListAuthProviders(request=ListAuthProvidersRequest()) + if res.status.code != CODE_OK: + self._log.error(f"List auth providers request failed, error: {res.status.message}") + raise Exception(res.status.message) + except grpc.RpcError as e: + self._log.error("List auth providers request failed") + raise ConnectionError(e) + return res.types + + @classmethod + def _check_token(cls, token: str) -> bool: + """ + Checks if the given token is set and valid. + + :param token: JWT token as a string. + :return: True if the token is valid, False otherwise. + """ + if not token: + return False + # Decode the token without verifying the signature + decoded_token = jwt.decode(jwt=token, algorithms=["HS256"], options={"verify_signature": False}) + now = datetime.datetime.now().timestamp() + token_expiration = decoded_token.get("exp") + if token_expiration and now > token_expiration: + return False + + return True diff --git a/src/exceptions/exceptions.py b/src/exceptions/exceptions.py new file mode 100644 index 0000000..ff140e1 --- /dev/null +++ b/src/exceptions/exceptions.py @@ -0,0 +1,55 @@ +""" +exceptions.py + +Custom exception classes for the CS3 client. + +Authors: Rasmus Welander, Diogo Castro, Giuseppe Lo Presti. +Emails: rasmus.oscar.welander@cern.ch, diogo.castro@cern.ch, giuseppe.lopresti@cern.ch +Last updated: 29/07/2024 +""" + + +class AuthenticationException(Exception): + """ + Standard error thrown when attempting an operation without the required access rights + """ + + def __init__(self, message: str = ""): + super().__init__("Operation not permitted" + " " + message) + + +class NotFoundException(IOError): + """ + Standard file missing message + """ + + def __init__(self, message: str = ""): + super().__init__("No such file or directory" + " " + message) + + +class SecretNotSetException(Exception): + """ + Standard file missing message + """ + + def __init__(self, message: str = ""): + super().__init__("The client secret (e.g. token, passowrd) is not set" + " " + message) + + +class FileLockedException(IOError): + """ + Standard error thrown when attempting to overwrite a file/xattr in O_EXCL mode + or when a lock operation cannot be performed because of failed preconditions + """ + + def __init__(self, message: str = ""): + super().__init__("File/xattr exists but EXCL mode requested, lock mismatch or lock expired" + " " + message) + + +class UnknownException(Exception): + """ + Standard exception to be thrown when we get an error that is unknown, e.g. not defined in the cs3api + """ + + def __init__(self, message: str = ""): + super().__init__(message) From 294fbd40d6fff1ef18ec0ba8b3342e840bae7da7 Mon Sep 17 00:00:00 2001 From: Rasmus Oscar Welander Date: Tue, 30 Jul 2024 14:30:06 +0200 Subject: [PATCH 3/5] Added test fixtures, test dependencies and tests for the cs3client class --- tests/fixtures.py | 159 ++++++++++++++++++++++++++++++++++++++++ tests/requirements.txt | 4 + tests/test_cs3client.py | 94 ++++++++++++++++++++++++ 3 files changed, 257 insertions(+) create mode 100644 tests/fixtures.py create mode 100644 tests/requirements.txt create mode 100644 tests/test_cs3client.py diff --git a/tests/fixtures.py b/tests/fixtures.py new file mode 100644 index 0000000..6878d9a --- /dev/null +++ b/tests/fixtures.py @@ -0,0 +1,159 @@ +""" +fixtures.py + +Contains the fixtures used in the tests. + +Authors: Rasmus Welander, Diogo Castro, Giuseppe Lo Presti. +Emails: rasmus.oscar.welander@cern.ch, diogo.castro@cern.ch, giuseppe.lopresti@cern.ch +Last updated: 26/07/2024 + +""" + +import pytest +from unittest.mock import Mock, patch +from configparser import ConfigParser +import cs3.rpc.v1beta1.code_pb2 as cs3code +from cs3client import CS3Client +from file import File +from auth import Auth +from config import Config +import base64 +import json + + +@pytest.fixture +def mock_config(): + config = ConfigParser() + config["cs3client"] = { + # client parameters + "host": "test_host:port", + "grpc_timeout": "10", + "chunk_size": "4194304", + "http_timeout": "10", + # TUS parameters + "tus_enabled": "False", + # Authentication parameters + "auth_login_type": "basic", + "auth_client_id": "einstein", + # SSL parameters + "ssl_enabled": "True", + "ssl_verify": "True", + "ssl_ca_cert": "test_ca_cert", + "ssl_client_key": "test_client_key", + "ssl_client_cert": "test_client_cert", + # Lock parameters + "lock_not_impl": "False", + "lock_by_setting_attr": "False", + "lock_expiration": "1800", + } + return config + + +def create_mock_jwt(): + header = base64.urlsafe_b64encode(json.dumps({"alg": "HS256", "typ": "JWT"}).encode()).decode().strip("=") + payload = ( + base64.urlsafe_b64encode(json.dumps({"sub": "1234567890", "name": "John Doe", "iat": 1516239022}).encode()) + .decode() + .strip("=") + ) + signature = base64.urlsafe_b64encode(b"signature").decode().strip("=") + return f"{header}.{payload}.{signature}" + + +@pytest.fixture +def mock_logger(): + return Mock() + + +# Here the order of patches correspond to the parameters of the function +@pytest.fixture +@patch("cs3.gateway.v1beta1.gateway_api_pb2_grpc.GatewayAPIStub") +def mock_gateway(mock_gateway_stub_class): + mock_gateway_stub = Mock() + mock_gateway_stub_class.return_value = mock_gateway_stub + # Set up mock response for Authenticate method + mocked_token = create_mock_jwt() + mock_authenticate_response = Mock() + mock_authenticate_response.status.code = cs3code.CODE_OK + mock_authenticate_response.status.message = "" + mock_authenticate_response.token = mocked_token + mock_gateway_stub.Authenticate.return_value = mock_authenticate_response + return mock_gateway_stub + + +# All the parameters are inferred by pytest from existing fixtures +@pytest.fixture +def mock_authentication(mock_gateway, mock_config, mock_logger): + # Set up mock response for Authenticate method + mock_authentication = Auth(Config(mock_config, "cs3client"), mock_logger, mock_gateway) + mock_authentication.set_client_secret("test") + return mock_authentication + + +# Here the order of patches correspond to the parameters of the function +# (patches are applied from the bottom up) +# and the last two parameters are inferred by pytest from existing fixtures +@pytest.fixture +@patch("cs3client.grpc.secure_channel", autospec=True) +@patch("cs3client.grpc.channel_ready_future", autospec=True) +@patch("cs3client.grpc.insecure_channel", autospec=True) +@patch("cs3client.cs3gw_grpc.GatewayAPIStub", autospec=True) +@patch("cs3client.grpc.ssl_channel_credentials", autospec=True) +def cs3_client_secure( + mock_ssl_channel_credentials, + mock_gateway_stub_class, + mock_insecure_channel, + mock_channel_ready_future, + mock_secure_channel, + mock_config, + mock_logger, +): + + # Create CS3Client instance + client = CS3Client(mock_config, "cs3client", mock_logger) + client.auth.set_client_secret("test") + + assert mock_secure_channel.called + assert mock_channel_ready_future.called + assert mock_ssl_channel_credentials.called + assert mock_insecure_channel.assert_not_called + + return client + + +# Here the order of patches correspond to the parameters of the function +# (patches are applied from the bottom up) +# and the last two parameters are inferred by pytest from existing fixtures +@pytest.fixture +@patch("cs3client.grpc.secure_channel") +@patch("cs3client.grpc.insecure_channel") +@patch("cs3client.grpc.channel_ready_future") +@patch("cs3client.cs3gw_grpc.GatewayAPIStub") +@patch("cs3client.grpc.ssl_channel_credentials") +def cs3_client_insecure( + mock_ssl_channel_credentials, + mock_gateway_stub_class, + mock_channel_ready_future, + mock_insecure_channel, + mock_secure_channel, + mock_config, + mock_logger, +): + mock_config["cs3client"]["ssl_enabled"] = "False" + + # Create CS3Client instance + client = CS3Client(mock_config, "cs3client", mock_logger) + client.auth.set_client_secret("test") + + assert mock_insecure_channel.called + assert mock_channel_ready_future.called + assert mock_secure_channel.assert_not_called + assert mock_ssl_channel_credentials.assert_not_called + return client + + +# All parameters are inferred by pytest from existing fixtures +@pytest.fixture +def file_instance(mock_authentication, mock_gateway, mock_config, mock_logger): + file = File(Config(mock_config, "cs3client"), mock_logger, mock_gateway, mock_authentication) + return file diff --git a/tests/requirements.txt b/tests/requirements.txt new file mode 100644 index 0000000..a0c5bc5 --- /dev/null +++ b/tests/requirements.txt @@ -0,0 +1,4 @@ +pytest +pytest-cov +pytest-mock +coverage \ No newline at end of file diff --git a/tests/test_cs3client.py b/tests/test_cs3client.py new file mode 100644 index 0000000..a8f157d --- /dev/null +++ b/tests/test_cs3client.py @@ -0,0 +1,94 @@ +""" +test_cs3client.py + +Tests the initialization of the CS3Client class. + +Authors: Rasmus Welander, Diogo Castro, Giuseppe Lo Presti. +Emails: rasmus.oscar.welander@cern.ch, diogo.castro@cern.ch, giuseppe.lopresti@cern.ch +Last updated: 26/07/2024 +""" + +from fixtures import ( # noqa: F401 (they are used, the framework is not detecting it) + cs3_client_insecure, + cs3_client_secure, + mock_config, + mock_logger, + mock_gateway, + create_mock_jwt, +) + + +def test_cs3client_initialization_secure(cs3_client_secure): # noqa: F811 (not a redefinition) + client = cs3_client_secure + + # Make sure configuration is correctly set + assert client._config.host == "test_host:port" + assert client._config.grpc_timeout == 10 + assert client._config.http_timeout == 10 + assert client._config.chunk_size == 4194304 + assert client._config.auth_login_type == "basic" + assert client._config.auth_client_id == "einstein" + assert client._config.tus_enabled is False + assert client._config.ssl_enabled is True + assert client._config.ssl_verify is True + assert client._config.ssl_client_cert == "test_client_cert" + assert client._config.ssl_client_key == "test_client_key" + assert client._config.ssl_ca_cert == "test_ca_cert" + assert client._config.lock_by_setting_attr is False + assert client._config.lock_not_impl is False + assert client._config.lock_expiration == 1800 + + # Make sure the gRPC channel is correctly created + assert client.channel is not None + assert client._gateway is not None + assert client.auth is not None + assert client.file is not None + + # Make sure auth objects are correctly set + assert client.auth._gateway is not None + assert client.auth._config is not None + assert client.auth._log is not None + + # Make sure file objects are correctly set + assert client.file._gateway is not None + assert client.file._auth is not None + assert client.file._config is not None + assert client.file._log is not None + + +def test_cs3client_initialization_insecure(cs3_client_insecure): # noqa: F811 (not a redefinition) + client = cs3_client_insecure + + # Make sure configuration is correctly set + assert client._config.host == "test_host:port" + assert client._config.grpc_timeout == 10 + assert client._config.http_timeout == 10 + assert client._config.chunk_size == 4194304 + assert client._config.auth_login_type == "basic" + assert client._config.auth_client_id == "einstein" + assert client._config.tus_enabled is False + assert client._config.ssl_enabled is False + assert client._config.ssl_verify is True + assert client._config.ssl_client_cert == "test_client_cert" + assert client._config.ssl_client_key == "test_client_key" + assert client._config.ssl_ca_cert == "test_ca_cert" + assert client._config.lock_by_setting_attr is False + assert client._config.lock_not_impl is False + assert client._config.lock_expiration == 1800 + + # Make sure the gRPC channel is correctly created + assert client.channel is not None + assert client._gateway is not None + assert client.auth is not None + assert client.file is not None + + # Make sure auth objects are correctly set + assert client.auth._gateway is not None + assert client.auth._config is not None + assert client.auth._log is not None + + # Make sure file objects are correctly set + assert client.file._gateway is not None + assert client.file._auth is not None + assert client.file._config is not None + assert client.file._log is not None From 3f9c3a2c709dcd7ac86b1410bda0b06cedb192d0 Mon Sep 17 00:00:00 2001 From: Rasmus Oscar Welander Date: Tue, 30 Jul 2024 14:32:00 +0200 Subject: [PATCH 4/5] Added file class, the resource class and an example --- examples/file_api_example.py | 125 ++++++++++++ src/cs3resource.py | 204 ++++++++++++++++++++ src/file.py | 362 +++++++++++++++++++++++++++++++++++ 3 files changed, 691 insertions(+) create mode 100644 examples/file_api_example.py create mode 100644 src/cs3resource.py create mode 100644 src/file.py diff --git a/examples/file_api_example.py b/examples/file_api_example.py new file mode 100644 index 0000000..d46f254 --- /dev/null +++ b/examples/file_api_example.py @@ -0,0 +1,125 @@ +""" +example.py + +Example script to demonstrate the usage of the CS3Client class. +Start with an empty directory and you should end up with a directory structure like this: + +test_directory1 +test_directory2 +test_directory3 +rename_file.txt (containing "Hello World") +text_file.txt + + +Authors: Rasmus Welander, Diogo Castro, Giuseppe Lo Presti. +Emails: rasmus.oscar.welander@cern.ch, diogo.castro@cern.ch, giuseppe.lopresti@cern.ch +Last updated: 29/07/2024 +""" + +import logging +import configparser +from cs3client import CS3Client +from cs3resource import Resource + +config = configparser.ConfigParser() +with open("default.conf") as fdef: + config.read_file(fdef) +# log +log = logging.getLogger(__name__) + +client = CS3Client(config, "cs3client", log) +client.auth.set_client_secret("relativity") + +# Authentication +print(client.auth.get_token()) + +res = None + +# mkdir +for i in range(1, 4): + directory_resource = Resource.from_file_ref_and_endpoint(f"/eos/user/r/rwelande/test_directory{i}") + res = client.file.make_dir(directory_resource) + if res is not None: + print(res) + +# touchfile +touch_resource = Resource.from_file_ref_and_endpoint("/eos/user/r/rwelande/touch_file.txt") +text_resource = Resource.from_file_ref_and_endpoint("/eos/user/r/rwelande/text_file.txt") +res = client.file.touch_file(touch_resource) +res = client.file.touch_file(text_resource) + +if res is not None: + print(res) + +# setxattr +resource = Resource.from_file_ref_and_endpoint("/eos/user/r/rwelande/text_file.txt") +res = client.file.set_xattr(resource, "iop.wopi.lastwritetime", str(1720696124)) + +if res is not None: + print(res) + +# rmxattr +res = client.file.remove_xattr(resource, "iop.wopi.lastwritetime") + +if res is not None: + print(res) + +# stat +res = client.file.stat(text_resource) + +if res is not None: + print(res) + +# removefile +res = client.file.remove_file(touch_resource) + +if res is not None: + print(res) + +res = client.file.touch_file(touch_resource) + +# rename +rename_resource = Resource.from_file_ref_and_endpoint("/eos/user/r/rwelande/rename_file.txt") +res = client.file.rename_file(resource, rename_resource) + +if res is not None: + print(res) + +# writefile +content = b"Hello World" +size = len(content) +res = client.file.write_file(rename_resource, content, size) + +if res is not None: + print(res) + +# rmdir (same as deletefile) +res = client.file.remove_file(directory_resource) + +if res is not None: + print(res) + +# listdir +list_directory_resource = Resource.from_file_ref_and_endpoint("/eos/user/r/rwelande") +res = client.file.list_dir(list_directory_resource) + +first_item = next(res, None) +if first_item is not None: + print(first_item) + for item in res: + print(item) +else: + print("empty response") + +# readfile +file_res = client.file.read_file(rename_resource) +content = b"" +try: + for chunk in file_res: + if isinstance(chunk, Exception): + raise chunk + content += chunk + print(content.decode("utf-8")) +except Exception as e: + print(f"An error occurred: {e}") + print(e) diff --git a/src/cs3resource.py b/src/cs3resource.py new file mode 100644 index 0000000..e2b458b --- /dev/null +++ b/src/cs3resource.py @@ -0,0 +1,204 @@ +""" +cs3resource.py + +Authors: Rasmus Welander, Diogo Castro, Giuseppe Lo Presti. +Emails: rasmus.oscar.welander@cern.ch, diogo.castro@cern.ch, giuseppe.lopresti@cern.ch +Last updated: 29/07/2024 +""" + +import cs3.storage.provider.v1beta1.resources_pb2 as cs3spr + + +class Resource: + """Class to handle CS3 resources, the class can be initialized with an absolute path, + a relative path or an opaque fileid as the "file" parameter (required). + + Absolute path example: `/path/to/file` + Relative path example: `/` or `///` + Opaque fileid example: `` + + The endpoint attribute contains the storage_id and space_id (always optional) separated by a `$` character, + this is optional if the file is an absolute path. + + endpoint example: `storage_id` or `storage_id$space_id` + + """ + + def __init__( + self, + abs_path: str | None = None, + rel_path: str | None = None, + opaque_id: str | None = None, + parent_id: str | None = None, + storage_id: str | None = None, + space_id: str | None = None, + ) -> None: + """ + initializes the Resource class, either abs_path, rel_path or opaque_id is required + and with rel_path the parent_id is also required, the rest parameters are fully optional. + + + :param abs_path: absolute path (semi-optional) + :param rel_path: relative path (semi-optional) + :param parent_id: parent id (semi-optional) + :param opaque_id: opaque id (semi-optional) + :param storage_id: storage id (optional) + :param space_id: space id (optional) + """ + self._abs_path: str | None = abs_path + self._rel_path: str | None = rel_path + self._parent_id: str | None = parent_id + self._opaque_id: str | None = opaque_id + self._space_id: str | None = space_id + self._storage_id: str | None = storage_id + + @classmethod + def from_file_ref_and_endpoint(cls, file: str, endpoint: str | None = None) -> "Resource": + """ + Extracts the attributes from the file and endpoint and returns a resource. + + :param file: The file reference + :param endpoint: The storage id and space id (optional) + :return: Resource object + """ + + abs_path = None + rel_path = None + opaque_id = None + parent_id = None + storage_id = None + space_id = None + + if file.startswith("/"): + # assume we have an absolute path + abs_path = file + else: + # try splitting endpoint + parts = endpoint.split("$", 2) + storage_id = parts[0] + if len(parts) == 2: + space_id = parts[1] + if file.find("/") > 0: + # assume we have an relative path, + parent_id = file[: file.find("/")] + rel_path = file[file.find("/") :] + else: + # assume we have an opaque fileid + opaque_id = file + return cls(abs_path, rel_path, opaque_id, parent_id, storage_id, space_id) + + @property + def ref(self) -> cs3spr.Reference: + """ + Generates a CS3 reference for a given resource, covering the following cases: + absolute path, relative hybrid path, fully opaque fileid. + + :return: The cs3 reference. + May throw ValueError (Invalid Resource) + """ + if self._abs_path: + return cs3spr.Reference(path=self._abs_path) + if self._rel_path: + return cs3spr.Reference( + resource_id=cs3spr.ResourceId( + storage_id=self._storage_id, + space_id=self._space_id, + opaque_id=self._parent_id, + ), + path="." + self._rel_path, + ) + if self._opaque_id: + return cs3spr.Reference( + resource_id=cs3spr.ResourceId( + storage_id=self._storage_id, + space_id=self._space_id, + opaque_id=self._opaque_id, + ), + path=".", + ) + raise ValueError("Invalid Resource") + + def recreate_endpoint_and_file(self) -> dict: + """ + Recreates the endpoint and file reference from the given resource + + :return: (dict) {"file": fileref, "endpoint": endpoint} + May throw ValueError (invalid resource) + """ + endpoint = self._storage_id + if self._space_id: + endpoint += f"${self._space_id}" + if self._abs_path: + return {"file": self._abs_path, "endpoint": endpoint} + if self._parent_id and self._rel_path: + return {"file": f"{self._parent_id}{self._rel_path}", "endpoint": endpoint} + if self._opaque_id: + return {"file": self._opaque_id, "endpoint": endpoint} + raise ValueError("Invalid Resource") + + @classmethod + def from_cs3_ref(cls, reference: cs3spr.Reference) -> "Resource": + """ + Alternate constructor that reverses a CS3 reference to obtain a resource.ß + + :param reference: The CS3 reference. + :return: Resource object. + May throw ValueError (Invalid reference) + """ + rel_path = None + opaque_id = None + parent_id = None + storage_id = None + space_id = None + + if reference.path and reference.path.startswith("/"): + # It's an absolute path, we can return straight away + return Resource(abs_path=reference.path) + elif reference.resource_id and reference.resource_id.storage_id: + storage_id = reference.resource_id.storage_id + if reference.resource_id.space_id: + space_id = reference.resource_id.space_id + if reference.path and len(reference.path) > 1: + # It's a relative path (remove the "." in the relative path) + rel_path = reference.path[1:] + # The opaque_id is a parent id since it's a relative pathß + parent_id = reference.resource_id.opaque_id + else: + opaque_id = reference.resource_id.opaque_id + return Resource( + abs_path=None, + rel_path=rel_path, + opaque_id=opaque_id, + parent_id=parent_id, + storage_id=storage_id, + space_id=space_id, + ) + raise ValueError("Invalid CS3 reference") + + # It is possible that the same resource is different if abs_path is used in one + # and the other is using opaque_id for example. + def __eq__(self, other): + """redefine the equality operator to compare two resources""" + if isinstance(other, Resource): + return ( + self._abs_path == other._abs_path + and self._rel_path == other._rel_path + and self._parent_id == other._parent_id + and self._opaque_id == other._opaque_id + and self._space_id == other._space_id + and self._storage_id == other._storage_id + ) + return False + + def get_file_ref_str(self): + """ + Generates a string from the file ref, '="fileref">' + + :return: str '="fileref">' + """ + if self._abs_path: + return f'absolute_path="{self._abs_path}"' + elif self._rel_path: + return f'relative_path="{self._parent_id}/{self._rel_path}"' + elif self._opaque_id: + return f'opaque_id="{self._opaque_id}"' diff --git a/src/file.py b/src/file.py new file mode 100644 index 0000000..8278f5b --- /dev/null +++ b/src/file.py @@ -0,0 +1,362 @@ +""" +file.py + +Authors: Rasmus Welander, Diogo Castro, Giuseppe Lo Presti. +Emails: rasmus.oscar.welander@cern.ch, diogo.castro@cern.ch, giuseppe.lopresti@cern.ch +Last updated: 29/07/2024 +""" + +import time +import logging +import http +import requests +import cs3.storage.provider.v1beta1.resources_pb2 as cs3spr +import cs3.storage.provider.v1beta1.provider_api_pb2 as cs3sp +from cs3.gateway.v1beta1.gateway_api_pb2_grpc import GatewayAPIStub +import cs3.rpc.v1beta1.code_pb2 as cs3code +import cs3.types.v1beta1.types_pb2 as types + +from config import Config +from typing import Generator +from exceptions.exceptions import AuthenticationException, FileLockedException, NotFoundException, UnknownException +from cs3resource import Resource +from auth import Auth + + +class File: + """ + File class to interact with the CS3 API. + """ + + def __init__(self, config: Config, log: logging.Logger, gateway: GatewayAPIStub, auth: Auth) -> None: + self._auth: Auth = auth + self._config: Config = config + self._log: logging.Logger = log + self._gateway: GatewayAPIStub = gateway + + # Note that res is of type any because it can be different types of respones + # depending on the method that calls this function, I do not think importing + # all the possible response types is a good idea + def _log_not_found_info(self, resource: Resource, res: any, operation: str) -> None: + self._log.info( + f'msg="File not found on {operation}" {resource.get_file_ref_str()} ' + f'userid="{self._config.auth_client_id}" trace="{res.status.trace}" ' + f'reason="{res.status.message.replace('"', "'")}"' + ) + + def _log_precondition_info(self, resource: Resource, res: any, operation: str) -> None: + self._log.info( + f'msg="Failed precondition on {operation}" {resource.get_file_ref_str()} ' + f'userid="{self._config.auth_client_id}" trace="{res.status.trace}"' + f'reason="{res.status.message.replace('"', "'")}"' + ) + + def _log_authentication_error(self, resource: Resource, res: any, operation: str) -> None: + self._log.error( + f'msg="Authentication failed on {operation}" {resource.get_file_ref_str()} ' + f'userid="{self._config.auth_client_id}" trace="{res.status.trace}"' + f'reason="{res.status.message.replace('"', "'")}"' + ) + + def _log_unknown_error(self, resource: Resource, res: any, operation: str) -> None: + self._log.error( + f'msg="Failed to {operation}, unknown error" {resource.get_file_ref_str()} ' + f'userid="{self._config.auth_client_id}" trace="{res.status.trace}"' + f'reason="{res.status.message.replace('"', "'")}"' + ) + + def _handle_errors(self, resource: Resource, res: any, operation: str, msg: str = "") -> None: + if res.status.code == cs3code.CODE_NOT_FOUND: + self._log_not_found_info(resource, res, operation) + self._log.info(f'msg="Invoked {operation} on missing file" {resource.get_file_ref_str()}') + raise NotFoundException(message=msg) + if res.status.code in [cs3code.CODE_FAILED_PRECONDITION, cs3code.CODE_ABORTED]: + self._log_precondition_info(resource, res, operation) + raise FileLockedException(message=msg) + if res.status.code == cs3code.CODE_UNAUTHENTICATED: + self._log_authentication_error(resource, res, operation) + raise AuthenticationException(message=msg) + if res.status.code != cs3code.CODE_OK: + if "path not found" in str(res): + self._log.info(f'msg="Invoked {operation} on missing file" {resource.get_file_ref_str()}') + raise NotFoundException(message=msg) + self._log_unknown_error(resource, res, operation) + raise UnknownException(res.status.message) + + def stat(self, resource: Resource) -> cs3spr.ResourceInfo: + """ + Stat a file and return the ResourceInfo object. + + :param resource: resource to stat. + :return: cs3.storage.provider.v1beta1.resources_pb2.ResourceInfo (success) + NotFoundException (File not found), + AuthenticationException (Authentication Failed), + or UnknownException (Unknown Error). + + """ + tstart = time.time() + res = self._gateway.Stat(request=cs3sp.StatRequest(ref=resource.ref), metadata=[self._auth.get_token()]) + tend = time.time() + self._handle_errors(resource, res, "stat") + self._log.debug( + f'msg="Invoked Stat" {resource.get_file_ref_str()} elapsedTimems="{(tend - tstart) * 1000:.1f}"' + ) + return res.info + + def set_xattr(self, resource: Resource, key: str, value: str) -> None: + """ + Set the extended attribute to for a resource. + + :param resource: resource that has the attribute. + :param key: attribute key. + :param value: value to set. + :return: None (Success) + May return FileLockedException (File is locked), + AuthenticationException (Authentication Failed), + or UnknownException (Unknown error). + """ + md = cs3spr.ArbitraryMetadata() + md.metadata.update({key: value}) # pylint: disable=no-member + req = cs3sp.SetArbitraryMetadataRequest(ref=resource.ref, arbitrary_metadata=md) + res = self._gateway.SetArbitraryMetadata(request=req, metadata=[self._auth.get_token()]) + # CS3 storages may refuse to set an xattr in case of lock mismatch: this is an overprotection, + # as the lock should concern the file's content, not its metadata, however we need to handle that + self._handle_errors(resource, res, "set extended attribute") + self._log.debug(f'msg="Invoked setxattr" result="{res}"') + + def remove_xattr(self, resource: Resource, key: str) -> None: + """ + Remove the extended attribute . + + :param resource: cs3client resource. + :param key: key for attribute to remove. + :return: None (Success) + May return FileLockedException (File is locked), + AuthenticationException (Authentication failed) or + UnknownException (Unknown error). + """ + req = cs3sp.UnsetArbitraryMetadataRequest(ref=resource.ref, arbitrary_metadata_keys=[key]) + res = self._gateway.UnsetArbitraryMetadata(request=req, metadata=[self._auth.get_token()]) + self._handle_errors(resource, res, "remove extended attribute") + self._log.debug(f'msg="Invoked rmxattr" result="{res.status}"') + + def rename_file(self, resource: Resource, newresource: Resource) -> None: + """ + Rename/move resource to new resource. + + :param resource: Original resource. + :param newresource: New resource. + :return: None (Success) + May return NotFoundException (Original resource not found), + FileLockException (Resource is locked), + AuthenticationException (Authentication Failed), + UnknownException (Unknown Error). + """ + req = cs3sp.MoveRequest(source=resource.ref, destination=newresource.ref) + res = self._gateway.Move(request=req, metadata=[self._auth.get_token()]) + self._handle_errors(resource, res, "rename_file") + self._log.debug(f'msg="Invoked renamefile" result="{res}"') + + def remove_file(self, resource: Resource) -> None: + """ + Remove a resource. + + :param resource: Resource to remove. + :return: None (Success) + May return AuthenticationException (Authentication Failed), + NotFoundException (Resource not found) or + UnknownException (Unknown error). + """ + req = cs3sp.DeleteRequest(ref=resource.ref) + res = self._gateway.Delete(request=req, metadata=[self._auth.get_token()]) + self._handle_errors(resource, res, "remove file") + self._log.debug(f'msg="Invoked removefile" result="{res}"') + + def touch_file(self, resource: Resource) -> None: + """ + Create a resource. + + :param resource: Resource to create. + :return: None (Success) + May return FileLockedException (File is locked), + AuthenticationException (Authentication Failed) or + UnknownException (Unknown error) + """ + req = cs3sp.TouchFileRequest( + ref=resource.ref, + opaque=types.Opaque(map={"Upload-Length": types.OpaqueEntry(decoder="plain", value=str.encode("0"))}), + ) + res = self._gateway.TouchFile(request=req, metadata=[self._auth.get_token()]) + self._handle_errors(resource, res, "touch file") + self._log.debug(f'msg="Invoked touchfile" result="{res}"') + + def write_file(self, resource: Resource, content: str | bytes, size: int) -> None: + """ + Write a file using the given userid as access token. The entire content is written + and any pre-existing file is deleted (or moved to the previous version if supported), + writing a file with size 0 is equivalent to "touch file" and should be used if the + implementation does not support touchfile. + + :param resource: Resource to write content to. + :param content: content to write + :param size: size of content (optional) + :return: None (Success) + May return FileLockedException (File is locked), + AuthenticationException (Authentication failed) or + UnknownException (Unknown error), + + """ + tstart = time.time() + # prepare endpoint + if size == -1: + if isinstance(content, str): + content = bytes(content, "UTF-8") + size = len(content) + req = cs3sp.InitiateFileUploadRequest( + ref=resource.ref, + opaque=types.Opaque(map={"Upload-Length": types.OpaqueEntry(decoder="plain", value=str.encode(str(size)))}), + ) + res = self._gateway.InitiateFileUpload(request=req, metadata=[self._auth.get_token()]) + self._handle_errors(resource, res, "write file") + tend = time.time() + self._log.debug( + f'msg="writefile: InitiateFileUploadRes returned" trace="{res.status.trace}" protocols="{res.protocols}"' + ) + + # Upload + try: + protocol = [p for p in res.protocols if p.protocol in ["simple", "spaces"]][0] + if self._config.tus_enabled: + headers = { + "Tus-Resumable": "1.0.0", + "File-Path": resource.file, + "File-Size": str(size), + "X-Reva-Transfer": protocol.token, + **dict([self._auth.get_token()]), + } + else: + headers = { + "Upload-Length": str(size), + "X-Reva-Transfer": protocol.token, + **dict([self._auth.get_token()]), + } + putres = requests.put( + url=protocol.upload_endpoint, + data=content, + headers=headers, + verify=self._config.ssl_verify, + timeout=self._config.http_timeout, + ) + except requests.exceptions.RequestException as e: + self._log.error(f'msg="Exception when uploading file to Reva" reason="{e}"') + raise IOError(e) from e + if putres.status_code == http.client.CONFLICT: + self._log.info( + f'msg="Got conflict on PUT, file is locked" reason="{putres.reason}" {resource.get_file_ref_str()}' + ) + raise FileLockedException() + if putres.status_code == http.client.UNAUTHORIZED: + self._log_authentication_error(resource, putres, "write") + raise AuthenticationException() + if putres.status_code != http.client.OK: + if ( + size == 0 + ): # 0-byte file uploads may have been finalized after InitiateFileUploadRequest, let's assume it's OK + # Should use TouchFileRequest instead + self._log.info( + f'msg="0-byte file written successfully" {resource.get_file_ref_str()} ' + f' elapsedTimems="{(tend - tstart) * 1000:.1f}"' + ) + return + + self._log.error( + f'msg="Error uploading file to Reva" code="{putres.status_code}" reason="{putres.reason}"' + ) + raise IOError(putres.reason) + self._log.info( + f'msg="File written successfully" {resource.get_file_ref_str()} ' + f'elapsedTimems="{(tend - tstart) * 1000:.1f}"' + ) + + def read_file(self, resource: Resource) -> Generator[bytes, None, None]: + """ + Read a file. Note that the function is a generator, managed by the app server. + + :param resource: Resource to read. + :return: Generator[Bytes, None, None] (Success) + May return NotFoundException (Resource not found), + AuthenticationException (Authentication Failed) or + UnknownException (Unknown Error). + """ + tstart = time.time() + + # prepare endpoint + req = cs3sp.InitiateFileDownloadRequest(ref=resource.ref) + res = self._gateway.InitiateFileDownload(request=req, metadata=[self._auth.get_token()]) + self._handle_errors(resource, res, "read file") + tend = time.time() + self._log.debug( + f'msg="readfile: InitiateFileDownloadRes returned" trace="{res.status.trace}" protocols="{res.protocols}"' + ) + + # Download + try: + protocol = [p for p in res.protocols if p.protocol in ["simple", "spaces"]][0] + headers = {"X-Reva-Transfer": protocol.token, **dict([self._auth.get_token()])} + fileget = requests.get( + url=protocol.download_endpoint, + headers=headers, + verify=self._config.ssl_verify, + timeout=self._config.http_timeout, + stream=True, + ) + except requests.exceptions.RequestException as e: + self._log.error(f'msg="Exception when downloading file from Reva" reason="{e}"') + raise IOError(e) + data = fileget.iter_content(self._config.chunk_size) + if fileget.status_code != http.client.OK: + self._log.error( + f'msg="Error downloading file from Reva" code="{fileget.status_code}" ' + f'reason="{fileget.reason.replace('"', "'")}"' + ) + raise IOError(fileget.reason) + else: + self._log.info( + f'msg="File open for read" {resource.get_file_ref_str()} elapsedTimems="{(tend - tstart) * 1000:.1f}"' + ) + for chunk in data: + yield chunk + + def make_dir(self, resource: Resource) -> None: + """ + Create a directory. + + :param resource: Direcotry to create. + :return: None (Success) + May return FileLockedException (File is locked), + AuthenticationException (Authentication failed) or + UnknownException (Unknown error). + """ + req = cs3sp.CreateContainerRequest(ref=resource.ref) + res = self._gateway.CreateContainer(request=req, metadata=[self._auth.get_token()]) + self._handle_errors(resource, res, "make directory") + self._log.debug(f'msg="Invoked CreateContainer" result="{res}"') + + def list_dir( + self, resource: Resource + ) -> Generator[cs3spr.ResourceInfo, None, None]: + """ + List the contents of a directory, note that the function is a generator. + + :param resource: the directory. + :return: Generator[cs3.storage.provider.v1beta1.resources_pb2.ResourceInfo, None, None] (Success) + May return NotFoundException (Resrouce not found), + AuthenticationException (Authentication Failed) or + UnknownException (Unknown error). + """ + req = cs3sp.ListContainerRequest(ref=resource.ref) + res = self._gateway.ListContainer(request=req, metadata=[self._auth.get_token()]) + self._handle_errors(resource, res, "list directory") + self._log.debug(f'msg="Invoked ListContainer" result="{res}"') + for info in res.infos: + yield info From 905790f83a2755a2944c1c7a6f332fa8887494c3 Mon Sep 17 00:00:00 2001 From: Rasmus Oscar Welander Date: Tue, 30 Jul 2024 14:32:42 +0200 Subject: [PATCH 5/5] Added tests for the file and resource classes --- tests/test_file.py | 577 +++++++++++++++++++++++++++++++++++++++++ tests/test_resource.py | 137 ++++++++++ 2 files changed, 714 insertions(+) create mode 100644 tests/test_file.py create mode 100644 tests/test_resource.py diff --git a/tests/test_file.py b/tests/test_file.py new file mode 100644 index 0000000..1c52545 --- /dev/null +++ b/tests/test_file.py @@ -0,0 +1,577 @@ +""" +test_file.py + +Tests that the File class methods work as expected. + +Authors: Rasmus Welander, Diogo Castro, Giuseppe Lo Presti. +Emails: rasmus.oscar.welander@cern.ch, diogo.castro@cern.ch, giuseppe.lopresti@cern.ch +Last updated: 26/07/2024 +""" + +from cs3resource import Resource +from exceptions.exceptions import ( + AuthenticationException, + NotFoundException, + FileLockedException, + UnknownException, +) +import cs3.rpc.v1beta1.code_pb2 as cs3code +from fixtures import ( # noqa: F401 (they are used, the framework is not detecting it) + cs3_client_insecure, + cs3_client_secure, + mock_config, + mock_logger, + mock_authentication, + mock_gateway, + create_mock_jwt, + file_instance, +) +from unittest.mock import Mock, patch +import pytest + + +# Test cases for File class + + +def test_stat(file_instance): # noqa: F811 (not a redefinition) + resource = Resource.from_file_ref_and_endpoint(endpoint="", file="testfile") + mock_response = Mock() + mock_response.status.code = cs3code.CODE_OK + mock_response.info = "resource_info" + + with patch.object(file_instance._gateway, "Stat", return_value=mock_response): + result = file_instance.stat(resource) + assert result == "resource_info" + + +def test_stat_not_found(file_instance): # noqa: F811 (not a redefinition) + resource = Resource.from_file_ref_and_endpoint(endpoint="", file="testfile") + mock_response = Mock() + mock_response.status.code = cs3code.CODE_NOT_FOUND + + with patch.object(file_instance._gateway, "Stat", return_value=mock_response): + with pytest.raises(NotFoundException): + file_instance.stat(resource) + + +def test_stat_unauthenticated(file_instance): # noqa: F811 (not a redefinition) + resource = Resource.from_file_ref_and_endpoint(endpoint="", file="testfile") + mock_response = Mock() + mock_response.status.code = cs3code.CODE_UNAUTHENTICATED + mock_response.status.message = "Authentication failed" + + with patch.object(file_instance._gateway, "Stat", return_value=mock_response): + with pytest.raises(AuthenticationException): + file_instance.stat(resource) + + +def test_stat_unknown_error(file_instance): # noqa: F811 (not a redefinition) + resource = Resource.from_file_ref_and_endpoint(endpoint="", file="testfile") + mock_response = Mock() + mock_response.status.code = "-1" + mock_response.status.message = "Internal error" + + with patch.object(file_instance._gateway, "Stat", return_value=mock_response): + with pytest.raises(UnknownException): + file_instance.stat(resource) + + +def test_setxattr(file_instance): # noqa: F811 (not a redefinition) + resource = Resource.from_file_ref_and_endpoint(endpoint="", file="testfile") + key = "testkey" + value = "testvalue" + mock_response = Mock() + mock_response.status.code = cs3code.CODE_OK + + with patch.object(file_instance._gateway, "SetArbitraryMetadata", return_value=mock_response): + file_instance.set_xattr(resource, key, value) + + +def test_setxattr_failed_precondition(file_instance): # noqa: F811 (not a redefinition) + resource = Resource.from_file_ref_and_endpoint(endpoint="", file="testfile") + key = "testkey" + value = "testvalue" + mock_response = Mock() + mock_response.status.code = cs3code.CODE_FAILED_PRECONDITION + mock_response.status.message = "Failed precondition" + + with patch.object(file_instance._gateway, "SetArbitraryMetadata", return_value=mock_response): + with pytest.raises(FileLockedException): + file_instance.set_xattr(resource, key, value) + + +def test_setxattr_failed_aborted(file_instance): # noqa: F811 (not a redefinition) + resource = Resource.from_file_ref_and_endpoint(endpoint="", file="testfile") + key = "testkey" + value = "testvalue" + mock_response = Mock() + mock_response.status.code = cs3code.CODE_ABORTED + mock_response.status.message = "Failed precondition" + + with patch.object(file_instance._gateway, "SetArbitraryMetadata", return_value=mock_response): + with pytest.raises(IOError): + file_instance.set_xattr(resource, key, value) + + +def test_setxattr_failed_unauthorized(file_instance): # noqa: F811 (not a redefinition) + resource = Resource.from_file_ref_and_endpoint(endpoint="", file="testfile") + key = "testkey" + value = "testvalue" + mock_response = Mock() + mock_response.status.code = cs3code.CODE_UNAUTHENTICATED + mock_response.status.message = "Failed to authenticate" + + with patch.object(file_instance._gateway, "SetArbitraryMetadata", return_value=mock_response): + with pytest.raises(AuthenticationException): + file_instance.set_xattr(resource, key, value) + + +def test_rmxattr(file_instance): # noqa: F811 (not a redefinition) + resource = Resource.from_file_ref_and_endpoint(endpoint="", file="testfile") + key = "testkey" + mock_response = Mock() + mock_response.status.code = cs3code.CODE_OK + + with patch.object(file_instance._gateway, "UnsetArbitraryMetadata", return_value=mock_response): + file_instance.remove_xattr(resource, key) + + +def test_rmxattr_failed_precondition(file_instance): # noqa: F811 (not a redefinition) + resource = Resource.from_file_ref_and_endpoint(endpoint="", file="testfile") + key = "testkey" + mock_response = Mock() + mock_response.status.code = cs3code.CODE_FAILED_PRECONDITION + mock_response.status.message = "Failed precondition" + + with patch.object(file_instance._gateway, "UnsetArbitraryMetadata", return_value=mock_response): + with pytest.raises(FileLockedException): + file_instance.remove_xattr(resource, key) + + +def test_rmxattr_failed_aborted(file_instance): # noqa: F811 (not a redefinition) + resource = Resource.from_file_ref_and_endpoint(endpoint="", file="testfile") + key = "testkey" + mock_response = Mock() + mock_response.status.code = cs3code.CODE_ABORTED + mock_response.status.message = "Failed aborted" + + with patch.object(file_instance._gateway, "UnsetArbitraryMetadata", return_value=mock_response): + with pytest.raises(IOError): + file_instance.remove_xattr(resource, key) + + +def test_rmxattr_failed_authentication(file_instance): # noqa: F811 (not a redefinition) + resource = Resource.from_file_ref_and_endpoint(endpoint="", file="testfile") + key = "testkey" + mock_response = Mock() + mock_response.status.code = cs3code.CODE_UNAUTHENTICATED + mock_response.status.message = "Authentication Failed" + + with patch.object(file_instance._gateway, "UnsetArbitraryMetadata", return_value=mock_response): + with pytest.raises(AuthenticationException): + file_instance.remove_xattr(resource, key) + + +def test_rmxattr_failed_unknown(file_instance): # noqa: F811 (not a redefinition) + resource = Resource.from_file_ref_and_endpoint(endpoint="", file="testfile") + key = "testkey" + mock_response = Mock() + mock_response.status.code = "-1" + mock_response.status.message = "Failed aborted" + + with patch.object(file_instance._gateway, "UnsetArbitraryMetadata", return_value=mock_response): + with pytest.raises(UnknownException): + file_instance.remove_xattr(resource, key) + + +def test_rename_file(file_instance): # noqa: F811 (not a redefinition) + resource = Resource.from_file_ref_and_endpoint(endpoint="", file="testfile") + newresource = Resource.from_file_ref_and_endpoint(endpoint="", file="newtestfile") + mock_response = Mock() + mock_response.status.code = cs3code.CODE_OK + + with patch.object(file_instance._gateway, "Move", return_value=mock_response): + file_instance.rename_file(resource, newresource) + + +def test_rename_file_failed_precondition(file_instance): # noqa: F811 (not a redefinition) + resource = Resource.from_file_ref_and_endpoint(endpoint="", file="testfile") + newresource = Resource.from_file_ref_and_endpoint(endpoint="", file="newtestfile") + mock_response = Mock() + mock_response.status.code = cs3code.CODE_FAILED_PRECONDITION + mock_response.status.message = "Failed precondition" + + with patch.object(file_instance._gateway, "Move", return_value=mock_response): + with pytest.raises(FileLockedException): + file_instance.rename_file(resource, newresource) + + +def test_rename_file_failed_aborted(file_instance): # noqa: F811 (not a redefinition) + resource = Resource.from_file_ref_and_endpoint(endpoint="", file="testfile") + newresource = Resource.from_file_ref_and_endpoint(endpoint="", file="newtestfile") + mock_response = Mock() + mock_response.status.code = cs3code.CODE_ABORTED + mock_response.status.message = "Failed arborted" + + with patch.object(file_instance._gateway, "Move", return_value=mock_response): + with pytest.raises(IOError): + file_instance.rename_file(resource, newresource) + + +def test_rename_file_not_found(file_instance): # noqa: F811 (not a redefinition) + resource = Resource.from_file_ref_and_endpoint(endpoint="", file="testfile") + newresource = Resource.from_file_ref_and_endpoint(endpoint="", file="newtestfile") + mock_response = Mock() + mock_response.status.code = cs3code.CODE_NOT_FOUND + mock_response.status.message = "Failed not found" + + with patch.object(file_instance._gateway, "Move", return_value=mock_response): + with pytest.raises(IOError): + file_instance.rename_file(resource, newresource) + + +def test_rename_file_unuathorized(file_instance): # noqa: F811 (not a redefinition) + resource = Resource.from_file_ref_and_endpoint(endpoint="", file="testfile") + newresource = Resource.from_file_ref_and_endpoint(endpoint="", file="newtestfile") + mock_response = Mock() + mock_response.status.code = cs3code.CODE_UNAUTHENTICATED + mock_response.status.message = "Failed to authenticate" + + with patch.object(file_instance._gateway, "Move", return_value=mock_response): + with pytest.raises(AuthenticationException): + file_instance.rename_file(resource, newresource) + + +def test_rename_file_unknown_error(file_instance): # noqa: F811 (not a redefinition) + resource = Resource.from_file_ref_and_endpoint(endpoint="", file="testfile") + newresource = Resource.from_file_ref_and_endpoint(endpoint="", file="newtestfile") + mock_response = Mock() + mock_response.status.code = "-1" + mock_response.status.message = "unknown error" + + with patch.object(file_instance._gateway, "Move", return_value=mock_response): + with pytest.raises(UnknownException): + file_instance.rename_file(resource, newresource) + + +def test_removefile(file_instance): # noqa: F811 (not a redefinition) + resource = Resource.from_file_ref_and_endpoint(endpoint="", file="testfile") + mock_response = Mock() + mock_response.status.code = cs3code.CODE_OK + + with patch.object(file_instance._gateway, "Delete", return_value=mock_response): + file_instance.remove_file(resource) + + +def test_removefile_not_found(file_instance): # noqa: F811 (not a redefinition) + resource = Resource.from_file_ref_and_endpoint(endpoint="", file="testfile") + mock_response = Mock() + mock_response.status.code = cs3code.CODE_NOT_FOUND + mock_response.status.message = "path not found" + + with patch.object(file_instance._gateway, "Delete", return_value=mock_response): + with pytest.raises(IOError): + file_instance.remove_file(resource) + + +def test_removefile_unauthorized(file_instance): # noqa: F811 (not a redefinition) + resource = Resource.from_file_ref_and_endpoint(endpoint="", file="testfile") + mock_response = Mock() + mock_response.status.code = cs3code.CODE_UNAUTHENTICATED + mock_response.status.message = "Failed to authenticate" + + with patch.object(file_instance._gateway, "Delete", return_value=mock_response): + with pytest.raises(AuthenticationException): + file_instance.remove_file(resource) + + +def test_removefile_unknown_error(file_instance): # noqa: F811 (not a redefinition) + resource = Resource.from_file_ref_and_endpoint(endpoint="", file="testfile") + mock_response = Mock() + mock_response.status.code = "-1" + mock_response.status.message = "Unknown error" + + with patch.object(file_instance._gateway, "Delete", return_value=mock_response): + with pytest.raises(UnknownException): + file_instance.remove_file(resource) + + +def test_touchfile(file_instance): # noqa: F811 (not a redefinition) + resource = Resource.from_file_ref_and_endpoint(endpoint="", file="testfile") + mock_response = Mock() + mock_response.status.code = cs3code.CODE_OK + + with patch.object(file_instance._gateway, "TouchFile", return_value=mock_response): + file_instance.touch_file(resource) + + +def test_touchfile_failed_precondition(file_instance): # noqa: F811 (not a redefinition) + resource = Resource.from_file_ref_and_endpoint(endpoint="", file="testfile") + mock_response = Mock() + mock_response.status.code = cs3code.CODE_FAILED_PRECONDITION + mock_response.status.message = "Failed precondition" + + with patch.object(file_instance._gateway, "TouchFile", return_value=mock_response): + with pytest.raises(FileLockedException): + file_instance.touch_file(resource) + + +def test_touchfile_failed_unauthenticated(file_instance): # noqa: F811 (not a redefinition) + resource = Resource.from_file_ref_and_endpoint(endpoint="", file="testfile") + mock_response = Mock() + mock_response.status.code = cs3code.CODE_UNAUTHENTICATED + mock_response.status.message = "Failed to authenticate" + + with patch.object(file_instance._gateway, "TouchFile", return_value=mock_response): + with pytest.raises(AuthenticationException): + file_instance.touch_file(resource) + + +def test_touchfile_unknown_error(file_instance): # noqa: F811 (not a redefinition) + resource = Resource.from_file_ref_and_endpoint(endpoint="", file="testfile") + mock_response = Mock() + mock_response.status.code = "-1" + mock_response.status.message = "Unknown error" + + with patch.object(file_instance._gateway, "TouchFile", return_value=mock_response): + with pytest.raises(UnknownException): + file_instance.touch_file(resource) + + +def test_writefile(file_instance): # noqa: F811 (not a redefinition) + resource = Resource.from_file_ref_and_endpoint(endpoint="", file="testfile") + content = "testcontent" + size = len(content) + mock_upload_response = Mock() + mock_upload_response.status.code = cs3code.CODE_OK + mock_upload_response.protocols = [Mock(protocol="simple", upload_endpoint="http://example.com", token="token")] + mock_put_response = Mock() + mock_put_response.status_code = 200 + + with patch.object(file_instance._gateway, "InitiateFileUpload", return_value=mock_upload_response): + with patch("requests.put", return_value=mock_put_response): + file_instance.write_file(resource, content, size) + + +def test_writefile_failed_precondition(file_instance): # noqa: F811 (not a redefinition) + resource = Resource.from_file_ref_and_endpoint(endpoint="", file="testfile") + content = "testcontent" + size = len(content) + mock_upload_response = Mock() + mock_upload_response.status.code = cs3code.CODE_FAILED_PRECONDITION + mock_upload_response.status.message = "Failed precondition" + + with patch.object(file_instance._gateway, "InitiateFileUpload", return_value=mock_upload_response): + with pytest.raises(FileLockedException): + file_instance.write_file(resource, content, size) + + +def test_writefile_failed_unauthenticated(file_instance): # noqa: F811 (not a redefinition) + resource = Resource.from_file_ref_and_endpoint(endpoint="", file="testfile") + content = "testcontent" + size = len(content) + mock_upload_response = Mock() + mock_upload_response.status.code = cs3code.CODE_UNAUTHENTICATED + mock_upload_response.status.message = "Failed to authenticate" + + with patch.object(file_instance._gateway, "InitiateFileUpload", return_value=mock_upload_response): + with pytest.raises(AuthenticationException): + file_instance.write_file(resource, content, size) + + +def test_writefile_unknown_error(file_instance): # noqa: F811 (not a redefinition) + resource = Resource.from_file_ref_and_endpoint(endpoint="", file="testfile") + content = "testcontent" + size = len(content) + mock_upload_response = Mock() + mock_upload_response.status.code = "-1" + mock_upload_response.status.message = "Unknown error" + + with patch.object(file_instance._gateway, "InitiateFileUpload", return_value=mock_upload_response): + with pytest.raises(UnknownException): + file_instance.write_file(resource, content, size) + + +def test_make_dir(file_instance): # noqa: F811 (not a redefinition) + resource = Resource.from_file_ref_and_endpoint(endpoint="", file="testdir") + mock_response = Mock() + mock_response.status.code = cs3code.CODE_OK + + with patch.object(file_instance._gateway, "CreateContainer", return_value=mock_response): + file_instance.make_dir(resource) + + +def test_make_dir_failed_precondition(file_instance): # noqa: F811 (not a redefinition) + resource = Resource.from_file_ref_and_endpoint(endpoint="", file="testdir") + mock_response = Mock() + mock_response.status.code = cs3code.CODE_FAILED_PRECONDITION + mock_response.status.message = "Failed precondition" + + with patch.object(file_instance._gateway, "CreateContainer", return_value=mock_response): + with pytest.raises(FileLockedException): + file_instance.make_dir(resource) + + +def test_make_dir_failed_unauthenticated(file_instance): # noqa: F811 (not a redefinition) + resource = Resource.from_file_ref_and_endpoint(endpoint="", file="testdir") + mock_response = Mock() + mock_response.status.code = cs3code.CODE_UNAUTHENTICATED + mock_response.status.message = "Failed to authenticate" + + with patch.object(file_instance._gateway, "CreateContainer", return_value=mock_response): + with pytest.raises(AuthenticationException): + file_instance.make_dir(resource) + + +def test_make_dir_unknown_error(file_instance): # noqa: F811 (not a redefinition) + resource = Resource.from_file_ref_and_endpoint(endpoint="", file="testdir") + mock_response = Mock() + mock_response.status.code = "-1" + mock_response.status.message = "Unknown error" + + with patch.object(file_instance._gateway, "CreateContainer", return_value=mock_response): + with pytest.raises(UnknownException): + file_instance.make_dir(resource) + + +def test_list_dir(file_instance): # noqa: F811 (not a redefinition) + resource = Resource.from_file_ref_and_endpoint(endpoint="", file="testdir") + mock_response = Mock() + mock_response.status.code = cs3code.CODE_OK + mock_response.infos = ["file1", "file2"] + + with patch.object(file_instance._gateway, "ListContainer", return_value=mock_response): + res = file_instance.list_dir(resource) + # Lazy evaluation + first_item = next(res, None) + if first_item is not None: + for _ in res: + pass + + +def test_list_dir_not_found(file_instance): # noqa: F811 (not a redefinition) + resource = Resource.from_file_ref_and_endpoint(endpoint="", file="testdir") + mock_response = Mock() + mock_response.status.code = cs3code.CODE_NOT_FOUND + mock_response.status.message = "Failed precondition" + + with patch.object(file_instance._gateway, "ListContainer", return_value=mock_response): + with pytest.raises(NotFoundException): + res = file_instance.list_dir(resource) + # Lazy evaluation + first_item = next(res, None) + if first_item is not None: + for _ in res: + pass + + +def test_list_dir_unauthenticated(file_instance): # noqa: F811 (not a redefinition) + resource = Resource.from_file_ref_and_endpoint(endpoint="", file="testdir") + mock_response = Mock() + mock_response.status.code = cs3code.CODE_UNAUTHENTICATED + mock_response.status.message = "Failed to authenticate" + + with patch.object(file_instance._gateway, "ListContainer", return_value=mock_response): + with pytest.raises(AuthenticationException): + res = file_instance.list_dir(resource) + # Lazy evaluation + first_item = next(res, None) + if first_item is not None: + for _ in res: + pass + + +def test_list_dir_unknown_error(file_instance): # noqa: F811 (not a redefinition) + resource = Resource.from_file_ref_and_endpoint(endpoint="", file="testdir") + mock_response = Mock() + mock_response.status.code = "-1" + mock_response.status.message = "Unknown error" + + with patch.object(file_instance._gateway, "ListContainer", return_value=mock_response): + with pytest.raises(UnknownException): + res = file_instance.list_dir(resource) + # Lazy evaluation + first_item = next(res, None) + if first_item is not None: + for _ in res: + pass + + +def test_readfile_success(file_instance): # noqa: F811 (not a redefinition) + resource = Resource.from_file_ref_and_endpoint(endpoint="", file="testfile") + mock_download_response = Mock() + mock_download_response.status.code = cs3code.CODE_OK + mock_download_response.protocols = [Mock(protocol="simple", download_endpoint="http://example.com", token="token")] + + mock_fileget_response = Mock() + mock_fileget_response.status_code = 200 + mock_fileget_response.iter_content = Mock(return_value=[b"chunk1", b"chunk2"]) + + with patch.object( + file_instance._gateway, + "InitiateFileDownload", + return_value=mock_download_response, + ): + with patch("requests.get", return_value=mock_fileget_response): + chunks = list(file_instance.read_file(resource)) + assert chunks == [b"chunk1", b"chunk2"] + + +def test_readfile_not_found(file_instance): # noqa: F811 (not a redefinition) + resource = Resource.from_file_ref_and_endpoint(endpoint="", file="testfile") + mock_fileget_response = Mock() + mock_fileget_response.status.code = cs3code.CODE_NOT_FOUND + mock_fileget_response.iter_content = Mock(return_value="None") + + with patch.object( + file_instance._gateway, + "InitiateFileDownload", + return_value=mock_fileget_response, + ): + with pytest.raises(NotFoundException): + res = file_instance.read_file(resource) + # Lazy evaluation + first_item = next(res, None) + if first_item is not None: + for _ in res: + pass + + +def test_readfile_unauthenticated(file_instance): # noqa: F811 (not a redefinition) + resource = Resource.from_file_ref_and_endpoint(endpoint="", file="testfile") + mock_fileget_response = Mock() + mock_fileget_response.status.code = cs3code.CODE_UNAUTHENTICATED + mock_fileget_response.status.message = "Failed to authenticate" + + with patch.object( + file_instance._gateway, + "InitiateFileDownload", + return_value=mock_fileget_response, + ): + with pytest.raises(AuthenticationException): + res = file_instance.read_file(resource) + # Lazy evaluation + first_item = next(res, None) + if first_item is not None: + for _ in res: + pass + + +def test_readfile_unknown_error(file_instance): # noqa: F811 (not a redefinition) + resource = Resource.from_file_ref_and_endpoint(endpoint="", file="testfile") + mock_fileget_response = Mock() + mock_fileget_response.status.code = "-1" + mock_fileget_response.status.message = "Unknown error" + + with patch.object( + file_instance._gateway, + "InitiateFileDownload", + return_value=mock_fileget_response, + ): + with pytest.raises(UnknownException): + res = file_instance.read_file(resource) + # Lazy evaluation + first_item = next(res, None) + if first_item is not None: + for _ in res: + pass diff --git a/tests/test_resource.py b/tests/test_resource.py new file mode 100644 index 0000000..32ea213 --- /dev/null +++ b/tests/test_resource.py @@ -0,0 +1,137 @@ +""" +test_resource.py + +Tests that the resource class methods work as expected. + +Authors: Rasmus Welander, Diogo Castro, Giuseppe Lo Presti. +Emails: rasmus.oscar.welander@cern.ch, diogo.castro@cern.ch, giuseppe.lopresti@cern.ch +Last updated: 26/07/2024 +""" + +import unittest +import cs3.storage.provider.v1beta1.resources_pb2 as cs3spr +from cs3resource import Resource + + +class TestResource(unittest.TestCase): + + def test_absolute_path(self): + res = Resource.from_file_ref_and_endpoint("/path/to/file") + self.assertEqual(res._abs_path, "/path/to/file") + self.assertIsNone(res._rel_path) + self.assertIsNone(res._parent_id) + self.assertIsNone(res._opaque_id) + self.assertIsNone(res._space_id) + self.assertIsNone(res._storage_id) + ref = res.ref + self.assertEqual(ref.path, "/path/to/file") + + def test_relative_path(self): + res = Resource.from_file_ref_and_endpoint("parent_id/path/to/file", "storage$space") + self.assertIsNone(res._abs_path) + self.assertEqual(res._rel_path, "/path/to/file") + self.assertEqual(res._parent_id, "parent_id") + self.assertIsNone(res._opaque_id) + self.assertEqual(res._space_id, "space") + self.assertEqual(res._storage_id, "storage") + ref = res.ref + self.assertEqual(ref.resource_id.storage_id, "storage") + self.assertEqual(ref.resource_id.space_id, "space") + self.assertEqual(ref.resource_id.opaque_id, "parent_id") + self.assertEqual(ref.path, "./path/to/file") + + def test_opaque_fileid(self): + res = Resource.from_file_ref_and_endpoint("opaque_id", "storage$space") + self.assertIsNone(res._abs_path) + self.assertIsNone(res._rel_path) + self.assertIsNone(res._parent_id) + self.assertEqual(res._opaque_id, "opaque_id") + self.assertEqual(res._space_id, "space") + self.assertEqual(res._storage_id, "storage") + ref = res.ref + self.assertEqual(ref.resource_id.storage_id, "storage") + self.assertEqual(ref.resource_id.space_id, "space") + self.assertEqual(ref.resource_id.opaque_id, "opaque_id") + self.assertEqual(ref.path, ".") + + def test_recreate_endpoint_and_file_absolute_path(self): + res = Resource.from_file_ref_and_endpoint("/path/to/file") + recreated = res.recreate_endpoint_and_file() + self.assertEqual(recreated["file"], "/path/to/file") + self.assertEqual(recreated["endpoint"], None) + + def test_recreate_endpoint_and_file_relative_path(self): + res = Resource.from_file_ref_and_endpoint("parent_id/path/to/file", "storage$space") + recreated = res.recreate_endpoint_and_file() + self.assertEqual(recreated["file"], "parent_id/path/to/file") + self.assertEqual(recreated["endpoint"], "storage$space") + + def test_recreate_endpoint_and_file_opaque_fileid(self): + res = Resource.from_file_ref_and_endpoint("opaque_id", "storage$space") + recreated = res.recreate_endpoint_and_file() + self.assertEqual(recreated["file"], "opaque_id") + self.assertEqual(recreated["endpoint"], "storage$space") + + def test_from_cs3_ref_absolute_path(self): + ref = cs3spr.Reference(path="/path/to/file") + res = Resource.from_cs3_ref(ref) + self.assertEqual(res._abs_path, "/path/to/file") + self.assertIsNone(res._rel_path) + self.assertIsNone(res._parent_id) + self.assertIsNone(res._opaque_id) + self.assertIsNone(res._space_id) + self.assertIsNone(res._storage_id) + + def test_from_cs3_ref_relative_path(self): + ref = cs3spr.Reference( + resource_id=cs3spr.ResourceId(storage_id="storage", space_id="space", opaque_id="parent_id"), + path="./path/to/file", + ) + res = Resource.from_cs3_ref(ref) + self.assertIsNone(res._abs_path) + self.assertEqual(res._rel_path, "/path/to/file") + self.assertEqual(res._parent_id, "parent_id") + self.assertIsNone(res._opaque_id) + self.assertEqual(res._space_id, "space") + self.assertEqual(res._storage_id, "storage") + + def test_from_cs3_ref_opaque_fileid(self): + ref = cs3spr.Reference( + resource_id=cs3spr.ResourceId(storage_id="storage", space_id="space", opaque_id="opaque_id"), + path=".", + ) + res = Resource.from_cs3_ref(ref) + self.assertIsNone(res._abs_path) + self.assertIsNone(res._rel_path) + self.assertIsNone(res._parent_id) + self.assertEqual(res._opaque_id, "opaque_id") + self.assertEqual(res._space_id, "space") + self.assertEqual(res._storage_id, "storage") + + def test_invalid_file_reference_in_ref(self): + res = Resource.from_file_ref_and_endpoint("/path/to/file") + res._abs_path = None # Manually invalidate the absolute path + with self.assertRaises(ValueError) as context: + _ = res.ref + self.assertEqual(str(context.exception), "Invalid Resource") + + def test_invalid_file_reference_in_recreate_endpoint_and_file(self): + res = Resource.from_file_ref_and_endpoint("/path/to/file") + res._abs_path = None # Manually invalidate the absolute path + with self.assertRaises(ValueError) as context: + _ = res.recreate_endpoint_and_file() + self.assertEqual(str(context.exception), "Invalid Resource") + + def test_from_cs3_ref_invalid_reference(self): + ref = cs3spr.Reference() # Create an empty reference + with self.assertRaises(ValueError) as context: + resource = Resource.from_cs3_ref(ref) + print(resource) + self.assertEqual(str(context.exception), "Invalid CS3 reference") + + def test_equality(self): + res1 = Resource.from_file_ref_and_endpoint("/path/to/file") + res2 = Resource.from_file_ref_and_endpoint("/path/to/file") + res3 = Resource.from_file_ref_and_endpoint("parent_id/path/to/file", "storage$space") + self.assertEqual(res1, res2) + self.assertNotEqual(res1, res3)