diff --git a/dirac.cfg b/dirac.cfg index 50b93e36f79..46475fb12a3 100644 --- a/dirac.cfg +++ b/dirac.cfg @@ -846,6 +846,11 @@ Resources StorageElementGroups { CERN-Storages = CERN-DST-EOS, CERN-USER + # Default SEs to be used when uploading output data from Payloads + SE-USER = CERN-USER + # Default SEs to be used as failover SEs uploading output data from Payloads. + # This option is used in the Job Wrapper and, if set, requires the RequestManagementSystem to be installed + Tier1-Failover = CERN-FAILOVER,CNAF-FAILOVER } } Operations diff --git a/docs/source/AdministratorGuide/Configuration/ConfReference/Resources/StorageElements/index.rst b/docs/source/AdministratorGuide/Configuration/ConfReference/Resources/StorageElements/index.rst deleted file mode 100644 index a0b1a8b7585..00000000000 --- a/docs/source/AdministratorGuide/Configuration/ConfReference/Resources/StorageElements/index.rst +++ /dev/null @@ -1,46 +0,0 @@ -Resources / StorageElements and StorageElementBases- Subsections -================================================================== - -All the storages elements available for the users are described in these subsections. Base Storage Elements, corresponding to abstract Storage Element, must be defined in the Resources/StorageElementBases section while other Storage Elements, like inherited and simple Storage Elements, must be configured in the Resources/StorageElement section. This information will be moved below the Sites section. - -+---------------------------------------------+--------------------------------------------------+-----------------------------+ -| **Name** | **Description** | **Example** | -+---------------------------------------------+--------------------------------------------------+-----------------------------+ -| *DefaultProtocols* | Default protocols than can be used to interact | DefaultProtocols = rfio | -| | with the storage elements. | DefaultProtocols += file | -| | | DefaultProtocols += root | -| | | DefaultProtocols += gsiftp | -+---------------------------------------------+--------------------------------------------------+-----------------------------+ -| *SITE-disk* | Subsection. DIRAC name for the storage element | CPPM-disk | -+---------------------------------------------+--------------------------------------------------+-----------------------------+ -| *SITE-disk/BackendType* | Type of storage element. Possible values are: | BackendType = dpm | -| | dmp, DISET, dCache, Storm | | -+---------------------------------------------+--------------------------------------------------+-----------------------------+ -| *SITE-disk/ReadAccess* | Allow read access | ReadAccess = Active | -| | Possible values are: Active, InActive | | -+---------------------------------------------+--------------------------------------------------+-----------------------------+ -| *SITE-disk/WriteAccess* | Allow write access | WriteAccess = Active | -| | Possible values are: Active, InActive | | -+---------------------------------------------+--------------------------------------------------+-----------------------------+ -| *SITE-disk/RemoveAccess* | Allow removal of files at this SE | RemoveAccess = Active | -| | Possible values are: Active, InActive | | -+---------------------------------------------+--------------------------------------------------+-----------------------------+ -| *SITE-disk/SEType* | Type of SE | SEType = T0D1 | -| | Possible values are: T0D1, T1D0, D1T0 | | -+---------------------------------------------+--------------------------------------------------+-----------------------------+ -| *SITE-disk/AccessProtocol.<#>* | Subsection. Access protocol number | AccessProtocol.1 | -+---------------------------------------------+--------------------------------------------------+-----------------------------+ -| *SITE-disk/AccessProtocol.<#>/Access* | Access type to the resource | Access = Remote | -+---------------------------------------------+--------------------------------------------------+-----------------------------+ -| *SITE-disk/AccessProtocol.<#>/Host* | Storage element fully qualified hostname | Host = se01.in2p3.fr | -+---------------------------------------------+--------------------------------------------------+-----------------------------+ -| *SITE-disk/AccessProtocol.<#>/Path* | Path in the SE just before the VO directory | Path = /dpm/in2p3.fr/home | -+---------------------------------------------+--------------------------------------------------+-----------------------------+ -| *SITE-disk/AccessProtocol.<#>/Port* | Port number to access the data | Port = 8446 | -+---------------------------------------------+--------------------------------------------------+-----------------------------+ -| *SITE-disk/AccessProtocol.<#>/Protocol* | Protocol to be used to interact with the SE | Protocol = srm | -+---------------------------------------------+--------------------------------------------------+-----------------------------+ -| *SITE-disk/AccessProtocol.<#>/PluginName* | Protocol name to be used to interact with the SE | PluginName = GFAL2_SRM2 | -+---------------------------------------------+--------------------------------------------------+-----------------------------+ -| *SITE-disk/AccessProtocol.<#>/WSUrl* | URL from WebServices | WSUrl = /srm/managerv2?SFN= | -+---------------------------------------------+--------------------------------------------------+-----------------------------+ diff --git a/docs/source/AdministratorGuide/Configuration/ConfReference/Resources/StorageElementsGroups/index.rst b/docs/source/AdministratorGuide/Configuration/ConfReference/Resources/StorageElementsGroups/index.rst deleted file mode 100644 index 00e64d13516..00000000000 --- a/docs/source/AdministratorGuide/Configuration/ConfReference/Resources/StorageElementsGroups/index.rst +++ /dev/null @@ -1,13 +0,0 @@ -Resources / StorageElementGroups - Subsections -============================================== - -All the storages elements groups available for the users are described in this subsection. - -+-----------------+----------------------------------------------------------------------------------------------------------+-------------------------------+ -| **Name** | **Description** | **Example** | -+-----------------+----------------------------------------------------------------------------------------------------------+-------------------------------+ -| SE-USER | Default SEs to be used when uploading output data from Payloads | CERN-USER | -+-----------------+----------------------------------------------------------------------------------------------------------+-------------------------------+ -| Tier1-Failover | Default SEs to be used as failover SEs uploading output data from Payloads. | CERN-FAILOVER,CNAF-FAILOVER | -| | This option is used in the Job Wrapper and, if set, requires the RequestManagementSystem to be installed | | -+-----------------+----------------------------------------------------------------------------------------------------------+-------------------------------+ diff --git a/docs/source/AdministratorGuide/Configuration/ConfReference/Resources/index.rst b/docs/source/AdministratorGuide/Configuration/ConfReference/Resources/index.rst index ad0915347d0..d18a2b3a176 100644 --- a/docs/source/AdministratorGuide/Configuration/ConfReference/Resources/index.rst +++ b/docs/source/AdministratorGuide/Configuration/ConfReference/Resources/index.rst @@ -11,5 +11,3 @@ In this section all the physical resources than can be used by DIRAC users are d FileCatalogs/index Sites/index - StorageElements/index - StorageElementsGroups/index diff --git a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/DataManagement/Services/StorageElement/index.rst b/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/DataManagement/Services/StorageElement/index.rst deleted file mode 100644 index 2ce9d8b7521..00000000000 --- a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/DataManagement/Services/StorageElement/index.rst +++ /dev/null @@ -1,10 +0,0 @@ -Systems / DataManagement / / Service / StorageElement - Sub-subsection -================================================================================= - -StorageElementHandler is the implementation of a simple StorageElement service in the DISET framework - -+------------+------------------------------------------+----------------------------+ -| **Name** | **Description** | **Example** | -+------------+------------------------------------------+----------------------------+ -| *BasePath* | Directory path used as base for DIRAC SE | BasePath = /opt/dirac/data | -+------------+------------------------------------------+----------------------------+ diff --git a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/DataManagement/Services/index.rst b/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/DataManagement/Services/index.rst index 2d272dbb680..2d5abbe5265 100644 --- a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/DataManagement/Services/index.rst +++ b/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/DataManagement/Services/index.rst @@ -32,5 +32,4 @@ DataManagement services are: :maxdepth: 2 FileCatalog/index - StorageElement/index StorageElementProxy/index diff --git a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/DataManagement/URLs/index.rst b/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/DataManagement/URLs/index.rst deleted file mode 100644 index 1c2fdb8c16e..00000000000 --- a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/DataManagement/URLs/index.rst +++ /dev/null @@ -1,23 +0,0 @@ -Systems / DataManagement / / URLs - Sub-subsection -=============================================================== - -DataManagement Services URLs. - -+------------------+----------------------------------------+--------------------------------------------------------------+ -| **Name** | **Description** | **Example** | -+------------------+----------------------------------------+--------------------------------------------------------------+ -| ** | URL associated with the service, value | dips://dirac.eela.if.ufrj.br:9197/DataManagement/FileCatalog | -| | URL using dips protocol | | -+------------------+----------------------------------------+--------------------------------------------------------------+ - -Services associated with DataManagement System: - -+------------------------+----------+ -| **Service** | **Port** | -+------------------------+----------+ -| *FileCatalog* | 9197 | -+------------------------+----------+ -| *StorageElement* | 9148 | -+------------------------+----------+ -| *StorageElementProxy* | 9139 | -+------------------------+----------+ diff --git a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/DataManagement/index.rst b/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/DataManagement/index.rst index 156822fdf60..e32dd9535f5 100644 --- a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/DataManagement/index.rst +++ b/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/DataManagement/index.rst @@ -8,4 +8,3 @@ In this subsection are described the databases, services and URLs related with t Databases/index Services/index - URLs/index diff --git a/src/DIRAC/Resources/Storage/GFAL2_SRM2Storage.py b/src/DIRAC/Resources/Storage/GFAL2_SRM2Storage.py index ced2823bb42..87fd84cf967 100644 --- a/src/DIRAC/Resources/Storage/GFAL2_SRM2Storage.py +++ b/src/DIRAC/Resources/Storage/GFAL2_SRM2Storage.py @@ -51,7 +51,7 @@ def __init__(self, storageName, parameters): self.gfal2requestLifetime = gConfig.getValue("/Resources/StorageElements/RequestLifeTime", 100) self.protocolsList = self.protocolParameters["OutputProtocols"] - self.log.debug("GFAL2_SRM2Storage: protocolsList = %s" % self.protocolsList) + self.log.debug(f"GFAL2_SRM2Storage: protocolsList = {self.protocolsList}") self.__setSRMOptionsToDefault() diff --git a/src/DIRAC/Resources/Storage/StorageElement.py b/src/DIRAC/Resources/Storage/StorageElement.py index 552a5639b2b..8c82834f309 100755 --- a/src/DIRAC/Resources/Storage/StorageElement.py +++ b/src/DIRAC/Resources/Storage/StorageElement.py @@ -1,4 +1,4 @@ -""" This is the StorageElement class. +""" This is the StorageElement module. It implements The StorageElementItem as well as the caching system """ # # custom duty @@ -41,6 +41,11 @@ class StorageElementCache(object): + """ + The StorageElementCache keeps StorageElementItem instances in a cache to save on initialization cost. + It keeps one instance per tuple (thread ID, seName, protocolSections, VO, proxy ) + """ + def __init__(self): self.seCache = DictCache() @@ -77,18 +82,26 @@ def __call__(self, name, protocolSections=None, vo=None, hideExceptions=False): class StorageElementItem(object): """ - .. class:: StorageElement + .. class:: StorageElementItem + + This class implements all the necessary logic to interact with the GRID storage Elements. Actual interaction with storages are delegated to ``StoragePlugins``. + + The role of the StorageElementItem is to: - common interface to the grid storage element + * Provide a single interface to all the grid storages + * Ensure the status of the Storage in RSS + * Select multiple protocols for various operations + * Support multiple protocols as failover + * Negociate protocols with other StorageElement for Third Party Copy - self.name is the resolved name of the StorageElement i.e CERN-tape - self.options is dictionary containing the general options defined in the CS e.g. self.options['Backend] = 'Castor2' - self.storages is a dict of the stub objects created by StorageFactory for the protocols found in the CS. Index by the protocol section name. - self.localProtocolSections is a list of the local protocols that were created by StorageFactory - self.remoteProtocolSections is a list of the remote protocols that were created by StorageFactory - self.protocolOptions is a list of dictionaries containing the options found in the CS. (should be removed) + :ivar str name: Resolved name of the StorageElement + :ivar dict options: dictionary containing the general options defined in the CS + :ivar dict storages: dict of the stub objects created by StorageFactory for the protocols found in the CS. Index by the protocol section name. + :ivar list localProtocolSections: list of the local protocols that were created by StorageFactory + :ivar list remoteProtocolSections: list of the remote protocols that were created by StorageFactory + :ivar list protocolOptions: list of dictionaries containing the options found in the CS. (should be removed)