[WIP][8.0]refactor gfal2 and storageElement

src/DIRAC/ConfigurationSystem/Client/LocalConfiguration.py

@@ -458,6 +458,7 @@ def __loadCFGFiles(self):
         if "DIRACSYSCONFIG" in os.environ:
             diracSysConfigFiles = os.environ["DIRACSYSCONFIG"].replace(" ", "").split(",")
             for diracSysConfigFile in reversed(diracSysConfigFiles):
+                gLogger.debug("Loading file from DIRACSYSCONFIG %s" % diracSysConfigFile)
                 gConfigurationData.loadFile(diracSysConfigFile)
         gConfigurationData.loadFile(os.path.expanduser("~/.dirac.cfg"))
         for fileName in self.additionalCFGFiles:

src/DIRAC/DataManagementSystem/Service/S3GatewayHandler.py

@@ -71,7 +71,7 @@ def initializeHandler(cls, serviceInfoDict):
             # TODO: once we finally merge _allProtocolParameters with the
             # standard paramaters in the StorageBase, this will be much neater
 
-            for storagePlugin in se.storages:
+            for storagePlugin in se.storages.values():
                 storageParam = storagePlugin._allProtocolParameters  # pylint: disable=protected-access
 
                 if (

src/DIRAC/Interfaces/API/DiracAdmin.py

@@ -672,16 +672,6 @@ def setSiteProtocols(self, site, protocolsList, printOutput=False):
         if not siteSEs:
             return S_ERROR("No SEs found for site %s in section %s" % (site, siteSection))
 
-        defaultProtocols = gConfig.getValue("/Resources/StorageElements/DefaultProtocols", [])
-        self.log.verbose("Default list of protocols are", ", ".join(defaultProtocols))
-
-        for protocol in protocolsList:
-            if protocol not in defaultProtocols:
-                return S_ERROR(
-                    "Requested to set protocol %s in list but %s is not "
-                    "in default list of protocols:\n%s" % (protocol, protocol, ", ".join(defaultProtocols))
-                )
-
         modifiedCS = False
         result = promptUser(
             "Do you want to add the following default protocols:"

src/DIRAC/Resources/Storage/GFAL2_SRM2Storage.py

@@ -30,8 +30,13 @@
 class GFAL2_SRM2Storage(GFAL2_StorageBase):
     """SRM2 SE class that inherits from GFAL2StorageBase"""
 
-    _INPUT_PROTOCOLS = ["file", "root", "srm", "gsiftp"]
-    _OUTPUT_PROTOCOLS = ["file", "root", "dcap", "gsidcap", "rfio", "srm", "gsiftp"]
+    _INPUT_PROTOCOLS = [
+        "file",
+        "gsiftp",
+        "root",
+        "srm",
+    ]
+    _OUTPUT_PROTOCOLS = ["gsiftp", "root", "srm"]
 
     def __init__(self, storageName, parameters):
         """ """
@@ -51,16 +56,10 @@ def __init__(self, storageName, parameters):
 
         self.gfal2requestLifetime = gConfig.getValue("/Resources/StorageElements/RequestLifeTime", 100)
 
-        self.__setSRMOptionsToDefault()
+        self.protocolsList = self.protocolParameters["OutputProtocols"]
+        self.log.debug("GFAL2_SRM2Storage: protocolsList = %s" % self.protocolsList)
 
-        # This lists contains the list of protocols to ask to SRM to get a URL
-        # It can be either defined in the plugin of the SE, or as a global option
-        if "ProtocolsList" in parameters:
-            self.protocolsList = parameters["ProtocolsList"].split(",")
-        else:
-            self.log.debug("GFAL2_SRM2Storage: No protocols provided, using the default protocols.")
-            self.protocolsList = self.defaultLocalProtocols
-            self.log.debug("GFAL2_SRM2Storage: protocolsList = %s" % self.protocolsList)
+        self.__setSRMOptionsToDefault()
 
     def __setSRMOptionsToDefault(self):
         """Resetting the SRM options back to default"""
@@ -69,8 +68,8 @@ def __setSRMOptionsToDefault(self):
             self.ctx.set_opt_string("SRM PLUGIN", "SPACETOKENDESC", self.spaceToken)
         self.ctx.set_opt_integer("SRM PLUGIN", "REQUEST_LIFETIME", self.gfal2requestLifetime)
         # Setting the TURL protocol to gsiftp because with other protocols we have authorisation problems
-        #    self.ctx.set_opt_string_list( "SRM PLUGIN", "TURL_PROTOCOLS", self.defaultLocalProtocols )
-        self.ctx.set_opt_string_list("SRM PLUGIN", "TURL_PROTOCOLS", ["gsiftp"])
+        self.ctx.set_opt_string_list("SRM PLUGIN", "TURL_PROTOCOLS", self.protocolsList)
+        self.ctx.set_opt_string_list("SRM PLUGIN", "TURL_3RD_PARTY_PROTOCOLS", self.protocolsList)
 
     def _updateMetadataDict(self, metadataDict, attributeDict):
         """Updating the metadata dictionary with srm specific attributes

src/DIRAC/Resources/Storage/GFAL2_StorageBase.py

@@ -116,8 +116,6 @@ def __init__(self, storageName, parameters):
         self.stageTimeout = gConfig.getValue("/Resources/StorageElements/StageTimeout", 12 * 60 * 60)
         # gfal2Timeout, amount of time it takes until an operation times out
         self.gfal2Timeout = gConfig.getValue("/Resources/StorageElements/GFAL_Timeout", 100)
-        # set the gfal2 default protocols, e.g. used when trying to retrieve transport url
-        self.defaultLocalProtocols = gConfig.getValue("/Resources/StorageElements/DefaultProtocols", [])
 
         # # set checksum type, by default this is 0 (GFAL_CKSM_NONE)
         self.checksumType = gConfig.getValue("/Resources/StorageElements/ChecksumType", "0")

src/DIRAC/Resources/Storage/OccupancyPlugins/WLCGAccountingJson.py

@@ -37,7 +37,7 @@ def _downloadJsonFile(self, occupancyLFN, filePath):
         :param filePath: destination path for the file
 
         """
-        for storage in self.se.storages:
+        for storage in self.se.storages.values():
             try:
                 ctx = gfal2.creat_context()
                 params = ctx.transfer_parameters()
@@ -103,7 +103,7 @@ def getOccupancy(self, **kwargs):
                 "Could not find SpaceReservation in CS, and get storageShares and spaceReservation from WLCGAccoutingJson."
             )
             shareLen = []
-            for storage in self.se.storages:
+            for storage in self.se.storages.values():
                 basePath = storage.getParameters()["Path"]
                 for share in storageShares:
                     shareLen.append((share, len(os.path.commonprefix([share["path"][0], basePath]))))

src/DIRAC/Resources/Storage/StorageBase.py

@@ -67,6 +67,9 @@ def __init__(self, name, parameterDict):
 
         self.name = name
         self.pluginName = ""
+        # This is set by the storageFactory and is the
+        # name of the protocol section in the CS
+        self.protocolSectionName = ""
         self.protocolParameters = {}
 
         self.__updateParameters(parameterDict)