diff --git a/src/DIRAC/ConfigurationSystem/Client/LocalConfiguration.py b/src/DIRAC/ConfigurationSystem/Client/LocalConfiguration.py index d1dae30c47e..e4f4473863e 100755 --- a/src/DIRAC/ConfigurationSystem/Client/LocalConfiguration.py +++ b/src/DIRAC/ConfigurationSystem/Client/LocalConfiguration.py @@ -458,6 +458,7 @@ def __loadCFGFiles(self): if "DIRACSYSCONFIG" in os.environ: diracSysConfigFiles = os.environ["DIRACSYSCONFIG"].replace(" ", "").split(",") for diracSysConfigFile in reversed(diracSysConfigFiles): + gLogger.debug("Loading file from DIRACSYSCONFIG %s" % diracSysConfigFile) gConfigurationData.loadFile(diracSysConfigFile) gConfigurationData.loadFile(os.path.expanduser("~/.dirac.cfg")) for fileName in self.additionalCFGFiles: diff --git a/src/DIRAC/DataManagementSystem/Service/S3GatewayHandler.py b/src/DIRAC/DataManagementSystem/Service/S3GatewayHandler.py index 9a58cb4e16d..cd7f312cf5a 100644 --- a/src/DIRAC/DataManagementSystem/Service/S3GatewayHandler.py +++ b/src/DIRAC/DataManagementSystem/Service/S3GatewayHandler.py @@ -71,7 +71,7 @@ def initializeHandler(cls, serviceInfoDict): # TODO: once we finally merge _allProtocolParameters with the # standard paramaters in the StorageBase, this will be much neater - for storagePlugin in se.storages: + for storagePlugin in se.storages.values(): storageParam = storagePlugin._allProtocolParameters # pylint: disable=protected-access if ( diff --git a/src/DIRAC/Interfaces/API/DiracAdmin.py b/src/DIRAC/Interfaces/API/DiracAdmin.py index 1e8867d08e0..fec1281af60 100755 --- a/src/DIRAC/Interfaces/API/DiracAdmin.py +++ b/src/DIRAC/Interfaces/API/DiracAdmin.py @@ -672,16 +672,6 @@ def setSiteProtocols(self, site, protocolsList, printOutput=False): if not siteSEs: return S_ERROR("No SEs found for site %s in section %s" % (site, siteSection)) - defaultProtocols = gConfig.getValue("/Resources/StorageElements/DefaultProtocols", []) - self.log.verbose("Default list of protocols are", ", ".join(defaultProtocols)) - - for protocol in protocolsList: - if protocol not in defaultProtocols: - return S_ERROR( - "Requested to set protocol %s in list but %s is not " - "in default list of protocols:\n%s" % (protocol, protocol, ", ".join(defaultProtocols)) - ) - modifiedCS = False result = promptUser( "Do you want to add the following default protocols:" diff --git a/src/DIRAC/Resources/Storage/GFAL2_SRM2Storage.py b/src/DIRAC/Resources/Storage/GFAL2_SRM2Storage.py index b87dfa24337..888a74dd48f 100644 --- a/src/DIRAC/Resources/Storage/GFAL2_SRM2Storage.py +++ b/src/DIRAC/Resources/Storage/GFAL2_SRM2Storage.py @@ -30,8 +30,13 @@ class GFAL2_SRM2Storage(GFAL2_StorageBase): """SRM2 SE class that inherits from GFAL2StorageBase""" - _INPUT_PROTOCOLS = ["file", "root", "srm", "gsiftp"] - _OUTPUT_PROTOCOLS = ["file", "root", "dcap", "gsidcap", "rfio", "srm", "gsiftp"] + _INPUT_PROTOCOLS = [ + "file", + "gsiftp", + "root", + "srm", + ] + _OUTPUT_PROTOCOLS = ["gsiftp", "root", "srm"] def __init__(self, storageName, parameters): """ """ @@ -51,16 +56,10 @@ def __init__(self, storageName, parameters): self.gfal2requestLifetime = gConfig.getValue("/Resources/StorageElements/RequestLifeTime", 100) - self.__setSRMOptionsToDefault() + self.protocolsList = self.protocolParameters["OutputProtocols"] + self.log.debug("GFAL2_SRM2Storage: protocolsList = %s" % self.protocolsList) - # This lists contains the list of protocols to ask to SRM to get a URL - # It can be either defined in the plugin of the SE, or as a global option - if "ProtocolsList" in parameters: - self.protocolsList = parameters["ProtocolsList"].split(",") - else: - self.log.debug("GFAL2_SRM2Storage: No protocols provided, using the default protocols.") - self.protocolsList = self.defaultLocalProtocols - self.log.debug("GFAL2_SRM2Storage: protocolsList = %s" % self.protocolsList) + self.__setSRMOptionsToDefault() def __setSRMOptionsToDefault(self): """Resetting the SRM options back to default""" @@ -69,8 +68,8 @@ def __setSRMOptionsToDefault(self): self.ctx.set_opt_string("SRM PLUGIN", "SPACETOKENDESC", self.spaceToken) self.ctx.set_opt_integer("SRM PLUGIN", "REQUEST_LIFETIME", self.gfal2requestLifetime) # Setting the TURL protocol to gsiftp because with other protocols we have authorisation problems - # self.ctx.set_opt_string_list( "SRM PLUGIN", "TURL_PROTOCOLS", self.defaultLocalProtocols ) - self.ctx.set_opt_string_list("SRM PLUGIN", "TURL_PROTOCOLS", ["gsiftp"]) + self.ctx.set_opt_string_list("SRM PLUGIN", "TURL_PROTOCOLS", self.protocolsList) + self.ctx.set_opt_string_list("SRM PLUGIN", "TURL_3RD_PARTY_PROTOCOLS", self.protocolsList) def _updateMetadataDict(self, metadataDict, attributeDict): """Updating the metadata dictionary with srm specific attributes diff --git a/src/DIRAC/Resources/Storage/GFAL2_StorageBase.py b/src/DIRAC/Resources/Storage/GFAL2_StorageBase.py index 030ef2a2d4e..675f481298e 100644 --- a/src/DIRAC/Resources/Storage/GFAL2_StorageBase.py +++ b/src/DIRAC/Resources/Storage/GFAL2_StorageBase.py @@ -116,8 +116,6 @@ def __init__(self, storageName, parameters): self.stageTimeout = gConfig.getValue("/Resources/StorageElements/StageTimeout", 12 * 60 * 60) # gfal2Timeout, amount of time it takes until an operation times out self.gfal2Timeout = gConfig.getValue("/Resources/StorageElements/GFAL_Timeout", 100) - # set the gfal2 default protocols, e.g. used when trying to retrieve transport url - self.defaultLocalProtocols = gConfig.getValue("/Resources/StorageElements/DefaultProtocols", []) # # set checksum type, by default this is 0 (GFAL_CKSM_NONE) self.checksumType = gConfig.getValue("/Resources/StorageElements/ChecksumType", "0") diff --git a/src/DIRAC/Resources/Storage/OccupancyPlugins/WLCGAccountingJson.py b/src/DIRAC/Resources/Storage/OccupancyPlugins/WLCGAccountingJson.py index 45fc787ddc1..90180afbf53 100644 --- a/src/DIRAC/Resources/Storage/OccupancyPlugins/WLCGAccountingJson.py +++ b/src/DIRAC/Resources/Storage/OccupancyPlugins/WLCGAccountingJson.py @@ -37,7 +37,7 @@ def _downloadJsonFile(self, occupancyLFN, filePath): :param filePath: destination path for the file """ - for storage in self.se.storages: + for storage in self.se.storages.values(): try: ctx = gfal2.creat_context() params = ctx.transfer_parameters() @@ -103,7 +103,7 @@ def getOccupancy(self, **kwargs): "Could not find SpaceReservation in CS, and get storageShares and spaceReservation from WLCGAccoutingJson." ) shareLen = [] - for storage in self.se.storages: + for storage in self.se.storages.values(): basePath = storage.getParameters()["Path"] for share in storageShares: shareLen.append((share, len(os.path.commonprefix([share["path"][0], basePath])))) diff --git a/src/DIRAC/Resources/Storage/StorageBase.py b/src/DIRAC/Resources/Storage/StorageBase.py index 1ef9a7e5cb4..ae57aed02b5 100755 --- a/src/DIRAC/Resources/Storage/StorageBase.py +++ b/src/DIRAC/Resources/Storage/StorageBase.py @@ -67,6 +67,9 @@ def __init__(self, name, parameterDict): self.name = name self.pluginName = "" + # This is set by the storageFactory and is the + # name of the protocol section in the CS + self.protocolSectionName = "" self.protocolParameters = {} self.__updateParameters(parameterDict) diff --git a/src/DIRAC/Resources/Storage/StorageElement.py b/src/DIRAC/Resources/Storage/StorageElement.py index 43b38c9f05f..eb73b474b1f 100755 --- a/src/DIRAC/Resources/Storage/StorageElement.py +++ b/src/DIRAC/Resources/Storage/StorageElement.py @@ -51,7 +51,7 @@ class StorageElementCache(object): def __init__(self): self.seCache = DictCache() - def __call__(self, name, plugins=None, vo=None, hideExceptions=False): + def __call__(self, name, protocolSections=None, vo=None, hideExceptions=False): self.seCache.purgeExpired(expiredInSeconds=60) tId = threading.current_thread().ident @@ -67,15 +67,15 @@ def __call__(self, name, plugins=None, vo=None, hideExceptions=False): # If we see its memory consumtpion exploding, this might be a place to look proxyLoc = getProxyLocation() - # ensure plugins is hashable! (tuple) - if isinstance(plugins, list): - plugins = tuple(plugins) + # ensure protocolSections is hashable! (tuple) + if isinstance(protocolSections, list): + protocolSections = tuple(protocolSections) - argTuple = (tId, name, plugins, vo, proxyLoc) + argTuple = (tId, name, protocolSections, vo, proxyLoc) seObj = self.seCache.get(argTuple) if not seObj: - seObj = StorageElementItem(name, plugins, vo, hideExceptions=hideExceptions) + seObj = StorageElementItem(name, protocolSections=protocolSections, vo=vo, hideExceptions=hideExceptions) # Add the StorageElement to the cache for 1/2 hour self.seCache.add(argTuple, 1800, seObj) @@ -92,9 +92,9 @@ class StorageElementItem(object): self.name is the resolved name of the StorageElement i.e CERN-tape self.options is dictionary containing the general options defined in the CS e.g. self.options['Backend] = 'Castor2' - self.storages is a list of the stub objects created by StorageFactory for the protocols found in the CS. - self.localPlugins is a list of the local protocols that were created by StorageFactory - self.remotePlugins is a list of the remote protocols that were created by StorageFactory + self.storages is a dict of the stub objects created by StorageFactory for the protocols found in the CS. Index by the protocol section name. + self.localProtocolSections is a list of the local protocols that were created by StorageFactory + self.remoteProtocolSections is a list of the remote protocols that were created by StorageFactory self.protocolOptions is a list of dictionaries containing the options found in the CS. (should be removed) @@ -164,19 +164,19 @@ class StorageElementItem(object): "getDirectory": {"localPath": False}, } - def __init__(self, name, plugins=None, vo=None, hideExceptions=False): + def __init__(self, name, protocolSections=None, vo=None, hideExceptions=False): """c'tor :param str name: SE name - :param list plugins: requested storage plugins + :param list protocolSections: requested storage protocolSections :param vo: vo """ self.methodName = None - if plugins is None: - plugins = [] + if protocolSections is None: + protocolSections = [] if vo: self.vo = vo @@ -202,7 +202,7 @@ def __init__(self, name, plugins=None, vo=None, hideExceptions=False): self.valid = True res = StorageFactory(useProxy=self.useProxy, vo=self.vo).getStorages( - name, pluginList=plugins, hideExceptions=hideExceptions + name, protocolSections=protocolSections, hideExceptions=hideExceptions ) if not res["OK"]: @@ -213,13 +213,13 @@ def __init__(self, name, plugins=None, vo=None, hideExceptions=False): factoryDict = res["Value"] self.name = factoryDict["StorageName"] self.options = factoryDict["StorageOptions"] - self.localPlugins = factoryDict["LocalPlugins"] - self.remotePlugins = factoryDict["RemotePlugins"] + self.localProtocolSections = factoryDict["LocalProtocolSections"] + self.remoteProtocolSections = factoryDict["RemoteProtocolSections"] self.storages = factoryDict["StorageObjects"] self.protocolOptions = factoryDict["ProtocolOptions"] self.turlProtocols = factoryDict["TurlProtocols"] - for storage in self.storages: + for storage in self.storages.values(): storage.setStorageElement(self) @@ -293,6 +293,7 @@ def __init__(self, name, plugins=None, vo=None, hideExceptions=False): self.okMethods = [ "getLocalProtocols", "getProtocols", + "getProtocolSections", "getRemoteProtocols", "storageElementName", "getStorageParameters", @@ -314,7 +315,7 @@ def dump(self): for key in sorted(self.options): outStr = "%s%s: %s\n" % (outStr, key.ljust(15), self.options[key]) - for storage in self.storages: + for storage in self.storages.values(): outStr = "%s============Protocol %s ============\n" % (outStr, i) storageParameters = storage.getParameters() for key in sorted(storageParameters): @@ -381,11 +382,11 @@ def isSameSE(self, otherSE): selfEndpoints = set() otherSEEndpoints = set() - for storage in self.storages: + for storage in self.storages.values(): storageParam = storage.getParameters() selfEndpoints.add((storageParam["Host"], storageParam["Path"])) - for storage in otherSE.storages: + for storage in otherSE.storages.values(): storageParam = storage.getParameters() otherSEEndpoints.add((storageParam["Host"], storageParam["Path"])) @@ -633,58 +634,61 @@ def isValid(self, operation=None): return S_ERROR(errno.EACCES, "SE.isValid: Remove access not currently permitted.") return S_OK() - def getPlugins(self): - """Get the list of all the plugins defined for this Storage Element""" - self.log.getSubLogger("getPlugins").debug("Obtaining all plugins of %s." % self.name) + def getProtocolSections(self): + """Get the list of all the ProtocolSections defined for this Storage Element""" + self.log.getSubLogger("getProtocolSections").debug("Obtaining all protocol sections of %s." % self.name) if not self.valid: return S_ERROR(self.errorReason) - allPlugins = self.localPlugins + self.remotePlugins - return S_OK(allPlugins) + allProtocolSections = self.localProtocolSections + self.remoteProtocolSections + return S_OK(allProtocolSections) - def getRemotePlugins(self): + def getRemoteProtocolSections(self): """Get the list of all the remote access protocols defined for this Storage Element""" - self.log.getSubLogger("getRemotePlugins").debug("Obtaining remote protocols for %s." % self.name) + self.log.getSubLogger("getRemoteProtocolSections").debug("Obtaining remote protocols for %s." % self.name) if not self.valid: return S_ERROR(self.errorReason) - return S_OK(self.remotePlugins) + return S_OK(self.remoteProtocolSections) - def getLocalPlugins(self): + def getLocalProtocolSections(self): """Get the list of all the local access protocols defined for this Storage Element""" - self.log.getSubLogger("getLocalPlugins").debug("Obtaining local protocols for %s." % self.name) + self.log.getSubLogger("getLocalProtocolSections").debug("Obtaining local protocols for %s." % self.name) if not self.valid: return S_ERROR(self.errorReason) - return S_OK(self.localPlugins) + return S_OK(self.localProtocolSections) - def getStorageParameters(self, plugin=None, protocol=None): + def getStorageParameters(self, protocolSection=None, protocol=None): """Get plugin specific options - :param plugin: plugin we are interested in + :param protocolSection: protocolSection we are interested in :param protocol: protocol we are interested in - Either plugin or protocol can be defined, not both, but at least one of them + Either protocolSection or protocol can be defined, not both, but at least one of them """ # both set - if plugin and protocol: + if protocolSection and protocol: return S_ERROR(errno.EINVAL, "plugin and protocol cannot be set together.") # both None - elif not (plugin or protocol): + elif not (protocolSection or protocol): return S_ERROR(errno.EINVAL, "plugin and protocol cannot be None together.") log = self.log.getSubLogger("getStorageParameters") - reqStr = "plugin %s" % plugin if plugin else "protocol %s" % protocol + reqStr = "protocolSection %s" % protocolSection if protocolSection else "protocol %s" % protocol log.debug("Obtaining storage parameters for %s for %s." % (self.name, reqStr)) - for storage in self.storages: - storageParameters = storage.getParameters() - if plugin and storageParameters["PluginName"] == plugin: - return S_OK(storageParameters) - elif protocol and storageParameters["Protocol"] == protocol: - return S_OK(storageParameters) + if protocolSection: + storage = self.storages.get(protocolSection) + if storage: + return S_OK(storage.getParameters()) + else: + for storage in self.storages.values(): + storageParameters = storage.getParameters() + if storageParameters["Protocol"] == protocol: + return S_OK(storageParameters) - errStr = "Requested plugin or protocol not available." + errStr = "Requested protocolSection or protocol not available." log.debug(errStr, "%s for %s" % (reqStr, self.name)) return S_ERROR(errno.ENOPROTOOPT, errStr) @@ -694,7 +698,9 @@ def __getAllProtocols(self, protoType): :param proto = InputProtocols or OutputProtocols """ - return set(reduce(lambda x, y: x + y, [plugin.protocolParameters[protoType] for plugin in self.storages])) + return set( + reduce(lambda x, y: x + y, [plugin.protocolParameters[protoType] for plugin in self.storages.values()]) + ) def _getAllInputProtocols(self): """Returns all the protocols supported by the SE for Input""" @@ -742,11 +748,11 @@ def generateTransferURLsBetweenSEs(self, lfns, sourceSE, protocols=None): # This is to favor for example the xroot plugin over the SRM plugin # even if both can provide xroot sourceSEStorages = sorted( - sourceSE.storages, key=lambda x: getIndexInList(x.getParameters()["Protocol"], commonProtocols) + sourceSE.storages.values(), key=lambda x: getIndexInList(x.getParameters()["Protocol"], commonProtocols) ) selfStorages = sorted( - self.storages, key=lambda x: getIndexInList(x.getParameters()["Protocol"], commonProtocols) + self.storages.values(), key=lambda x: getIndexInList(x.getParameters()["Protocol"], commonProtocols) ) # Taking each protocol at the time, we try to generate src and dest URLs @@ -890,7 +896,7 @@ def __getURLPath(self, url): # Check all available storages and check whether the url is for that protocol urlPath = "" - for storage in self.storages: + for storage in self.storages.values(): res = storage.isNativeURL(url) if res["OK"]: if res["Value"]: @@ -1050,7 +1056,6 @@ def __filterPlugins(self, methodName, protocols=None, inputProtocol=None): """ log = self.log.getSubLogger("__filterPlugins") - log.debug( "Filtering plugins for %s (protocol = %s ; inputProtocol = %s)" % (methodName, protocols, inputProtocol) ) @@ -1076,18 +1081,24 @@ def __filterPlugins(self, methodName, protocols=None, inputProtocol=None): # otherwise we return them all if protocols: setProtocol = set(protocols) - for plugin in self.storages: + for plugin in self.storages.values(): if set(plugin.protocolParameters.get("OutputProtocols", [])) & setProtocol: log.debug("Plugin %s can generate compatible protocol" % plugin.pluginName) pluginsToUse.append(plugin) else: - pluginsToUse = self.storages + pluginsToUse = list(self.storages.values()) # The closest list for "OK" methods is the AccessProtocol preference, so we sort based on that - pluginsToUse.sort( - key=lambda x: getIndexInList(x.protocolParameters["Protocol"], self.localAccessProtocolList) + + pluginsToUse = sorted( + pluginsToUse, + key=lambda x: ( + getIndexInList(x.protocolParameters["Protocol"], self.localAccessProtocolList), + x.protocolSectionName in self.remoteProtocolSections, + ), ) - log.debug("Plugins to be used for %s: %s" % (methodName, [p.pluginName for p in pluginsToUse])) + + log.debug("Plugins to be used for %s: %s" % (methodName, [p.protocolSectionName for p in pluginsToUse])) return pluginsToUse log.debug("Allowed protocol: %s" % allowedProtocols) @@ -1102,36 +1113,45 @@ def __filterPlugins(self, methodName, protocols=None, inputProtocol=None): localSE = self.__isLocalSE()["Value"] - for plugin in self.storages: + for protocolSection, plugin in self.storages.items(): # Determine whether to use this storage object pluginParameters = plugin.getParameters() - pluginName = pluginParameters.get("PluginName") + isProxyPlugin = pluginParameters.get("PluginName") == "Proxy" if not pluginParameters: - log.debug("Failed to get storage parameters.", "%s %s" % (self.name, pluginName)) + log.debug("Failed to get storage parameters.", "%s %s" % (self.name, protocolSection)) continue - if not (pluginName in self.remotePlugins) and not localSE and not pluginName == "Proxy": + if not (protocolSection in self.remoteProtocolSections) and not localSE and not isProxyPlugin: # If the SE is not local then we can't use local protocols - log.debug("Local protocol not appropriate for remote use: %s." % pluginName) + log.debug("Local protocol not appropriate for remote use: %s." % protocolSection) continue if pluginParameters["Protocol"] not in potentialProtocols: - log.debug("Plugin %s not allowed for %s." % (pluginName, methodName)) + log.debug("Plugin %s not allowed for %s." % (protocolSection, methodName)) continue # If we are attempting a putFile and we know the inputProtocol if methodName == "putFile" and inputProtocol: if inputProtocol not in pluginParameters["InputProtocols"]: - log.debug("Plugin %s not appropriate for %s protocol as input." % (pluginName, inputProtocol)) + log.debug("Plugin %s not appropriate for %s protocol as input." % (protocolSection, inputProtocol)) continue pluginsToUse.append(plugin) # sort the plugins according to the lists in the CS - pluginsToUse.sort(key=lambda x: getIndexInList(x.protocolParameters["Protocol"], allowedProtocols)) + # and then favor local plugins over remote ones + # note: False < True, so to have local plugin first, + # we test if the plugin is in the remote list + pluginsToUse = sorted( + pluginsToUse, + key=lambda x: ( + getIndexInList(x.protocolParameters["Protocol"], allowedProtocols), + x.protocolSectionName in self.remoteProtocolSections, + ), + ) - log.debug("Plugins to be used for %s: %s" % (methodName, [p.pluginName for p in pluginsToUse])) + log.debug("Plugins to be used for %s: %s" % (methodName, [p.protocolSectionName for p in pluginsToUse])) return pluginsToUse diff --git a/src/DIRAC/Resources/Storage/StorageFactory.py b/src/DIRAC/Resources/Storage/StorageFactory.py index b7fc4d2b119..6e2d4438c22 100755 --- a/src/DIRAC/Resources/Storage/StorageFactory.py +++ b/src/DIRAC/Resources/Storage/StorageFactory.py @@ -42,12 +42,12 @@ def __init__(self, useProxy=False, vo=None): self.vo = result["Value"] else: RuntimeError("Can not get the current VO context") - self.remotePlugins = [] - self.localPlugins = [] + self.remoteProtocolSections = [] + self.localProtocolSections = [] self.name = "" self.options = {} self.protocols = {} - self.storages = [] + self.storages = {} ########################################################################################### # @@ -67,7 +67,7 @@ def getStorage(self, parameterDict, hideExceptions=False): gLogger.error(errStr) return S_ERROR(errStr) - # PluginName must be supplied otherwise nothing with work. + # PluginName must be supplied otherwise nothing will work. pluginName = parameterDict.get("PluginName") if not pluginName: errStr = "StorageFactory.getStorage: PluginName must be supplied" @@ -76,24 +76,24 @@ def getStorage(self, parameterDict, hideExceptions=False): return self.__generateStorageObject(storageName, pluginName, parameterDict, hideExceptions=hideExceptions) - def getStorages(self, storageName, pluginList=None, hideExceptions=False): + def getStorages(self, storageName, protocolSections=None, hideExceptions=False): """Get an instance of a Storage based on the DIRAC SE name based on the CS entries CS :param storageName: is the DIRAC SE name i.e. 'CERN-RAW' - :param pluginList: is an optional list of protocols if a sub-set is desired i.e ['SRM2','SRM1'] + :param protocolSections: is an optional list of protocols if a sub-set is desired i.e ['SRM2','SRM1'] :return: dictionary containing storage elements and information about them """ - self.remotePlugins = [] - self.localPlugins = [] + self.remoteProtocolSections = [] + self.localProtocolSections = [] self.name = "" self.options = {} self.protocols = {} - self.storages = [] - if pluginList is None: - pluginList = [] - elif isinstance(pluginList, six.string_types): - pluginList = [pluginList] + self.storages = {} + if protocolSections is None: + protocolSections = [] + elif isinstance(protocolSections, six.string_types): + protocolSections = [protocolSections] if not self.vo: gLogger.warn("No VO information available") @@ -122,6 +122,7 @@ def getStorages(self, storageName, pluginList=None, hideExceptions=False): res = self._getConfigStorageOptions( storageName, derivedStorageName=derivedStorageName, seConfigPath=seConfigPath ) + if not res["OK"]: return res self.options = res["Value"] @@ -134,26 +135,30 @@ def getStorages(self, storageName, pluginList=None, hideExceptions=False): return res self.protocols = res["Value"] - requestedLocalPlugins = [] - requestedRemotePlugins = [] + requestLocalProtocolSections = [] + requestRemoteProtocolSections = [] requestedProtocolDetails = [] turlProtocols = [] # Generate the protocol specific plug-ins - for protocolSection, protocolDetails in self.protocols.items(): - pluginName = protocolDetails.get("PluginName", protocolSection) - if pluginList and pluginName not in pluginList: + for protocolSectionName, protocolDetails in self.protocols.items(): + # Type of plugins to use + pluginName = protocolDetails.get("PluginName", protocolSectionName) + # If that section is not requested, continue + if protocolSections and protocolSectionName not in protocolSections: continue protocol = protocolDetails["Protocol"] result = self.__generateStorageObject( storageName, pluginName, protocolDetails, hideExceptions=hideExceptions ) if result["OK"]: - self.storages.append(result["Value"]) - if pluginName in self.localPlugins: + storageObj = result["Value"] + storageObj.protocolSectionName = protocolSectionName + self.storages[protocolSectionName] = storageObj + if protocolSectionName in self.localProtocolSections: turlProtocols.append(protocol) - requestedLocalPlugins.append(pluginName) - if pluginName in self.remotePlugins: - requestedRemotePlugins.append(pluginName) + requestLocalProtocolSections.append(protocolSectionName) + if protocolSectionName in self.remoteProtocolSections: + requestRemoteProtocolSections.append(protocolSectionName) requestedProtocolDetails.append(protocolDetails) else: gLogger.info(result["Message"]) @@ -163,8 +168,8 @@ def getStorages(self, storageName, pluginList=None, hideExceptions=False): resDict["StorageName"] = self.name resDict["StorageOptions"] = self.options resDict["StorageObjects"] = self.storages - resDict["LocalPlugins"] = requestedLocalPlugins - resDict["RemotePlugins"] = requestedRemotePlugins + resDict["LocalProtocolSections"] = requestLocalProtocolSections + resDict["RemoteProtocolSections"] = requestRemoteProtocolSections resDict["ProtocolOptions"] = requestedProtocolDetails resDict["TurlProtocols"] = turlProtocols return S_OK(resDict) @@ -292,6 +297,7 @@ def _getConfigStorageProtocols(self, storageName, derivedStorageName=None, seCon :return: dictionary of protocols like {protocolSection: {protocolOptions}} """ + # Get the sections res = self.__getProtocolsSections(storageName, seConfigPath=seConfigPath) if not res["OK"]: @@ -312,7 +318,7 @@ def _getConfigStorageProtocols(self, storageName, derivedStorageName=None, seCon return res for protocolSection in res["Value"]: res = self._getConfigStorageProtocolDetails( - derivedStorageName, protocolSection, seConfigPath=SE_CONFIG_PATH, checkAccess=False + derivedStorageName, protocolSection, seConfigPath=SE_CONFIG_PATH ) if not res["OK"]: return res @@ -329,11 +335,24 @@ def _getConfigStorageProtocols(self, storageName, derivedStorageName=None, seCon # If not matched, consider it a new protocol if not inheritanceMatched: self.protocols[protocolSection] = detail + + for protocolSectionName, protocolDict in self.protocols.items(): + # Now update the local and remote protocol lists. + # A warning will be given if the Access option is not set to local or remote. + if protocolDict["Access"].lower() == "remote": + self.remoteProtocolSections.append(protocolSectionName) + elif protocolDict["Access"].lower() == "local": + self.localProtocolSections.append(protocolSectionName) + else: + errStr = ( + "StorageFactory.__getProtocolDetails: The 'Access' option \ + for %s:%s is neither 'local' or 'remote'." + % (storageName, protocolSectionName) + ) + gLogger.warn(errStr) return S_OK(self.protocols) - def _getConfigStorageProtocolDetails( - self, storageName, protocolSection, seConfigPath=SE_CONFIG_PATH, checkAccess=True - ): + def _getConfigStorageProtocolDetails(self, storageName, protocolSection, seConfigPath=SE_CONFIG_PATH): """ Parse the contents of the protocol block @@ -341,7 +360,6 @@ def _getConfigStorageProtocolDetails( :param protocolSection: name of the protocol section to find information :param seConfigPath: the path of the storage section. It can be /Resources/StorageElements or StorageElementBases - :param checkAccess: if not set, don't complain if "Access" is not in the section :return: dictionary of the protocol options """ @@ -370,22 +388,6 @@ def _getConfigStorageProtocolDetails( if voPath: protocolDict["Path"] = voPath - # Now update the local and remote protocol lists. - # A warning will be given if the Access option is not set and the plugin is not already in remote or local. - plugin = protocolDict.get("PluginName", protocolSection) - if protocolDict["Access"].lower() == "remote": - self.remotePlugins.append(plugin) - elif protocolDict["Access"].lower() == "local": - self.localPlugins.append(plugin) - # If it is a derived SE, this is normal, no warning - elif checkAccess and protocolSection not in self.protocols: - errStr = ( - "StorageFactory.__getProtocolDetails: The 'Access' option \ - for %s:%s is neither 'local' or 'remote'." - % (storageName, protocolSection) - ) - gLogger.warn(errStr) - return S_OK(protocolDict) ########################################################################################### diff --git a/src/DIRAC/Resources/Storage/test/FIXME_Test_StorageElement.py b/src/DIRAC/Resources/Storage/test/FIXME_Test_StorageElement.py index 867b2cef026..474e847d4a6 100755 --- a/src/DIRAC/Resources/Storage/test/FIXME_Test_StorageElement.py +++ b/src/DIRAC/Resources/Storage/test/FIXME_Test_StorageElement.py @@ -67,21 +67,21 @@ def test_isValid(self): res = self.storageElement.isValid() self.assertTrue(res["OK"]) - def test_getRemotePlugins(self): + def test_getRemoteProtocolSections(self): print( "\n\n#########################################################" "################\n\n\t\t\tGet remote protocols test\n" ) - res = self.storageElement.getRemotePlugins() + res = self.storageElement.getRemoteProtocolSections() self.assertTrue(res["OK"]) self.assertEqual(type(res["Value"]), list) - def test_getLocalPlugins(self): + def test_getLocalProtocolSections(self): print( "\n\n#########################################################" "################\n\n\t\t\tGet local protocols test\n" ) - res = self.storageElement.getLocalPlugins() + res = self.storageElement.getLocalProtocolSections() self.assertTrue(res["OK"]) self.assertEqual(type(res["Value"]), list) diff --git a/src/DIRAC/Resources/Storage/test/Test_FilePlugin.py b/src/DIRAC/Resources/Storage/test/Test_FilePlugin.py index d8c9d00c29a..fd2c56903e4 100644 --- a/src/DIRAC/Resources/Storage/test/Test_FilePlugin.py +++ b/src/DIRAC/Resources/Storage/test/Test_FilePlugin.py @@ -84,7 +84,7 @@ def setUp( self.basePath = tempfile.mkdtemp(dir="/tmp") # Update the basePath of the plugin - self.se.storages[0].basePath = self.basePath + list(self.se.storages.values())[0].basePath = self.basePath self.srcPath = tempfile.mkdtemp(dir="/tmp") diff --git a/src/DIRAC/Resources/Storage/test/Test_StorageFactory.py b/src/DIRAC/Resources/Storage/test/Test_StorageFactory.py index 43ba00fe9b5..d227e650092 100755 --- a/src/DIRAC/Resources/Storage/test/Test_StorageFactory.py +++ b/src/DIRAC/Resources/Storage/test/Test_StorageFactory.py @@ -4,300 +4,332 @@ __RCSID__ = "$Id$" -import copy -import unittest -import mock -from DIRAC import S_OK, S_ERROR, gConfig, gLogger +import os +import tempfile +import pytest + + +from diraccfg import CFG +import DIRAC +from DIRAC.ConfigurationSystem.private.ConfigurationClient import ConfigurationClient +from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData +from DIRAC import S_OK from DIRAC.Resources.Storage.StorageFactory import StorageFactory -from functools import reduce - -gLogger.setLevel("DEBUG") -dict_cs = { - "Resources": { - "StorageElementBases": { - "CERN-BASE-WITH-TWO-SAME-PLUGINS": { - "BackendType": "Eos", - "SEType": "T0D1", - "AccessProtocol.1": { - "Host": "srm-eoslhcb.cern.ch", - "Port": 8443, - "PluginName": "GFAL2_SRM2", - "Protocol": "srm", - "Access": "remote", - "WSUrl": "/srm/v2/server?SFN:", - }, - "AccessProtocol.2": { - "Host": "eoslhcb.cern.ch", - "Port": 8443, - "PluginName": "GFAL2_SRM2", - "Protocol": "root", - "Access": "remote", - "WSUrl": "/srm/v2/server?SFN:", - }, - }, - # Pure abstract because no path or SpaceToken - "CERN-ABSTRACT": { - "BackendType": "Eos", - "SEType": "T0D1", - "AccessProtocol.1": { - "Host": "srm-eoslhcb.cern.ch", - "Port": 8443, - "PluginName": "GFAL2_SRM2", - "Protocol": "srm", - "Access": "remote", - "WSUrl": "/srm/v2/server?SFN:", - }, - "AccessProtocol.2": { - "Host": "eoslhcb.cern.ch", - "PluginName": "GFAL2_XROOT", - "Protocol": "root", - "Access": "remote", - }, - }, - "CERN-BASE": { - "BackendType": "Eos", - "SEType": "T0D1", - "AccessProtocol.1": { - "Host": "srm-eoslhcb.cern.ch", - "Port": 8443, - "PluginName": "GFAL2_SRM2", - "Protocol": "srm", - "Path": "/eos/lhcb/grid/prod", - "Access": "remote", - "SpaceToken": "LHCb-EOS", - "WSUrl": "/srm/v2/server?SFN:", - }, - }, - }, - "StorageElements": { - # This SE must be in the section above. We kept the backward - # compatibility for a while, but now it is time to remove it - "CERN-BASE-WRONGLOCATION": { - "BackendType": "Eos", - "SEType": "T0D1", - "AccessProtocol.1": { - "Host": "srm-eoslhcb.cern.ch", - "Port": 8443, - "PluginName": "GFAL2_SRM2", - "Protocol": "srm", - "Path": "/eos/lhcb/grid/prod", - "Access": "remote", - "SpaceToken": "LHCb-EOS", - "WSUrl": "/srm/v2/server?SFN:", - }, - }, - # This should not work anymore because - # CERN-BASE-WRONGLOCATION is in StorageElements - "CERN-WRONGLOCATION": { - "BaseSE": "CERN-BASE-WRONGLOCATION", - "AccessProtocol.1": { - "PluginName": "GFAL2_SRM2", - "Path": "/eos/lhcb/grid/prod", - }, - }, - # Alias in StorageElements location should still work - "CERN-WRONGLOCATION-ALIAS": {"Alias": "CERN-BASE-WRONGLOCATION"}, - "CERN-SIMPLE": { - "BackendType": "Eos", - "SEType": "T0D1", - "RemoteAccessProtocol": { - "Host": "srm-eoslhcb.cern.ch", - "Port": 8443, - "PluginName": "GFAL2_SRM2", - "Protocol": "srm", - "Path": "/eos/lhcb/grid/prod", - "Access": "remote", - "SpaceToken": "LHCb-EOS", - "WSUrl": "/srm/v2/server?SFN:", - }, - "LocalAccessProtocol": { - "Host": "eoslhcb.cern.ch", - "PluginName": "File", - "Protocol": "file", - "Path": "/eos/lhcb/grid/prod", - "Access": "local", - "SpaceToken": "LHCb-EOS", - }, - }, - # Just inherit, overwrite a SpaceToken and Path and add an option - "CERN-USER": { - "BaseSE": "CERN-BASE", - "PledgedSpace": 205, - "AccessProtocol.1": { - "PluginName": "GFAL2_SRM2", - "Path": "/eos/lhcb/grid/user", - "SpaceToken": "LHCb_USER", - }, - }, - "CERN-DST": { - "BaseSE": "CERN-BASE", - "AccessProtocol.1": { - "PluginName": "GFAL2_SRM2", - "Path": "/eos/lhcb/grid/prod", - }, - }, - # Does not redefine anything, it is just to give another name - "CERN-NO-DEF": { - "BaseSE": "CERN-BASE", - }, - # Uses the plugin name of the Base SE - "CERN-NO-PLUGIN-NAME": { - "BaseSE": "CERN-BASE", - "AccessProtocol.1": { - "Path": "/eos/lhcb/grid/user", - }, - }, - # Redefines the plugin name in the protocol section - "CERN-BAD-PLUGIN-NAME": { - "BaseSE": "CERN-BASE", - "AccessProtocol.1": { - "PluginName": "AnotherPluginName", - "Path": "/eos/lhcb/grid/prod", - "Access": "local", - }, - }, - # Gives 2 protocol sections with the same plugin name - "CERN-REDEFINE-PLUGIN-NAME": { - "BaseSE": "CERN-BASE", - "AccessProtocol.OtherName": { - "PluginName": "GFAL2_SRM2", - "Path": "/eos/lhcb/grid/other", - "Access": "remote", - }, - }, - # The plugin name of the GFAL2_SRM2 section should be GFAL2_SRM2 - "CERN-USE-PLUGIN-AS-PROTOCOL-NAME": { - "BaseSE": "CERN-BASE", - "GFAL2_SRM2": { - "Host": "srm-eoslhcb.cern.ch", - "Port": 8443, - "Protocol": "srm", - "Path": "/eos/lhcb/grid/user", - "Access": "remote", - "SpaceToken": "LHCb-EOS", - "WSUrl": "/srm/v2/server?SFN:", - }, - }, - # Plugin name should be GFAL2_XROOT here - "CERN-USE-PLUGIN-AS-PROTOCOL-NAME-WITH-PLUGIN-NAME": { - "BaseSE": "CERN-BASE", - "GFAL2_SRM2": { - "Host": "srm-eoslhcb.cern.ch", - "Port": 8443, - "Protocol": "srm", - "Path": "/eos/lhcb/grid/user", - "Access": "remote", - "SpaceToken": "LHCb-EOS", - "PluginName": "GFAL2_XROOT", - "WSUrl": "/srm/v2/server?SFN:", - }, - }, - # Defines two same plugins in two protocol sections - "CERN-CHILD-INHERIT-FROM-BASE-WITH-TWO-SAME-PLUGINS": { - "BaseSE": "CERN-BASE-WITH-TWO-SAME-PLUGINS", - "AccessProtocol.1": { - "Path": "/eos/lhcb/grid/user", - }, - }, - # More because add an extra protocol compared to the parent - "CERN-MORE": { - "BaseSE": "CERN-BASE", - "AccessProtocol.1": { - "PluginName": "GFAL2_SRM2", - "Path": "/eos/lhcb/grid/user", - }, - "AccessProtocol.More": { - "Host": "srm-eoslhcb.cern.ch", - "Port": 8443, - "PluginName": "Extra", - "Protocol": "srm", - "Path": "/eos/lhcb/grid/prod", - "Access": "remote", - "SpaceToken": "LHCb-EOS", - }, - }, - # Inherits from ABSTRACT - "CERN-CHILD": { - "BaseSE": "CERN-ABSTRACT", - "AccessProtocol.1": { - "PluginName": "GFAL2_SRM2", - "Path": "/eos/lhcb/grid/user", - "SpaceToken": "LHCb_USER", - }, - "AccessProtocol.2": { - "PluginName": "GFAL2_XROOT", - "Path": "/eos/lhcb/grid/xrootuser", - }, - }, - }, + + +@pytest.fixture(scope="module", autouse=True) +def generateConfig(): + """ + Generate the configuration that will be used for all the test + """ + + testCfgFileName = os.path.join(tempfile.gettempdir(), "Test_StorageFactory.cfg") + cfgContent = """ + DIRAC + { + VirtualOrganization = lhcb } -} + Resources + { + StorageElementBases + { + CERN-BASE-WITH-TWO-SAME-PLUGINS + { + BackendType = Eos + SEType = T0D1 + AccessProtocol.1 + { + Host = srm-eoslhcb.cern.ch + Port = 8443 + PluginName = GFAL2_SRM2 + Protocol = srm + Access = remote + WSUrl = /srm/v2/server?SFN: + } + AccessProtocol.2 + { + Host = eoslhcb.cern.ch + Port = 8443 + PluginName = GFAL2_SRM2 + Protocol = root + Access = remote + WSUrl = /srm/v2/server?SFN: + } + } + CERN-ABSTRACT + { + BackendType = Eos + SEType = T0D1 + AccessProtocol.1 + { + Host = srm-eoslhcb.cern.ch + Port = 8443 + PluginName = GFAL2_SRM2 + Protocol = srm + Access = remote + WSUrl = /srm/v2/server?SFN: + } + AccessProtocol.2 + { + Host = eoslhcb.cern.ch + PluginName = GFAL2_XROOT + Protocol = root + Access = remote + } + } + CERN-BASE + { + BackendType = Eos + SEType = T0D1 + AccessProtocol.1 + { + Host = srm-eoslhcb.cern.ch + Port = 8443 + PluginName = GFAL2_SRM2 + Protocol = srm + Path = /eos/lhcb/grid/prod + Access = remote + SpaceToken = LHCb-EOS + WSUrl = /srm/v2/server?SFN: + } + } + } + StorageElements + { + CERN-BASE-WRONGLOCATION + { + BackendType = Eos + SEType = T0D1 + AccessProtocol.1 + { + Host = srm-eoslhcb.cern.ch + Port = 8443 + PluginName = GFAL2_SRM2 + Protocol = srm + Path = /eos/lhcb/grid/prod + Access = remote + SpaceToken = LHCb-EOS + WSUrl = /srm/v2/server?SFN: + } + } + CERN-WRONGLOCATION + { + BaseSE = CERN-BASE-WRONGLOCATION + AccessProtocol.1 + { + PluginName = GFAL2_SRM2 + Path = /eos/lhcb/grid/prod + } + } + CERN-WRONGLOCATION-ALIAS + { + Alias = CERN-BASE-WRONGLOCATION + } + CERN-SIMPLE + { + BackendType = Eos + SEType = T0D1 + RemoteAccessProtocol + { + Host = srm-eoslhcb.cern.ch + Port = 8443 + PluginName = GFAL2_SRM2 + Protocol = srm + Path = /eos/lhcb/grid/prod + Access = remote + SpaceToken = LHCb-EOS + WSUrl = /srm/v2/server?SFN: + } + LocalAccessProtocol + { + Host = eoslhcb.cern.ch + PluginName = File + Protocol = file + Path = /eos/lhcb/grid/prod + Access = local + SpaceToken = LHCb-EOS + } + } + CERN-USER + { + BaseSE = CERN-BASE + PledgedSpace = 205 + AccessProtocol.1 + { + PluginName = GFAL2_SRM2 + Path = /eos/lhcb/grid/user + SpaceToken = LHCb_USER + } + } + CERN-DST + { + BaseSE = CERN-BASE + AccessProtocol.1 + { + PluginName = GFAL2_SRM2 + Path = /eos/lhcb/grid/prod + } + } + CERN-NO-DEF + { + BaseSE = CERN-BASE + } + CERN-NO-PLUGIN-NAME + { + BaseSE = CERN-BASE + AccessProtocol.1 + { + Path = /eos/lhcb/grid/user + } + } + CERN-BAD-PLUGIN-NAME + { + BaseSE = CERN-BASE + AccessProtocol.1 + { + PluginName = AnotherPluginName + Path = /eos/lhcb/grid/prod + Access = local + } + } + CERN-REDEFINE-PLUGIN-NAME + { + BaseSE = CERN-BASE + AccessProtocol.OtherName + { + PluginName = GFAL2_SRM2 + Path = /eos/lhcb/grid/other + Access = remote + } + } + CERN-USE-PLUGIN-AS-PROTOCOL-NAME + { + BaseSE = CERN-BASE + GFAL2_SRM2 + { + Host = srm-eoslhcb.cern.ch + Port = 8443 + Protocol = srm + Path = /eos/lhcb/grid/user + Access = remote + SpaceToken = LHCb-EOS + WSUrl = /srm/v2/server?SFN: + } + } + CERN-USE-PLUGIN-AS-PROTOCOL-NAME-WITH-PLUGIN-NAME + { + BaseSE = CERN-BASE + GFAL2_SRM2 + { + Host = srm-eoslhcb.cern.ch + Port = 8443 + Protocol = srm + Path = /eos/lhcb/grid/user + Access = remote + SpaceToken = LHCb-EOS + PluginName = GFAL2_XROOT + WSUrl = /srm/v2/server?SFN: + } + } + CERN-CHILD-INHERIT-FROM-BASE-WITH-TWO-SAME-PLUGINS + { + BaseSE = CERN-BASE-WITH-TWO-SAME-PLUGINS + AccessProtocol.1 + { + Path = /eos/lhcb/grid/user + } + } + CERN-MORE + { + BaseSE = CERN-BASE + AccessProtocol.1 + { + PluginName = GFAL2_SRM2 + Path = /eos/lhcb/grid/user + } + AccessProtocol.More + { + Host = srm-eoslhcb.cern.ch + Port = 8443 + PluginName = Extra + Protocol = srm + Path = /eos/lhcb/grid/prod + Access = remote + SpaceToken = LHCb-EOS + } + } + CERN-CHILD + { + BaseSE = CERN-ABSTRACT + AccessProtocol.1 + { + PluginName = GFAL2_SRM2 + Path = /eos/lhcb/grid/user + SpaceToken = LHCb_USER + } + AccessProtocol.2 + { + PluginName = GFAL2_XROOT + Path = /eos/lhcb/grid/xrootuser + } + } + } + } + """ + with open(testCfgFileName, "w") as f: + f.write(cfgContent) + # Load the configuration + ConfigurationClient(fileToLoadList=[testCfgFileName]) # we replace the configuration by our own one. + yield -class fake_gConfig(object): - @staticmethod - def crawlCS(path): - # How nice :-) - # split the path and recursively dig down the dict_cs dictionary - return reduce(lambda d, e: d.get(e, {}), path.strip("/").split("/"), dict_cs) - - def getValue(self, path, defaultValue=""): - if "StorageElements" not in path and "StorageElementBases" not in path: - return gConfig.getValue(path, defaultValue) - csValue = self.crawlCS(path) - if not csValue: - csValue = defaultValue - return csValue - - def getOptionsDict(self, path): - """Mock the getOptionsDict call of gConfig - It reads from dict_cs - """ - if "StorageElements" not in path and "StorageElementBases" not in path: - return gConfig.getOptionsDict(path) - csSection = self.crawlCS(path) - if not csSection: - return S_ERROR("Not a valid section") - options = dict((opt, val) for opt, val in csSection.items() if not isinstance(val, dict)) - return S_OK(options) - - def getOptions(self, path): - """Mock the getOptions call of gConfig - It reads from dict_cs - """ - if "StorageElements" not in path and "StorageElementBases" not in path: - return gConfig.getOptions(path) - csSection = self.crawlCS(path) - if not csSection: - return S_ERROR("Not a valid section") - options = [opt for opt, val in csSection.items() if not isinstance(val, dict)] - return S_OK(options) - - def getSections(self, path): - """Mock the getOptions call of gConfig - It reads from dict_cs - """ - if "StorageElements" not in path and "StorageElementBases" not in path: - return gConfig.getSections(path) - csSection = self.crawlCS(path) - if not csSection: - return S_ERROR("Not a valid section") - sections = [opt for opt, val in csSection.items() if isinstance(val, dict)] - return S_OK(sections) + try: + os.remove(testCfgFileName) + except OSError: + pass + # SUPER UGLY: one must recreate the CFG objects of gConfigurationData + # not to conflict with other tests that might be using a local dirac.cfg + gConfigurationData.localCFG = CFG() + gConfigurationData.remoteCFG = CFG() + gConfigurationData.mergedCFG = CFG() + gConfigurationData.generateNewVersion() def mock_StorageFactory__generateStorageObject(*args, **kwargs): """Don't really load the plugin, just create an object""" - return S_OK(object()) + # We create this FakeStorage object because if we just + # return a plain object, we get a lot of AttributeErrors + # later in the test + class FakeStorage: + pass + + return S_OK(FakeStorage()) -def mock_resourceStatus_getElementStatus(seName, elementType="StorageElement"): +def mock_resourceStatus_getElementStatus(self, seName, elementType="StorageElement"): """We shut up RSS""" return S_OK({seName: {}}) -sf_gConfig = fake_gConfig() +@pytest.fixture(scope="function", autouse=True) +def monkeypatchForAllTest(monkeypatch): + """ " This fixture will run for all test method and will mockup + a few DMS methods + """ + monkeypatch.setattr( + DIRAC.Resources.Storage.StorageFactory.StorageFactory, + "_StorageFactory__generateStorageObject", + mock_StorageFactory__generateStorageObject, + ) + + monkeypatch.setattr( + DIRAC.ResourceStatusSystem.Client.ResourceStatus.ResourceStatus, + "getElementStatus", + mock_resourceStatus_getElementStatus, + ) + + mandatoryProtocolOptions = { "Access": "", "Host": "", @@ -309,489 +341,443 @@ def mock_resourceStatus_getElementStatus(seName, elementType="StorageElement"): } -@mock.patch("DIRAC.Resources.Storage.StorageFactory.gConfig", new=sf_gConfig) -@mock.patch( - "DIRAC.Resources.Storage.StorageFactory.StorageFactory._StorageFactory__generateStorageObject", - side_effect=mock_StorageFactory__generateStorageObject, -) -@mock.patch( - "DIRAC.ResourceStatusSystem.Client.ResourceStatus.ResourceStatus.getElementStatus", - side_effect=mock_resourceStatus_getElementStatus, -) -class StorageFactoryStandaloneTestCase(unittest.TestCase): - """Base class for the StorageFactory test cases""" - - def test_standalone(self, _sf_generateStorageObject, _rss_getSEStatus): - """Test loading a storage element with everything defined in itself. - It should have two storage plugins - """ - - sf = StorageFactory(vo="lhcb") - storages = sf.getStorages("CERN-SIMPLE") - - self.assertTrue(storages["OK"], storages) - storages = storages["Value"] - - self.assertListEqual(storages["LocalPlugins"], ["File"]) - self.assertListEqual(storages["RemotePlugins"], ["GFAL2_SRM2"]) - - allProtocols = [] - for protocol in ["RemoteAccessProtocol", "LocalAccessProtocol"]: - protocolDef = copy.copy(mandatoryProtocolOptions) - protocolDef.update(fake_gConfig.crawlCS("/Resources/StorageElements/CERN-SIMPLE/%s" % protocol)) - allProtocols.append(protocolDef) - - self.assertEqual(len(storages["ProtocolOptions"]), len(allProtocols)) - self.assertEqual(len(storages["StorageObjects"]), len(allProtocols)) - - self.assertListEqual( - sorted(allProtocols, key=lambda x: x["Host"]), sorted(storages["ProtocolOptions"], key=lambda x: x["Host"]) - ) - self.assertDictEqual(storages["StorageOptions"], {"BackendType": "Eos", "SEType": "T0D1"}) - - -@mock.patch("DIRAC.Resources.Storage.StorageFactory.gConfig", new=sf_gConfig) -@mock.patch( - "DIRAC.Resources.Storage.StorageFactory.StorageFactory._StorageFactory__generateStorageObject", - side_effect=mock_StorageFactory__generateStorageObject, -) -@mock.patch( - "DIRAC.ResourceStatusSystem.Client.ResourceStatus.ResourceStatus.getElementStatus", - side_effect=mock_resourceStatus_getElementStatus, -) -class StorageFactorySimpleInheritance(unittest.TestCase): - """In this class we perform simple inheritance test, with only one - protocol and no extra protocol definition +def test_standalone(): + """Test loading a storage element with everything defined in itself. + It should have two storage plugins + """ + + sf = StorageFactory(vo="lhcb") + storages = sf.getStorages("CERN-SIMPLE") + + assert storages["OK"], storages + storages = storages["Value"] + + assert storages["LocalProtocolSections"] == ["LocalAccessProtocol"] + assert storages["RemoteProtocolSections"] == ["RemoteAccessProtocol"] + + assert len(storages["ProtocolOptions"]) == 2 + assert len(storages["StorageObjects"]) == 2 + + assert storages["StorageOptions"] == {"BackendType": "Eos", "SEType": "T0D1"} + + +def test_simple_inheritance_overwrite(): + """In this test, we load a storage element CERN-USER that inherits from CERN-BASE, + add a storage option, redefine the path and the space token + """ + + sf = StorageFactory(vo="lhcb") + storages = sf.getStorages("CERN-USER") + + assert storages["OK"], storages + storages = storages["Value"] + + assert storages["RemoteProtocolSections"] == ["AccessProtocol.1"] + + # There should be a single protocol + assert len(storages["ProtocolOptions"]) == 1 + # There should be one storage object + assert len(storages["StorageObjects"]) == 1 + + protocolDetail = storages["ProtocolOptions"][0] + # These are the values we expect + assert protocolDetail["Access"] == "remote" + assert protocolDetail["Host"] == "srm-eoslhcb.cern.ch" + assert protocolDetail["Path"] == "/eos/lhcb/grid/user" + assert protocolDetail["PluginName"] == "GFAL2_SRM2" + assert protocolDetail["Port"] == "8443" + assert protocolDetail["Protocol"] == "srm" + assert protocolDetail["SpaceToken"] == "LHCb_USER" + assert protocolDetail["WSUrl"] == "/srm/v2/server?SFN:" + + assert storages["StorageOptions"] == { + "BackendType": "Eos", + "SEType": "T0D1", + "PledgedSpace": "205", + "BaseSE": "CERN-BASE", + } + + +def test_simple_inheritance(): + """In this test, we load a storage element CERN-DST that inherits from CERN-BASE, + and redefine the same value for Path and PluginName + """ + + sf = StorageFactory(vo="lhcb") + storages = sf.getStorages("CERN-DST") + + assert storages["OK"], storages + storages = storages["Value"] + + assert storages["RemoteProtocolSections"] == ["AccessProtocol.1"] + + # There should be a single protocol + assert len(storages["ProtocolOptions"]) == 1 + # There should be one storage object + assert len(storages["StorageObjects"]) == 1 + + protocolDetail = storages["ProtocolOptions"][0] + # These are the values we expect + assert protocolDetail["Access"] == "remote" + assert protocolDetail["Host"] == "srm-eoslhcb.cern.ch" + assert protocolDetail["Path"] == "/eos/lhcb/grid/prod" + assert protocolDetail["PluginName"] == "GFAL2_SRM2" + assert protocolDetail["Port"] == "8443" + assert protocolDetail["Protocol"] == "srm" + assert protocolDetail["SpaceToken"] == "LHCb-EOS" + assert protocolDetail["WSUrl"] == "/srm/v2/server?SFN:" + + assert storages["StorageOptions"] == {"BackendType": "Eos", "SEType": "T0D1", "BaseSE": "CERN-BASE"} + + +def test_pure_inheritance(): + """In this test, we load a storage element CERN-NO-DEF that inherits from CERN-BASE, + but does not redefine ANYTHING. We expect it to be just like the parent """ - def test_simple_inheritance_overwrite(self, _sf_generateStorageObject, _rss_getSEStatus): - """In this test, we load a storage element CERN-USER that inherits from CERN-BASE, - add a storage option, redefine the path and the space token - """ - - sf = StorageFactory(vo="lhcb") - storages = sf.getStorages("CERN-USER") - - self.assertTrue(storages["OK"], storages) - storages = storages["Value"] - - self.assertListEqual(storages["RemotePlugins"], ["GFAL2_SRM2"]) - - # There should be a single protocol - self.assertEqual(len(storages["ProtocolOptions"]), 1) - # There should be one storage object - self.assertEqual(len(storages["StorageObjects"]), 1) - - protocolDetail = storages["ProtocolOptions"][0] - # These are the values we expect - self.assertEqual(protocolDetail["Access"], "remote") - self.assertEqual(protocolDetail["Host"], "srm-eoslhcb.cern.ch") - self.assertEqual(protocolDetail["Path"], "/eos/lhcb/grid/user") - self.assertEqual(protocolDetail["PluginName"], "GFAL2_SRM2") - self.assertEqual(protocolDetail["Port"], 8443) - self.assertEqual(protocolDetail["Protocol"], "srm") - self.assertEqual(protocolDetail["SpaceToken"], "LHCb_USER") - self.assertEqual(protocolDetail["WSUrl"], "/srm/v2/server?SFN:") - - self.assertDictEqual( - storages["StorageOptions"], - {"BackendType": "Eos", "SEType": "T0D1", "PledgedSpace": 205, "BaseSE": "CERN-BASE"}, - ) - - def test_simple_inheritance(self, _sf_generateStorageObject, _rss_getSEStatus): - """In this test, we load a storage element CERN-DST that inherits from CERN-BASE, - and redefine the same value for Path and PluginName - """ - - sf = StorageFactory(vo="lhcb") - storages = sf.getStorages("CERN-DST") - - self.assertTrue(storages["OK"], storages) - storages = storages["Value"] - - self.assertListEqual(storages["RemotePlugins"], ["GFAL2_SRM2"]) - - # There should be a single protocol - self.assertEqual(len(storages["ProtocolOptions"]), 1) - # There should be one storage object - self.assertEqual(len(storages["StorageObjects"]), 1) - - protocolDetail = storages["ProtocolOptions"][0] - # These are the values we expect - self.assertEqual(protocolDetail["Access"], "remote") - self.assertEqual(protocolDetail["Host"], "srm-eoslhcb.cern.ch") - self.assertEqual(protocolDetail["Path"], "/eos/lhcb/grid/prod") - self.assertEqual(protocolDetail["PluginName"], "GFAL2_SRM2") - self.assertEqual(protocolDetail["Port"], 8443) - self.assertEqual(protocolDetail["Protocol"], "srm") - self.assertEqual(protocolDetail["SpaceToken"], "LHCb-EOS") - self.assertEqual(protocolDetail["WSUrl"], "/srm/v2/server?SFN:") - - self.assertDictEqual( - storages["StorageOptions"], {"BackendType": "Eos", "SEType": "T0D1", "BaseSE": "CERN-BASE"} - ) - - def test_pure_inheritance(self, _sf_generateStorageObject, _rss_getSEStatus): - """In this test, we load a storage element CERN-NO-DEF that inherits from CERN-BASE, - but does not redefine ANYTHING. We expect it to be just like the parent - """ - - sf = StorageFactory(vo="lhcb") - storages = sf.getStorages("CERN-NO-DEF") - - self.assertTrue(storages["OK"], storages) - storages = storages["Value"] - - self.assertListEqual(storages["RemotePlugins"], ["GFAL2_SRM2"]) - - # There should be a single protocol - self.assertEqual(len(storages["ProtocolOptions"]), 1) - # There should be one storage object - self.assertEqual(len(storages["StorageObjects"]), 1) - - protocolDetail = storages["ProtocolOptions"][0] - # These are the values we expect - self.assertEqual(protocolDetail["Access"], "remote") - self.assertEqual(protocolDetail["Host"], "srm-eoslhcb.cern.ch") - self.assertEqual(protocolDetail["Path"], "/eos/lhcb/grid/prod") - self.assertEqual(protocolDetail["PluginName"], "GFAL2_SRM2") - self.assertEqual(protocolDetail["Port"], 8443) - self.assertEqual(protocolDetail["Protocol"], "srm") - self.assertEqual(protocolDetail["SpaceToken"], "LHCb-EOS") - self.assertEqual(protocolDetail["WSUrl"], "/srm/v2/server?SFN:") - - self.assertDictEqual( - storages["StorageOptions"], {"BackendType": "Eos", "SEType": "T0D1", "BaseSE": "CERN-BASE"} - ) - - -@mock.patch("DIRAC.Resources.Storage.StorageFactory.gConfig", new=sf_gConfig) -@mock.patch( - "DIRAC.Resources.Storage.StorageFactory.StorageFactory._StorageFactory__generateStorageObject", - side_effect=mock_StorageFactory__generateStorageObject, -) -@mock.patch( - "DIRAC.ResourceStatusSystem.Client.ResourceStatus.ResourceStatus.getElementStatus", - side_effect=mock_resourceStatus_getElementStatus, -) -class StorageFactoryWeirdDefinition(unittest.TestCase): - """In this class, we test very specific cases to highlight inheritance of the StorageElement""" - - def test_no_plugin_name(self, _sf_generateStorageObject, _rss_getSEStatus): - """In this test, we load a storage element CERN-NO-PLUGIN-NAME that inherits from CERN-BASE, - and redefine the same protocol but with no PluginName - """ - - sf = StorageFactory(vo="lhcb") - storages = sf.getStorages("CERN-NO-PLUGIN-NAME") - - self.assertTrue(storages["OK"], storages) - storages = storages["Value"] - - self.assertListEqual(storages["RemotePlugins"], ["GFAL2_SRM2"]) - - expectedProtocols = [ - { - "Access": "remote", - "Host": "srm-eoslhcb.cern.ch", - "Path": "/eos/lhcb/grid/user", - "PluginName": "GFAL2_SRM2", - "Port": 8443, - "Protocol": "srm", - "SpaceToken": "LHCb-EOS", - "WSUrl": "/srm/v2/server?SFN:", - } - ] - - self.assertListEqual(storages["ProtocolOptions"], expectedProtocols) - - def test_bad_plugin_name(self, _sf_generateStorageObject, _rss_getSEStatus): - """In this test, we load a storage element CERN-BAD-PLUGIN-NAME that inherits from CERN-BASE, - and redefine the same protocol but with a different PluginName. - """ - - sf = StorageFactory(vo="lhcb") - storages = sf.getStorages("CERN-BAD-PLUGIN-NAME") - self.assertTrue(storages["OK"], storages) - storages = storages["Value"] - - self.assertListEqual(storages["RemotePlugins"], []) - self.assertListEqual(storages["LocalPlugins"], ["AnotherPluginName"]) - - expectedProtocols = [ - { - "Access": "local", - "Host": "srm-eoslhcb.cern.ch", - "Path": "/eos/lhcb/grid/prod", - "PluginName": "AnotherPluginName", - "Port": 8443, - "Protocol": "srm", - "SpaceToken": "LHCb-EOS", - "WSUrl": "/srm/v2/server?SFN:", - } - ] - - self.assertListEqual(storages["ProtocolOptions"], expectedProtocols) - - def test_redefine_plugin_name(self, _sf_generateStorageObject, _rss_getSEStatus): - """In this test, we load a storage element CERN-REDEFINE-PLUGIN-NAME that inherits from CERN-BASE, - and uses the same Plugin with a different section. - """ - - sf = StorageFactory(vo="lhcb") - storages = sf.getStorages("CERN-REDEFINE-PLUGIN-NAME") - self.assertTrue(storages["OK"], storages) - storages = storages["Value"] - - self.assertListEqual(storages["RemotePlugins"], ["GFAL2_SRM2", "GFAL2_SRM2"]) - - expectedProtocols = [ - { - "Access": "remote", - "Host": "srm-eoslhcb.cern.ch", - "Path": "/eos/lhcb/grid/prod", - "PluginName": "GFAL2_SRM2", - "Port": 8443, - "Protocol": "srm", - "SpaceToken": "LHCb-EOS", - "WSUrl": "/srm/v2/server?SFN:", - }, - { - "Access": "remote", - "Host": "", - "Path": "/eos/lhcb/grid/other", - "PluginName": "GFAL2_SRM2", - "Port": "", - "Protocol": "", - "SpaceToken": "", - "WSUrl": "", - }, - ] - - self.assertListEqual(storages["ProtocolOptions"], expectedProtocols) - - def test_use_plugin_as_protocol_name(self, _sf_generateStorageObject, _rss_getSEStatus): - """In this test, we load a storage element CERN-USE-PLUGIN-AS-PROTOCOL-NAME that inherits from CERN-BASE, - and uses a protocol named as a plugin name, the plugin name is not present. - """ - sf = StorageFactory(vo="lhcb") - storages = sf.getStorages("CERN-USE-PLUGIN-AS-PROTOCOL-NAME") - self.assertTrue(storages["OK"], storages) - storages = storages["Value"] - - self.assertListEqual(storages["RemotePlugins"], ["GFAL2_SRM2", "GFAL2_SRM2"]) - - expectedProtocols = [ - { - "Access": "remote", - "Host": "srm-eoslhcb.cern.ch", - "Path": "/eos/lhcb/grid/prod", - "PluginName": "GFAL2_SRM2", - "Port": 8443, - "Protocol": "srm", - "SpaceToken": "LHCb-EOS", - "WSUrl": "/srm/v2/server?SFN:", - }, - { - "Access": "remote", - "Host": "srm-eoslhcb.cern.ch", - "Path": "/eos/lhcb/grid/user", - "Port": 8443, - "Protocol": "srm", - "SpaceToken": "LHCb-EOS", - "WSUrl": "/srm/v2/server?SFN:", - }, - ] - - self.assertListEqual(storages["ProtocolOptions"], expectedProtocols) - - def test_use_plugin_as_protocol_name_with_plugin_name(self, _sf_generateStorageObject, _rss_getSEStatus): - """In this test, we load a storage element CERN-USE-PLUGIN-AS-PROTOCOL-NAME that inherits from CERN-BASE, - and uses a protocol named as a plugin name, the plugin name is also present. - """ - sf = StorageFactory(vo="lhcb") - storages = sf.getStorages("CERN-USE-PLUGIN-AS-PROTOCOL-NAME-WITH-PLUGIN-NAME") - self.assertTrue(storages["OK"], storages) - storages = storages["Value"] - - self.assertListEqual(storages["RemotePlugins"], ["GFAL2_SRM2", "GFAL2_XROOT"]) - - expectedProtocols = [ - { - "Access": "remote", - "Host": "srm-eoslhcb.cern.ch", - "Path": "/eos/lhcb/grid/prod", - "PluginName": "GFAL2_SRM2", - "Port": 8443, - "Protocol": "srm", - "SpaceToken": "LHCb-EOS", - "WSUrl": "/srm/v2/server?SFN:", - }, - { - "Access": "remote", - "Host": "srm-eoslhcb.cern.ch", - "Path": "/eos/lhcb/grid/user", - "PluginName": "GFAL2_XROOT", - "Port": 8443, - "Protocol": "srm", - "SpaceToken": "LHCb-EOS", - "WSUrl": "/srm/v2/server?SFN:", - }, - ] - - self.assertListEqual(storages["ProtocolOptions"], expectedProtocols) - - def test_more_protocol(self, _sf_generateStorageObject, _rss_getSEStatus): - """In this test, we load a storage element CERN-MORE that inherits from CERN-BASE, - and adds an extra protocol - """ - - sf = StorageFactory(vo="lhcb") - storages = sf.getStorages("CERN-MORE") - self.assertTrue(storages["OK"], storages) - storages = storages["Value"] - - self.assertSetEqual(set(storages["RemotePlugins"]), set(["Extra", "GFAL2_SRM2"])) - - expectedProtocols = [ - { - "Access": "remote", - "Host": "srm-eoslhcb.cern.ch", - "Path": "/eos/lhcb/grid/prod", - "PluginName": "Extra", - "Port": 8443, - "Protocol": "srm", - "SpaceToken": "LHCb-EOS", - "WSUrl": "", - }, - { - "Access": "remote", - "Host": "srm-eoslhcb.cern.ch", - "Path": "/eos/lhcb/grid/user", - "PluginName": "GFAL2_SRM2", - "Port": 8443, - "Protocol": "srm", - "SpaceToken": "LHCb-EOS", - "WSUrl": "/srm/v2/server?SFN:", - }, - ] - - self.assertListEqual( - sorted(storages["ProtocolOptions"], key=lambda x: x["PluginName"]), - expectedProtocols, - ) - - def test_child_inherit_from_base_with_two_same_plugins(self, _sf_generateStorageObject, _rss_getSEStatus): - """In this test, we load a storage element CERN-CHILD-INHERIT-FROM-BASE-WITH-TWO-SAME-PLUGINS that inherits - from CERN-BASE-WITH-TWO-SAME-PLUGINS, using two identical plugin names in two sections. - """ - sf = StorageFactory(vo="lhcb") - storages = sf.getStorages("CERN-CHILD-INHERIT-FROM-BASE-WITH-TWO-SAME-PLUGINS") - self.assertTrue(storages["OK"], storages) - storages = storages["Value"] - - self.assertListEqual(storages["RemotePlugins"], ["GFAL2_SRM2", "GFAL2_SRM2"]) - - expectedProtocols = [ - { - "Access": "remote", - "Host": "srm-eoslhcb.cern.ch", - "Path": "/eos/lhcb/grid/user", - "PluginName": "GFAL2_SRM2", - "Port": 8443, - "Protocol": "srm", - "SpaceToken": "", - "WSUrl": "/srm/v2/server?SFN:", - }, - { - "Access": "remote", - "Host": "eoslhcb.cern.ch", - "Path": "", - "PluginName": "GFAL2_SRM2", - "Port": 8443, - "Protocol": "root", - "SpaceToken": "", - "WSUrl": "/srm/v2/server?SFN:", - }, - ] - - self.assertListEqual(storages["ProtocolOptions"], expectedProtocols) - - def test_baseSE_in_SEDefinition(self, _sf_generateStorageObject, _rss_getSEStatus): - """In this test, a storage inherits from a baseSE which is declared in the - StorageElements section instead of the BaseStorageElements section. - It used to be possible, but we remove this compatibility layer. - """ - - sf = StorageFactory(vo="lhcb") - storages = sf.getStorages("CERN-WRONGLOCATION") - - self.assertFalse(storages["OK"], storages) - - def test_aliasSE_in_SEDefinition(self, _sf_generateStorageObject, _rss_getSEStatus): - """In this test, a storage aliases a baseSE which is declared in the - StorageElements section. That should remain possible - """ - - sf = StorageFactory(vo="lhcb") - storages = sf.getStorages("CERN-WRONGLOCATION-ALIAS") - - self.assertTrue(storages["OK"], storages) - - -@mock.patch("DIRAC.Resources.Storage.StorageFactory.gConfig", new=sf_gConfig) -@mock.patch( - "DIRAC.Resources.Storage.StorageFactory.StorageFactory._StorageFactory__generateStorageObject", - side_effect=mock_StorageFactory__generateStorageObject, -) -@mock.patch( - "DIRAC.ResourceStatusSystem.Client.ResourceStatus.ResourceStatus.getElementStatus", - side_effect=mock_resourceStatus_getElementStatus, -) -class StorageFactoryPureAbstract(unittest.TestCase): - """In this class, we test pure abstract inheritance""" - - def test_pure_abstract(self, _sf_generateStorageObject, _rss_getSEStatus): - """In this test, we load a storage element CERN-CHILD that inherits from CERN-ABSTRACT. - CERN-ABSTRACT has two uncomplete protocols, and CERN-CHILD defines them - """ - - sf = StorageFactory(vo="lhcb") - storages = sf.getStorages("CERN-CHILD") - self.assertTrue(storages["OK"], storages) - storages = storages["Value"] - - self.assertListEqual(storages["RemotePlugins"], ["GFAL2_SRM2", "GFAL2_XROOT"]) - - expectedProtocols = [ - { - "Access": "remote", - "Host": "srm-eoslhcb.cern.ch", - "Path": "/eos/lhcb/grid/user", - "PluginName": "GFAL2_SRM2", - "Port": 8443, - "Protocol": "srm", - "SpaceToken": "LHCb_USER", - "WSUrl": "/srm/v2/server?SFN:", - }, - { - "Access": "remote", - "Host": "eoslhcb.cern.ch", - "Path": "/eos/lhcb/grid/xrootuser", - "PluginName": "GFAL2_XROOT", - "Port": "", - "Protocol": "root", - "SpaceToken": "", - "WSUrl": "", - }, - ] - - self.assertListEqual(storages["ProtocolOptions"], expectedProtocols) - - -if __name__ == "__main__": - suite = unittest.defaultTestLoader.loadTestsFromTestCase(StorageFactoryStandaloneTestCase) - suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(StorageFactorySimpleInheritance)) - suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(StorageFactoryWeirdDefinition)) - suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(StorageFactoryPureAbstract)) - testResult = unittest.TextTestRunner(verbosity=3).run(suite) + sf = StorageFactory(vo="lhcb") + storages = sf.getStorages("CERN-NO-DEF") + + assert storages["OK"], storages + storages = storages["Value"] + + assert storages["RemoteProtocolSections"] == ["AccessProtocol.1"] + + # There should be a single protocol + assert len(storages["ProtocolOptions"]) == 1 + # There should be one storage object + assert len(storages["StorageObjects"]) == 1 + + protocolDetail = storages["ProtocolOptions"][0] + # These are the values we expect + assert protocolDetail["Access"] == "remote" + assert protocolDetail["Host"] == "srm-eoslhcb.cern.ch" + assert protocolDetail["Path"] == "/eos/lhcb/grid/prod" + assert protocolDetail["PluginName"] == "GFAL2_SRM2" + assert protocolDetail["Port"] == "8443" + assert protocolDetail["Protocol"] == "srm" + assert protocolDetail["SpaceToken"] == "LHCb-EOS" + assert protocolDetail["WSUrl"] == "/srm/v2/server?SFN:" + + assert storages["StorageOptions"] == {"BackendType": "Eos", "SEType": "T0D1", "BaseSE": "CERN-BASE"} + + +def test_no_plugin_name(): + """In this test, we load a storage element CERN-NO-PLUGIN-NAME that inherits from CERN-BASE, + and redefine the same protocol but with no PluginName + """ + + sf = StorageFactory(vo="lhcb") + storages = sf.getStorages("CERN-NO-PLUGIN-NAME") + + assert storages["OK"], storages + storages = storages["Value"] + + assert storages["RemoteProtocolSections"] == ["AccessProtocol.1"] + + expectedProtocols = [ + { + "Access": "remote", + "Host": "srm-eoslhcb.cern.ch", + "Path": "/eos/lhcb/grid/user", + "PluginName": "GFAL2_SRM2", + "Port": "8443", + "Protocol": "srm", + "SpaceToken": "LHCb-EOS", + "WSUrl": "/srm/v2/server?SFN:", + } + ] + + assert storages["ProtocolOptions"] == expectedProtocols + + +def test_bad_plugin_name(): + """In this test, we load a storage element CERN-BAD-PLUGIN-NAME that inherits from CERN-BASE, + and redefine the same protocol but with a different PluginName. + """ + + sf = StorageFactory(vo="lhcb") + storages = sf.getStorages("CERN-BAD-PLUGIN-NAME") + assert storages["OK"], storages + storages = storages["Value"] + + assert storages["RemoteProtocolSections"] == [] + assert storages["LocalProtocolSections"] == ["AccessProtocol.1"] + + expectedProtocols = [ + { + "Access": "local", + "Host": "srm-eoslhcb.cern.ch", + "Path": "/eos/lhcb/grid/prod", + "PluginName": "AnotherPluginName", + "Port": "8443", + "Protocol": "srm", + "SpaceToken": "LHCb-EOS", + "WSUrl": "/srm/v2/server?SFN:", + } + ] + + assert storages["ProtocolOptions"] == expectedProtocols + + +def test_redefine_plugin_name(): + """In this test, we load a storage element CERN-REDEFINE-PLUGIN-NAME that inherits from CERN-BASE, + and uses the same Plugin with a different section. + """ + + sf = StorageFactory(vo="lhcb") + storages = sf.getStorages("CERN-REDEFINE-PLUGIN-NAME") + assert storages["OK"], storages + storages = storages["Value"] + + assert storages["RemoteProtocolSections"] == ["AccessProtocol.1", "AccessProtocol.OtherName"] + + expectedProtocols = [ + { + "Access": "remote", + "Host": "srm-eoslhcb.cern.ch", + "Path": "/eos/lhcb/grid/prod", + "PluginName": "GFAL2_SRM2", + "Port": "8443", + "Protocol": "srm", + "SpaceToken": "LHCb-EOS", + "WSUrl": "/srm/v2/server?SFN:", + }, + { + "Access": "remote", + "Host": "", + "Path": "/eos/lhcb/grid/other", + "PluginName": "GFAL2_SRM2", + "Port": "", + "Protocol": "", + "SpaceToken": "", + "WSUrl": "", + }, + ] + + assert storages["ProtocolOptions"] == expectedProtocols + + +def test_use_plugin_as_protocol_name(): + """In this test, we load a storage element CERN-USE-PLUGIN-AS-PROTOCOL-NAME that inherits from CERN-BASE, + and uses a protocol named as a plugin name, the plugin name is not present. + """ + sf = StorageFactory(vo="lhcb") + storages = sf.getStorages("CERN-USE-PLUGIN-AS-PROTOCOL-NAME") + assert storages["OK"], storages + storages = storages["Value"] + + assert storages["RemoteProtocolSections"] == ["AccessProtocol.1", "GFAL2_SRM2"] + + expectedProtocols = [ + { + "Access": "remote", + "Host": "srm-eoslhcb.cern.ch", + "Path": "/eos/lhcb/grid/prod", + "PluginName": "GFAL2_SRM2", + "Port": "8443", + "Protocol": "srm", + "SpaceToken": "LHCb-EOS", + "WSUrl": "/srm/v2/server?SFN:", + }, + { + "Access": "remote", + "Host": "srm-eoslhcb.cern.ch", + "Path": "/eos/lhcb/grid/user", + "Port": "8443", + "Protocol": "srm", + "SpaceToken": "LHCb-EOS", + "WSUrl": "/srm/v2/server?SFN:", + }, + ] + + assert storages["ProtocolOptions"] == expectedProtocols + + +def test_use_plugin_as_protocol_name_with_plugin_name(): + """In this test, we load a storage element CERN-USE-PLUGIN-AS-PROTOCOL-NAME that inherits from CERN-BASE, + and uses a protocol named as a plugin name, the plugin name is also present. + """ + sf = StorageFactory(vo="lhcb") + storages = sf.getStorages("CERN-USE-PLUGIN-AS-PROTOCOL-NAME-WITH-PLUGIN-NAME") + assert storages["OK"], storages + storages = storages["Value"] + + assert storages["RemoteProtocolSections"] == ["AccessProtocol.1", "GFAL2_SRM2"] + + expectedProtocols = [ + { + "Access": "remote", + "Host": "srm-eoslhcb.cern.ch", + "Path": "/eos/lhcb/grid/prod", + "PluginName": "GFAL2_SRM2", + "Port": "8443", + "Protocol": "srm", + "SpaceToken": "LHCb-EOS", + "WSUrl": "/srm/v2/server?SFN:", + }, + { + "Access": "remote", + "Host": "srm-eoslhcb.cern.ch", + "Path": "/eos/lhcb/grid/user", + "PluginName": "GFAL2_XROOT", + "Port": "8443", + "Protocol": "srm", + "SpaceToken": "LHCb-EOS", + "WSUrl": "/srm/v2/server?SFN:", + }, + ] + + assert storages["ProtocolOptions"] == expectedProtocols + + +def test_more_protocol(): + """In this test, we load a storage element CERN-MORE that inherits from CERN-BASE, + and adds an extra protocol + """ + + sf = StorageFactory(vo="lhcb") + storages = sf.getStorages("CERN-MORE") + assert storages["OK"], storages + storages = storages["Value"] + + assert set(storages["RemoteProtocolSections"]) == set(["AccessProtocol.1", "AccessProtocol.More"]) + + expectedProtocols = [ + { + "Access": "remote", + "Host": "srm-eoslhcb.cern.ch", + "Path": "/eos/lhcb/grid/prod", + "PluginName": "Extra", + "Port": "8443", + "Protocol": "srm", + "SpaceToken": "LHCb-EOS", + "WSUrl": "", + }, + { + "Access": "remote", + "Host": "srm-eoslhcb.cern.ch", + "Path": "/eos/lhcb/grid/user", + "PluginName": "GFAL2_SRM2", + "Port": "8443", + "Protocol": "srm", + "SpaceToken": "LHCb-EOS", + "WSUrl": "/srm/v2/server?SFN:", + }, + ] + + assert sorted(storages["ProtocolOptions"], key=lambda x: x["PluginName"]) == expectedProtocols + + +def test_child_inherit_from_base_with_two_same_plugins(): + """In this test, we load a storage element CERN-CHILD-INHERIT-FROM-BASE-WITH-TWO-SAME-PLUGINS that inherits + from CERN-BASE-WITH-TWO-SAME-PLUGINS, using two identical plugin names in two sections. + """ + sf = StorageFactory(vo="lhcb") + storages = sf.getStorages("CERN-CHILD-INHERIT-FROM-BASE-WITH-TWO-SAME-PLUGINS") + assert storages["OK"], storages + storages = storages["Value"] + + assert storages["RemoteProtocolSections"] == ["AccessProtocol.1", "AccessProtocol.2"] + + expectedProtocols = [ + { + "Access": "remote", + "Host": "srm-eoslhcb.cern.ch", + "Path": "/eos/lhcb/grid/user", + "PluginName": "GFAL2_SRM2", + "Port": "8443", + "Protocol": "srm", + "SpaceToken": "", + "WSUrl": "/srm/v2/server?SFN:", + }, + { + "Access": "remote", + "Host": "eoslhcb.cern.ch", + "Path": "", + "PluginName": "GFAL2_SRM2", + "Port": "8443", + "Protocol": "root", + "SpaceToken": "", + "WSUrl": "/srm/v2/server?SFN:", + }, + ] + + assert storages["ProtocolOptions"] == expectedProtocols + + +def test_baseSE_in_SEDefinition(): + """In this test, a storage inherits from a baseSE which is declared in the + StorageElements section instead of the BaseStorageElements section. + It used to be possible, but we remove this compatibility layer. + """ + + sf = StorageFactory(vo="lhcb") + storages = sf.getStorages("CERN-WRONGLOCATION") + + assert not storages["OK"], storages + + +def test_aliasSE_in_SEDefinition(): + """In this test, a storage aliases a baseSE which is declared in the + StorageElements section. That should remain possible + """ + + sf = StorageFactory(vo="lhcb") + storages = sf.getStorages("CERN-WRONGLOCATION-ALIAS") + + assert storages["OK"], storages + + +def test_pure_abstract(): + """In this test, we load a storage element CERN-CHILD that inherits from CERN-ABSTRACT. + CERN-ABSTRACT has two uncomplete protocols, and CERN-CHILD defines them + """ + + sf = StorageFactory(vo="lhcb") + storages = sf.getStorages("CERN-CHILD") + assert storages["OK"], storages + storages = storages["Value"] + + assert storages["RemoteProtocolSections"] == ["AccessProtocol.1", "AccessProtocol.2"] + + expectedProtocols = [ + { + "Access": "remote", + "Host": "srm-eoslhcb.cern.ch", + "Path": "/eos/lhcb/grid/user", + "PluginName": "GFAL2_SRM2", + "Port": "8443", + "Protocol": "srm", + "SpaceToken": "LHCb_USER", + "WSUrl": "/srm/v2/server?SFN:", + }, + { + "Access": "remote", + "Host": "eoslhcb.cern.ch", + "Path": "/eos/lhcb/grid/xrootuser", + "PluginName": "GFAL2_XROOT", + "Port": "", + "Protocol": "root", + "SpaceToken": "", + "WSUrl": "", + }, + ] + + assert storages["ProtocolOptions"] == expectedProtocols + + +def test_getStorages_protocolSections(): + """The idea is to test getStorages with different combinations of + requested protocolSections""" + + sf = StorageFactory(vo="lhcb") + seName = "CERN-CHILD-INHERIT-FROM-BASE-WITH-TWO-SAME-PLUGINS" + res = sf.getStorages(seName) + assert res + allStorages = res["Value"] + remoteProtocolSections = allStorages["RemoteProtocolSections"] + allStorages["LocalProtocolSections"] + + res = sf.getStorages(seName, protocolSections=["AccessProtocol.1", "AccessProtocol.2"]) + assert res["OK"] + specificStorages = res["Value"] diff --git a/src/DIRAC/StorageManagementSystem/Agent/StageMonitorAgent.py b/src/DIRAC/StorageManagementSystem/Agent/StageMonitorAgent.py index b710d5ca2d9..8d104e49c06 100755 --- a/src/DIRAC/StorageManagementSystem/Agent/StageMonitorAgent.py +++ b/src/DIRAC/StorageManagementSystem/Agent/StageMonitorAgent.py @@ -96,7 +96,7 @@ def __monitorStorageElementStageRequests(self, storageElement, seReplicaIDs, rep oAccounting = DataOperation() oAccounting.setStartTime() - res = StorageElement(storageElement, plugins=self.storagePlugins).getFileMetadata(lfnRepIDs) + res = StorageElement(storageElement, protocolSections=self.storagePlugins).getFileMetadata(lfnRepIDs) if not res["OK"]: gLogger.error( "StageMonitor.__monitorStorageElementStageRequests: Completely failed to monitor stage requests for replicas", diff --git a/tests/Integration/Resources/Storage/Test_Resources_GFAL2StorageBase.py b/tests/Integration/Resources/Storage/Test_Resources_GFAL2StorageBase.py index 97768928047..9f249b8175b 100644 --- a/tests/Integration/Resources/Storage/Test_Resources_GFAL2StorageBase.py +++ b/tests/Integration/Resources/Storage/Test_Resources_GFAL2StorageBase.py @@ -74,7 +74,7 @@ if len(posArgs) > 1: AVAILABLE_PLUGINS = posArgs[1].split(",") else: - res = StorageElement(STORAGE_NAME).getPlugins() + res = StorageElement(STORAGE_NAME).getProtocolSections() if not res["OK"]: gLogger.error("Failed fetching available plugins", res["Message"]) sys.exit(2) @@ -151,7 +151,7 @@ def setUp(self, pluginToTest): # When testing for a given plugin, this plugin might not be able to # write or read. In this case, we use this specific plugins # ONLY for the operations it is allowed to - specSE = StorageElement(self.storageName, plugins=pluginToTest) + specSE = StorageElement(self.storageName, protocolSections=pluginToTest) genericSE = StorageElement(self.storageName) pluginProtocol = specSE.protocolOptions[0]["Protocol"] diff --git a/tests/System/StorageTests/Test_Resources_StorageElement.py b/tests/System/StorageTests/Test_Resources_StorageElement.py new file mode 100644 index 00000000000..00deb6abb1c --- /dev/null +++ b/tests/System/StorageTests/Test_Resources_StorageElement.py @@ -0,0 +1,270 @@ +""" +This integration tests will perform basic operations on a storage element, depending on which protocols are available. +It creates a local hierarchy, and then tries to upload, download, remove, get metadata etc + +Examples: +: will test all the gfal2 plugins defined for CERN-GFAL2 +: will test the GFAL2_XROOT plugins defined for CERN-GFAL2 + + +""" +from asyncio import protocols +import os +import sys +import tempfile +import shutil + +import pytest + + +from DIRAC.Core.Base.Script import parseCommandLine + + +argv, sys.argv = sys.argv, ["Dummy"] + +parseCommandLine() +from DIRAC.Resources.Storage.StorageElement import StorageElement +from DIRAC.Core.Security.ProxyInfo import getProxyInfo +from DIRAC.Core.Utilities.Adler import fileAdler +from DIRAC.Core.Utilities.File import getSize +from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getVOForGroup +from DIRAC.Core.Utilities.ReturnValues import S_OK, S_ERROR, convertToReturnValue, returnValueOrRaise + + +# pylint: disable=unspecified-encoding + +# Size in bytes of the file we want to produce +FILE_SIZE = 5 * 1024 # 5kB + + +@pytest.fixture(scope="module") +def prepare_local_testDir(): + """Create the following structure in a local directory + FolderA + -- FolderAA + -- -- FileAA + -- FileA + FolderB + -- FileB + File1 + File2 + File3 + + """ + + proxyInfo = returnValueOrRaise(getProxyInfo()) + + username = proxyInfo["username"] + vo = "" + if "group" in proxyInfo: + vo = getVOForGroup(proxyInfo["group"]) + + destinationPath = f"/{vo}/user/{username[0]}/{username}/gfaltests" + # local path containing test files. There should be a folder called Workflow containing (the files can be simple textfiles) + + def _mul(txt): + """Multiply the input text enough time so that we + reach the expected file size + """ + return txt * (max(1, int(FILE_SIZE / len(txt)))) + + # create the local structure + localWorkDir = tempfile.mkdtemp() + try: + workPath = os.path.join(localWorkDir, "Workflow") + os.mkdir(workPath) + + os.mkdir(os.path.join(workPath, "FolderA")) + with open(os.path.join(workPath, "FolderA", "FileA"), "wt") as f: + f.write(_mul("FileA")) + + os.mkdir(os.path.join(workPath, "FolderA", "FolderAA")) + with open(os.path.join(workPath, "FolderA", "FolderAA", "FileAA"), "w") as f: + f.write(_mul("FileAA")) + + os.mkdir(os.path.join(workPath, "FolderB")) + with open(os.path.join(workPath, "FolderB", "FileB"), "w") as f: + f.write(_mul("FileB")) + + for fn in ["File1", "File2", "File3"]: + with open(os.path.join(workPath, fn), "w") as f: + f.write(_mul(fn)) + + except FileExistsError: + pass + + yield localWorkDir, destinationPath + shutil.rmtree(localWorkDir) + + +@pytest.fixture +def prepare_seObj_fixture(seName, protocolSection, prepare_local_testDir): + + localWorkDir, destinationPath = prepare_local_testDir + + # When testing for a given plugin, this plugin might not be able to + # write or read. In this case, we use this specific plugins + # ONLY for the operations it is allowed to + specSE = StorageElement(seName, protocolSections=protocolSection) + genericSE = StorageElement(seName) + + pluginProtocol = specSE.protocolOptions[0]["Protocol"] + if pluginProtocol in specSE.localAccessProtocolList: + print("Using specific SE with %s only for reading" % protocolSection) + readSE = specSE + else: + print("Plugin %s is not available for read. Use a generic SE" % protocolSection) + readSE = genericSE + + if pluginProtocol in specSE.localWriteProtocolList: + print("Using specific SE with %s only for writing" % protocolSection) + writeSE = specSE + else: + print("Plugin %s is not available for write. Use a generic SE" % protocolSection) + writeSE = genericSE + + # Make sure we are testing the specific plugin at least for one + assert readSE == specSE or writeSE == specSE, "Using only generic SE does not make sense!!" + + yield seName, protocolSection, readSE, writeSE, localWorkDir, destinationPath + + print("==================================================") + print("==== Removing the older Directory ================") + workflow_folder = destinationPath + "/Workflow" + res = writeSE.removeDirectory(workflow_folder) + if not res["OK"]: + print("basicTest.clearDirectory: Workflow folder maybe not empty") + print("==================================================") + + +@pytest.fixture +def fixture_using_other(seName, protocolSection): + print("I am first preparing") + + return seName + "after", protocolSection + "after" + + +def test_storage_element(prepare_seObj_fixture): + """Perform basic operations on a given storage Elements""" + + seName, protocolSection, readSE, writeSE, localWorkDir, destinationPath = prepare_seObj_fixture + print(f"{seName}: {protocolSection}") + assert not seName.isalpha() + + putDir = { + os.path.join(destinationPath, "Workflow/FolderA"): os.path.join(localWorkDir, "Workflow/FolderA"), + os.path.join(destinationPath, "Workflow/FolderB"): os.path.join(localWorkDir, "Workflow/FolderB"), + } + + createDir = [ + os.path.join(destinationPath, "Workflow/FolderA/FolderAA"), + os.path.join(destinationPath, "Workflow/FolderA/FolderABA"), + os.path.join(destinationPath, "Workflow/FolderA/FolderAAB"), + ] + + putFile = { + os.path.join(destinationPath, "Workflow/FolderA/File1"): os.path.join(localWorkDir, "Workflow/File1"), + os.path.join(destinationPath, "Workflow/FolderAA/File1"): os.path.join(localWorkDir, "Workflow/File1"), + os.path.join(destinationPath, "Workflow/FolderBB/File2"): os.path.join(localWorkDir, "Workflow/File2"), + os.path.join(destinationPath, "Workflow/FolderB/File2"): os.path.join(localWorkDir, "Workflow/File2"), + os.path.join(destinationPath, "Workflow/File3"): os.path.join(localWorkDir, "Workflow/File3"), + } + + isFile = { + os.path.join(destinationPath, "Workflow/FolderA/File1"): os.path.join(localWorkDir, "Workflow/File1"), + os.path.join(destinationPath, "Workflow/FolderB/FileB"): os.path.join(localWorkDir, "Workflow/FolderB/FileB"), + } + + listDir = [ + os.path.join(destinationPath, "Workflow"), + os.path.join(destinationPath, "Workflow/FolderA"), + os.path.join(destinationPath, "Workflow/FolderB"), + ] + + getDir = [ + os.path.join(destinationPath, "Workflow/FolderA"), + os.path.join(destinationPath, "Workflow/FolderB"), + ] + + removeFile = [os.path.join(destinationPath, "Workflow/FolderA/File1")] + rmdir = [os.path.join(destinationPath, "Workflow")] + + ##### Computing local adler and size ##### + + fileAdlers = {} + fileSizes = {} + + for lfn, localFn in isFile.items(): + fileAdlers[lfn] = fileAdler(localFn) + fileSizes[lfn] = getSize(localFn) + + ########## uploading directory ############# + res = writeSE.putDirectory(putDir) + assert res["OK"], res + # time.sleep(5) + res = readSE.listDirectory(listDir) + assert any( + os.path.join(destinationPath, "Workflow/FolderA/FileA") in dictKey + for dictKey in res["Value"]["Successful"][os.path.join(destinationPath, "Workflow/FolderA")]["Files"].keys() + ), res + assert any( + os.path.join(destinationPath, "Workflow/FolderB/FileB") in dictKey + for dictKey in res["Value"]["Successful"][os.path.join(destinationPath, "Workflow/FolderB")]["Files"].keys() + ), res + + ########## createDir ############# + res = writeSE.createDirectory(createDir) + assert res["OK"], res + res = res["Value"] + assert res["Successful"][createDir[0]], res + assert res["Successful"][createDir[1]], res + assert res["Successful"][createDir[2]], res + + ######## putFile ######## + res = writeSE.putFile(putFile) + assert res["OK"], res + # time.sleep(5) + res = readSE.isFile(isFile) + assert res["OK"], res + assert all([x for x in res["Value"]["Successful"].values()]) + # self.assertEqual( res['Value']['Successful'][isFile[0]], True ) + # self.assertEqual( res['Value']['Successful'][isFile[1]], True ) + + ######## getMetadata ########### + res = readSE.getFileMetadata(isFile) + assert res["OK"], res + res = res["Value"]["Successful"] + assert any(path in resKey for path in isFile for resKey in res.keys()) + + # Checking that the checksums and sizes are correct + for lfn in isFile: + assert res[lfn]["Checksum"] == fileAdlers[lfn], f"{res[lfn]['Checksum']} != {fileAdlers[lfn]}" + assert res[lfn]["Size"] == fileSizes[lfn], f"{res[lfn]['Size']} != {fileSizes[lfn]}" + + ####### getDirectory ###### + res = readSE.getDirectory(getDir, os.path.join(localWorkDir, "getDir")) + assert res["OK"], res + res = res["Value"] + assert any(getDir[0] in dictKey for dictKey in res["Successful"]), res + assert any(getDir[1] in dictKey for dictKey in res["Successful"]), res + + ###### removeFile ########## + res = writeSE.removeFile(removeFile) + assert res["OK"], res + res = readSE.exists(removeFile) + assert res["OK"], res + assert not res["Value"]["Successful"][removeFile[0]], res + + ###### remove non existing file ##### + res = writeSE.removeFile(removeFile) + assert res["OK"], res + res = readSE.exists(removeFile) + assert res["OK"], res + assert not res["Value"]["Successful"][removeFile[0]], res + + ########### removing directory ########### + res = writeSE.removeDirectory(rmdir, True) + + res = readSE.exists(rmdir) + assert res["OK"], res + assert not res["Value"]["Successful"][rmdir[0]], res diff --git a/tests/System/StorageTests/__init__.py b/tests/System/StorageTests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/System/StorageTests/conftest.py b/tests/System/StorageTests/conftest.py new file mode 100644 index 00000000000..27d3770fc69 --- /dev/null +++ b/tests/System/StorageTests/conftest.py @@ -0,0 +1,35 @@ +# content of conftest.py + + +def pytest_addoption(parser): + parser.addoption( + "--seName", + action="store", + nargs=1, + required=True, + help="StorageElement name to test", + ) + parser.addoption( + "--protocolSection", + action="append", + required=False, + help="Protocol sections of the SE to test", + ) + + +def pytest_generate_tests(metafunc): + if metafunc.definition.name == "test_storage_element": + # Make sure we only test one SE at the time + seNameList = metafunc.config.getoption("seName") + + metafunc.parametrize("seName", seNameList) + + protocolSections = metafunc.config.getoption("protocolSection") + + if not protocolSections: + from DIRAC.Resources.Storage.StorageElement import StorageElement + + protocolSections = StorageElement(seNameList[0]).getProtocolSections()["Value"] + # protocolSections = ["a", "b", "c"] + + metafunc.parametrize("protocolSection", protocolSections)