diff --git a/tests/System/StorageTests/Test_Resources_StorageElement.py b/tests/System/StorageTests/Test_Resources_StorageElement.py new file mode 100644 index 00000000000..00deb6abb1c --- /dev/null +++ b/tests/System/StorageTests/Test_Resources_StorageElement.py @@ -0,0 +1,270 @@ +""" +This integration tests will perform basic operations on a storage element, depending on which protocols are available. +It creates a local hierarchy, and then tries to upload, download, remove, get metadata etc + +Examples: +: will test all the gfal2 plugins defined for CERN-GFAL2 +: will test the GFAL2_XROOT plugins defined for CERN-GFAL2 + + +""" +from asyncio import protocols +import os +import sys +import tempfile +import shutil + +import pytest + + +from DIRAC.Core.Base.Script import parseCommandLine + + +argv, sys.argv = sys.argv, ["Dummy"] + +parseCommandLine() +from DIRAC.Resources.Storage.StorageElement import StorageElement +from DIRAC.Core.Security.ProxyInfo import getProxyInfo +from DIRAC.Core.Utilities.Adler import fileAdler +from DIRAC.Core.Utilities.File import getSize +from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getVOForGroup +from DIRAC.Core.Utilities.ReturnValues import S_OK, S_ERROR, convertToReturnValue, returnValueOrRaise + + +# pylint: disable=unspecified-encoding + +# Size in bytes of the file we want to produce +FILE_SIZE = 5 * 1024 # 5kB + + +@pytest.fixture(scope="module") +def prepare_local_testDir(): + """Create the following structure in a local directory + FolderA + -- FolderAA + -- -- FileAA + -- FileA + FolderB + -- FileB + File1 + File2 + File3 + + """ + + proxyInfo = returnValueOrRaise(getProxyInfo()) + + username = proxyInfo["username"] + vo = "" + if "group" in proxyInfo: + vo = getVOForGroup(proxyInfo["group"]) + + destinationPath = f"/{vo}/user/{username[0]}/{username}/gfaltests" + # local path containing test files. There should be a folder called Workflow containing (the files can be simple textfiles) + + def _mul(txt): + """Multiply the input text enough time so that we + reach the expected file size + """ + return txt * (max(1, int(FILE_SIZE / len(txt)))) + + # create the local structure + localWorkDir = tempfile.mkdtemp() + try: + workPath = os.path.join(localWorkDir, "Workflow") + os.mkdir(workPath) + + os.mkdir(os.path.join(workPath, "FolderA")) + with open(os.path.join(workPath, "FolderA", "FileA"), "wt") as f: + f.write(_mul("FileA")) + + os.mkdir(os.path.join(workPath, "FolderA", "FolderAA")) + with open(os.path.join(workPath, "FolderA", "FolderAA", "FileAA"), "w") as f: + f.write(_mul("FileAA")) + + os.mkdir(os.path.join(workPath, "FolderB")) + with open(os.path.join(workPath, "FolderB", "FileB"), "w") as f: + f.write(_mul("FileB")) + + for fn in ["File1", "File2", "File3"]: + with open(os.path.join(workPath, fn), "w") as f: + f.write(_mul(fn)) + + except FileExistsError: + pass + + yield localWorkDir, destinationPath + shutil.rmtree(localWorkDir) + + +@pytest.fixture +def prepare_seObj_fixture(seName, protocolSection, prepare_local_testDir): + + localWorkDir, destinationPath = prepare_local_testDir + + # When testing for a given plugin, this plugin might not be able to + # write or read. In this case, we use this specific plugins + # ONLY for the operations it is allowed to + specSE = StorageElement(seName, protocolSections=protocolSection) + genericSE = StorageElement(seName) + + pluginProtocol = specSE.protocolOptions[0]["Protocol"] + if pluginProtocol in specSE.localAccessProtocolList: + print("Using specific SE with %s only for reading" % protocolSection) + readSE = specSE + else: + print("Plugin %s is not available for read. Use a generic SE" % protocolSection) + readSE = genericSE + + if pluginProtocol in specSE.localWriteProtocolList: + print("Using specific SE with %s only for writing" % protocolSection) + writeSE = specSE + else: + print("Plugin %s is not available for write. Use a generic SE" % protocolSection) + writeSE = genericSE + + # Make sure we are testing the specific plugin at least for one + assert readSE == specSE or writeSE == specSE, "Using only generic SE does not make sense!!" + + yield seName, protocolSection, readSE, writeSE, localWorkDir, destinationPath + + print("==================================================") + print("==== Removing the older Directory ================") + workflow_folder = destinationPath + "/Workflow" + res = writeSE.removeDirectory(workflow_folder) + if not res["OK"]: + print("basicTest.clearDirectory: Workflow folder maybe not empty") + print("==================================================") + + +@pytest.fixture +def fixture_using_other(seName, protocolSection): + print("I am first preparing") + + return seName + "after", protocolSection + "after" + + +def test_storage_element(prepare_seObj_fixture): + """Perform basic operations on a given storage Elements""" + + seName, protocolSection, readSE, writeSE, localWorkDir, destinationPath = prepare_seObj_fixture + print(f"{seName}: {protocolSection}") + assert not seName.isalpha() + + putDir = { + os.path.join(destinationPath, "Workflow/FolderA"): os.path.join(localWorkDir, "Workflow/FolderA"), + os.path.join(destinationPath, "Workflow/FolderB"): os.path.join(localWorkDir, "Workflow/FolderB"), + } + + createDir = [ + os.path.join(destinationPath, "Workflow/FolderA/FolderAA"), + os.path.join(destinationPath, "Workflow/FolderA/FolderABA"), + os.path.join(destinationPath, "Workflow/FolderA/FolderAAB"), + ] + + putFile = { + os.path.join(destinationPath, "Workflow/FolderA/File1"): os.path.join(localWorkDir, "Workflow/File1"), + os.path.join(destinationPath, "Workflow/FolderAA/File1"): os.path.join(localWorkDir, "Workflow/File1"), + os.path.join(destinationPath, "Workflow/FolderBB/File2"): os.path.join(localWorkDir, "Workflow/File2"), + os.path.join(destinationPath, "Workflow/FolderB/File2"): os.path.join(localWorkDir, "Workflow/File2"), + os.path.join(destinationPath, "Workflow/File3"): os.path.join(localWorkDir, "Workflow/File3"), + } + + isFile = { + os.path.join(destinationPath, "Workflow/FolderA/File1"): os.path.join(localWorkDir, "Workflow/File1"), + os.path.join(destinationPath, "Workflow/FolderB/FileB"): os.path.join(localWorkDir, "Workflow/FolderB/FileB"), + } + + listDir = [ + os.path.join(destinationPath, "Workflow"), + os.path.join(destinationPath, "Workflow/FolderA"), + os.path.join(destinationPath, "Workflow/FolderB"), + ] + + getDir = [ + os.path.join(destinationPath, "Workflow/FolderA"), + os.path.join(destinationPath, "Workflow/FolderB"), + ] + + removeFile = [os.path.join(destinationPath, "Workflow/FolderA/File1")] + rmdir = [os.path.join(destinationPath, "Workflow")] + + ##### Computing local adler and size ##### + + fileAdlers = {} + fileSizes = {} + + for lfn, localFn in isFile.items(): + fileAdlers[lfn] = fileAdler(localFn) + fileSizes[lfn] = getSize(localFn) + + ########## uploading directory ############# + res = writeSE.putDirectory(putDir) + assert res["OK"], res + # time.sleep(5) + res = readSE.listDirectory(listDir) + assert any( + os.path.join(destinationPath, "Workflow/FolderA/FileA") in dictKey + for dictKey in res["Value"]["Successful"][os.path.join(destinationPath, "Workflow/FolderA")]["Files"].keys() + ), res + assert any( + os.path.join(destinationPath, "Workflow/FolderB/FileB") in dictKey + for dictKey in res["Value"]["Successful"][os.path.join(destinationPath, "Workflow/FolderB")]["Files"].keys() + ), res + + ########## createDir ############# + res = writeSE.createDirectory(createDir) + assert res["OK"], res + res = res["Value"] + assert res["Successful"][createDir[0]], res + assert res["Successful"][createDir[1]], res + assert res["Successful"][createDir[2]], res + + ######## putFile ######## + res = writeSE.putFile(putFile) + assert res["OK"], res + # time.sleep(5) + res = readSE.isFile(isFile) + assert res["OK"], res + assert all([x for x in res["Value"]["Successful"].values()]) + # self.assertEqual( res['Value']['Successful'][isFile[0]], True ) + # self.assertEqual( res['Value']['Successful'][isFile[1]], True ) + + ######## getMetadata ########### + res = readSE.getFileMetadata(isFile) + assert res["OK"], res + res = res["Value"]["Successful"] + assert any(path in resKey for path in isFile for resKey in res.keys()) + + # Checking that the checksums and sizes are correct + for lfn in isFile: + assert res[lfn]["Checksum"] == fileAdlers[lfn], f"{res[lfn]['Checksum']} != {fileAdlers[lfn]}" + assert res[lfn]["Size"] == fileSizes[lfn], f"{res[lfn]['Size']} != {fileSizes[lfn]}" + + ####### getDirectory ###### + res = readSE.getDirectory(getDir, os.path.join(localWorkDir, "getDir")) + assert res["OK"], res + res = res["Value"] + assert any(getDir[0] in dictKey for dictKey in res["Successful"]), res + assert any(getDir[1] in dictKey for dictKey in res["Successful"]), res + + ###### removeFile ########## + res = writeSE.removeFile(removeFile) + assert res["OK"], res + res = readSE.exists(removeFile) + assert res["OK"], res + assert not res["Value"]["Successful"][removeFile[0]], res + + ###### remove non existing file ##### + res = writeSE.removeFile(removeFile) + assert res["OK"], res + res = readSE.exists(removeFile) + assert res["OK"], res + assert not res["Value"]["Successful"][removeFile[0]], res + + ########### removing directory ########### + res = writeSE.removeDirectory(rmdir, True) + + res = readSE.exists(rmdir) + assert res["OK"], res + assert not res["Value"]["Successful"][rmdir[0]], res diff --git a/tests/System/StorageTests/__init__.py b/tests/System/StorageTests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/System/StorageTests/conftest.py b/tests/System/StorageTests/conftest.py new file mode 100644 index 00000000000..27d3770fc69 --- /dev/null +++ b/tests/System/StorageTests/conftest.py @@ -0,0 +1,35 @@ +# content of conftest.py + + +def pytest_addoption(parser): + parser.addoption( + "--seName", + action="store", + nargs=1, + required=True, + help="StorageElement name to test", + ) + parser.addoption( + "--protocolSection", + action="append", + required=False, + help="Protocol sections of the SE to test", + ) + + +def pytest_generate_tests(metafunc): + if metafunc.definition.name == "test_storage_element": + # Make sure we only test one SE at the time + seNameList = metafunc.config.getoption("seName") + + metafunc.parametrize("seName", seNameList) + + protocolSections = metafunc.config.getoption("protocolSection") + + if not protocolSections: + from DIRAC.Resources.Storage.StorageElement import StorageElement + + protocolSections = StorageElement(seNameList[0]).getProtocolSections()["Value"] + # protocolSections = ["a", "b", "c"] + + metafunc.parametrize("protocolSection", protocolSections)