Skip to content

Commit

Permalink
refactor (DataManager): getReplicasFromDirectory and __cleanDirectory…
Browse files Browse the repository at this point in the history 
… use getDirectoryDump; remove __getCatalogDirectoryContents
  • Loading branch information
@chaen
chaen committed Jan 18, 2023
1 parent a58e623 commit 8cc2b53
Show file tree
Hide file tree
Showing 2 changed files with 16 additions and 118 deletions.
57 changes: 16 additions & 41 deletions src/DIRAC/DataManagementSystem/Client/DataManager.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -160,24 +160,22 @@ def __cleanDirectory(self, folder):
errStr = "Write access not permitted for this credential."
log.debug(errStr, folder)
return S_ERROR(errStr)
res = self.__getCatalogDirectoryContents([folder], includeDirectories=True)

res = returnSingleResult(self.fileCatalog.getDirectoryDump([folder]))

if not res["OK"]:
return res

if not res["Value"]:
if not (res["Value"]["Files"] or res["Value"]["SubDirs"]):
# folder is empty, just remove it and return
res = returnSingleResult(self.fileCatalog.removeDirectory(folder, recursive=True))
return res

# create a list of folders so that empty folders are also deleted
areDirs = self.fileCatalog.isDirectory(res["Value"])
if not areDirs["OK"]:
return areDirs
listOfFolders = [aDir for aDir in areDirs["Value"]["Successful"] if areDirs["Value"]["Successful"][aDir]]
for lfn in listOfFolders:
res["Value"].pop(lfn)

res = self.removeFile(res["Value"])
listOfFolders = res["Value"]["SubDirs"]
listOfFiles = res["Value"]["Files"]

res = self.removeFile(listOfFiles)
if not res["OK"]:
return res
for lfn, reason in res["Value"]["Failed"].items(): # can be an iterator
Expand Down Expand Up @@ -238,34 +236,6 @@ def __removeStorageDirectory(self, directory, storageElement):
)
return S_OK()

def __getCatalogDirectoryContents(self, directories, includeDirectories=False):
"""ls recursively all files in directories
:param self: self reference
:param list directories: folder names
:param bool includeDirectories: if True includes directories in the return dictionary
:return: S_OK with dict of LFNs and their attribute dictionary
"""
log = self.log.getSubLogger("__getCatalogDirectoryContents")
log.debug("Obtaining the catalog contents for %d directories:" % len(directories))
activeDirs = directories
allFiles = {}
while len(activeDirs) > 0:
currentDir = activeDirs[0]
res = returnSingleResult(self.fileCatalog.listDirectory(currentDir, verbose=True))
activeDirs.remove(currentDir)

if not res["OK"]:
log.debug("Problem getting the %s directory content" % currentDir, res["Message"])
else:
dirContents = res["Value"]
activeDirs.extend(dirContents["SubDirs"])
allFiles.update(dirContents["Files"])
if includeDirectories:
allFiles.update(dirContents["SubDirs"])
log.debug("Found %d files" % len(allFiles))
return S_OK(allFiles)

def getReplicasFromDirectory(self, directory):
"""get all replicas from a given directory
Expand All @@ -276,11 +246,16 @@ def getReplicasFromDirectory(self, directory):
directories = [directory]
else:
directories = directory
res = self.__getCatalogDirectoryContents(directories)
res = returnSingleResult(self.fileCatalog.getDirectoryDump(directories))
if not res["OK"]:
return res
allReplicas = {lfn: metadata["Replicas"] for lfn, metadata in res["Value"].items()} # can be an iterator
return S_OK(allReplicas)

lfns = res["Value"]["Files"]
res = self.fileCatalog.getReplicas(lfns, allStatus=True)
if not res["OK"]:
return res
res["Value"] = res["Value"]["Successful"]
return res

def getFilesFromDirectory(self, directory, days=0, wildcard="*"):
"""get all files from :directory: older than :days: days matching to :wildcard:
Expand Down
77 changes: 0 additions & 77 deletions src/DIRAC/DataManagementSystem/Client/test/Test_Client_DataManagementSystem.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,83 +100,6 @@ def test__getFileTypesCount(self):
# res = self.ci.catalogDirectoryToSE(lfnDir)
# self.assertTrue(res['OK'])

def test__getCatalogDirectoryContents(self):
lfnDirs = ["/this/is/dir1/", "/this/is/dir2/"]

res = self.ci._getCatalogDirectoryContents(lfnDirs)
self.assertTrue(res["OK"])

resExpected = {
"Metadata": {
"/this/is/dir1/file1.txt": {
"MetaData": {
"Checksum": "7149ed85",
"ChecksumType": "Adler32",
"CreationDate": datetime.datetime(2014, 12, 4, 12, 16, 56),
"FileID": 156301805,
"GID": 2695,
"GUID": "6A5C6C86-AD7B-E411-9EDB",
"Mode": 436,
"ModificationDate": datetime.datetime(2014, 12, 4, 12, 16, 56),
"Owner": "phicharp",
"OwnerGroup": "lhcb_prod",
"Size": 206380531,
"Status": "AprioriGood",
"Type": "File",
"UID": 19503,
}
},
"/this/is/dir1/file2.foo.bar": {
"MetaData": {
"Checksum": "7149ed86",
"ChecksumType": "Adler32",
"CreationDate": datetime.datetime(2014, 12, 4, 12, 16, 56),
"FileID": 156301805,
"GID": 2695,
"GUID": "6A5C6C86-AD7B-E411-9EDB",
"Mode": 436,
"ModificationDate": datetime.datetime(2014, 12, 4, 12, 16, 56),
"Owner": "phicharp",
"OwnerGroup": "lhcb_prod",
"Size": 206380532,
"Status": "AprioriGood",
"Type": "File",
"UID": 19503,
}
},
"/this/is/dir2/subdir1/file3.pippo": {
"MetaData": {
"Checksum": "7149ed86",
"ChecksumType": "Adler32",
"CreationDate": datetime.datetime(2014, 12, 4, 12, 16, 56),
"FileID": 156301805,
"GID": 2695,
"GUID": "6A5C6C86-AD7B-E411-9EDB",
"Mode": 436,
"ModificationDate": datetime.datetime(2014, 12, 4, 12, 16, 56),
"Owner": "phicharp",
"OwnerGroup": "lhcb_prod",
"Size": 206380532,
"Status": "AprioriGood",
"Type": "File",
"UID": 19503,
}
},
},
"Replicas": {
"/this/is/dir1/file1.txt": {
"SE1": "smr://srm.SE1.ch:8443/srm/v2/server?SFN=/this/is/dir1/file1.txt",
"SE2": "smr://srm.SE2.fr:8443/srm/v2/server?SFN=/this/is/dir1/file1.txt",
},
"/this/is/dir1/file2.foo.bar": {
"SE1": "smr://srm.SE1.ch:8443/srm/v2/server?SFN=/this/is/dir1/file2.foo.bar",
"SE3": "smr://srm.SE3.es:8443/srm/v2/server?SFN=/this/is/dir1/file2.foo.bar",
},
},
}

self.assertEqual(res["Value"], resExpected)


if __name__ == "__main__":
suite = unittest.defaultTestLoader.loadTestsFromTestCase(UtilitiesTestCase)
Expand Down

0 comments on commit 8cc2b53

Please sign in to comment.