diff --git a/src/tesk_core/filer.py b/src/tesk_core/filer.py index 745c03d..4af12df 100755 --- a/src/tesk_core/filer.py +++ b/src/tesk_core/filer.py @@ -13,6 +13,7 @@ import requests from tesk_core.exception import UnknownProtocol, FileProtocolDisabled import shutil +from glob import glob from tesk_core.path import containerPath, getPath, fileEnabled try: @@ -157,6 +158,21 @@ def copyDir(src, dst): shutil.copytree(src, dst) +def copyFile(src, dst): + ''' + Limitations of shutil.copy: + + It does not interpret * as a glob, but as a character. + ''' + + # If there is any * in 'dst', use only the dirname (base path) + p = re.compile('.*\*.*') + if p.match(dst): + dst=os.path.dirname(dst) + + for file in glob(src): + shutil.copy(file, dst) + class FileTransput(Transput): def __init__(self, path, url, ftype): @@ -172,7 +188,7 @@ def transfer(self, copyFn, src, dst): def download_file(self): self.transfer(shutil.copy , self.urlContainerPath , self.path) def download_dir(self): self.transfer(copyDir , self.urlContainerPath , self.path) - def upload_file(self): self.transfer(shutil.copy , self.path , self.urlContainerPath) + def upload_file(self): self.transfer(copyFile , self.path , self.urlContainerPath) def upload_dir(self): self.transfer(copyDir , self.path , self.urlContainerPath) @@ -454,8 +470,8 @@ def process_file(ttype, filedata): scheme = urlparse(filedata['url']).scheme if scheme == '': - logging.error('Could not determine protocol for url: "%s"', filedata['url']) - return 1 + logging.info('Could not determine protocol for url: "%s", assuming "file"', filedata['url']) + scheme='file' trans = newTransput(scheme) diff --git a/tests/test_filer.py b/tests/test_filer.py index 9aebe18..27eb92b 100644 --- a/tests/test_filer.py +++ b/tests/test_filer.py @@ -1,6 +1,6 @@ import unittest from tesk_core.filer import newTransput, FTPTransput, HTTPTransput, FileTransput,\ - process_file, logConfig, getPath, copyDir + process_file, logConfig, getPath, copyDir, copyFile from tesk_core.exception import UnknownProtocol, InvalidHostPath,\ FileProtocolDisabled from tesk_core.path import containerPath @@ -94,8 +94,8 @@ def test_upload_dir(self, copyMock, copyDirMock): , '/transfer/tmphrtip1o8') @patch('tesk_core.filer.copyDir') - @patch('tesk_core.filer.shutil.copy') - def test_upload_file(self, copyMock, copyDirMock): + @patch('tesk_core.filer.copyFile') + def test_upload_file(self, copyFileMock, copyDirMock): filedata = { "url": "file:///home/tfga/workspace/cwl-tes/tmphrtip1o8/md5", @@ -108,10 +108,29 @@ def test_upload_file(self, copyMock, copyDirMock): copyDirMock.assert_not_called() - copyMock.assert_called_once_with( '/TclSZU/md5' + copyFileMock.assert_called_once_with( '/TclSZU/md5' , '/transfer/tmphrtip1o8/md5') + @patch('tesk_core.filer.copyDir') + @patch('tesk_core.filer.copyFile') + def test_upload_file_glob(self, copyFileMock, copyDirMock): + + filedata = { + "url": "file:///home/tfga/workspace/cwl-tes/tmphrtip1o8/md5*", + "path": "/TclSZU/md5*", + "type": "FILE", + "name": "stdout" + } + + process_file('outputs', filedata) + + copyDirMock.assert_not_called() + + copyFileMock.assert_called_once_with( '/TclSZU/md5*' + , '/transfer/tmphrtip1o8/md5*') + + def test_copyDir(self): def rmDir(d): @@ -170,6 +189,11 @@ def test_getPath(self): self.assertEquals( getPath('file:///home/tfga/workspace/cwl-tes/tmphrtip1o8/md5') , '/home/tfga/workspace/cwl-tes/tmphrtip1o8/md5') + def test_getPathNoScheme(self): + + self.assertEquals( getPath('/home/tfga/workspace/cwl-tes/tmphrtip1o8/md5') + , '/home/tfga/workspace/cwl-tes/tmphrtip1o8/md5') + def test_containerPath(self):