From 258080189df4eb603f3af4301fabfe46c66c9a85 Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Tue, 15 Jun 2021 19:24:51 +0200 Subject: [PATCH 01/12] Fix handling Subprocess calls which produce invalid output --- src/DIRAC/Core/Utilities/Subprocess.py | 11 +++++----- .../Core/Utilities/test/Test_Subprocess.py | 21 ++++++++++++++++++- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/src/DIRAC/Core/Utilities/Subprocess.py b/src/DIRAC/Core/Utilities/Subprocess.py index a34dde68c5d..4e3225e7de6 100644 --- a/src/DIRAC/Core/Utilities/Subprocess.py +++ b/src/DIRAC/Core/Utilities/Subprocess.py @@ -379,7 +379,11 @@ def __readFromFile(self, fd, baseLength): nB = fd.read(1) if not nB: break - dataString += nB.decode() + try: + dataString += nB.decode() + except UnicodeDecodeError as e: + self.log.warn("Unicode decode error in readFromFile", "(%r): %r" % (e, dataString)) + dataString += nB.decode("utf-8", "replace") # break out of potential infinite loop, indicated by dataString growing beyond reason if len(dataString) + baseLength > self.bufferLimit: self.log.error("DataString is getting too long (%s): %s " % (len(dataString), dataString[-10000:])) @@ -426,10 +430,7 @@ def systemCall(self, cmdSeq, callbackFunction=None, shell=False, env=None): self.cmdSeq = cmdSeq self.callback = callbackFunction - if sys.platform.find("win") == 0: - closefd = False - else: - closefd = True + closefd = sys.platform.find("win") != 0 try: self.child = subprocess.Popen(self.cmdSeq, shell=shell, diff --git a/src/DIRAC/Core/Utilities/test/Test_Subprocess.py b/src/DIRAC/Core/Utilities/test/Test_Subprocess.py index 28e80413e2a..f5813adfc7a 100644 --- a/src/DIRAC/Core/Utilities/test/Test_Subprocess.py +++ b/src/DIRAC/Core/Utilities/test/Test_Subprocess.py @@ -27,7 +27,7 @@ from subprocess import Popen # SUT -from DIRAC.Core.Utilities.Subprocess import systemCall, shellCall, pythonCall, getChildrenPIDs +from DIRAC.Core.Utilities.Subprocess import systemCall, shellCall, pythonCall, getChildrenPIDs, Subprocess # Mark this entire module as slow pytestmark = pytest.mark.slow @@ -66,3 +66,22 @@ def test_getChildrenPIDs(): assert isinstance(p, int) mainProcess.wait() + + +def test_decodingCommandOutput(): + retVal = systemCall(10, ["echo", "-e", "-n", r"\xdf"]) + assert retVal["OK"] + assert retVal["Value"] == (0, u"\ufffd", "") + + retVal = systemCall(10, ["echo", "-e", r"\xdf"]) + assert retVal["OK"] + assert retVal["Value"] == (0, u"\ufffd\n", "") + + sp = Subprocess() + retVal = sp.systemCall(r"echo -e -n '\xdf' >&2", shell=True) + assert retVal["OK"] + assert retVal["Value"] == (0, "", u"\ufffd") + + retVal = sp.systemCall(r"echo -e '\xdf' >&2", shell=True) + assert retVal["OK"] + assert retVal["Value"] == (0, "", u"\ufffd\n") From 99e6dabd4befb7bf79f505eec097e31eee315888 Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Wed, 16 Jun 2021 22:43:25 +0200 Subject: [PATCH 02/12] Try to fix test for bash's builtin echo function --- src/DIRAC/Core/Utilities/test/Test_Subprocess.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/DIRAC/Core/Utilities/test/Test_Subprocess.py b/src/DIRAC/Core/Utilities/test/Test_Subprocess.py index f5813adfc7a..739b7fc2efc 100644 --- a/src/DIRAC/Core/Utilities/test/Test_Subprocess.py +++ b/src/DIRAC/Core/Utilities/test/Test_Subprocess.py @@ -78,10 +78,6 @@ def test_decodingCommandOutput(): assert retVal["Value"] == (0, u"\ufffd\n", "") sp = Subprocess() - retVal = sp.systemCall(r"echo -e -n '\xdf' >&2", shell=True) + retVal = sp.systemCall(r"""python -c 'import os; os.fdopen(2, "wb").write(b"\xdf")'""", shell=True) assert retVal["OK"] assert retVal["Value"] == (0, "", u"\ufffd") - - retVal = sp.systemCall(r"echo -e '\xdf' >&2", shell=True) - assert retVal["OK"] - assert retVal["Value"] == (0, "", u"\ufffd\n") From 005eb25eb0aa5943d3d5320b7be5364f46c60a4b Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Wed, 16 Jun 2021 22:55:50 +0200 Subject: [PATCH 03/12] Install an older commit of rucio --- environment-py3.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment-py3.yml b/environment-py3.yml index e753c00f490..47ae78486b6 100644 --- a/environment-py3.yml +++ b/environment-py3.yml @@ -93,5 +93,5 @@ dependencies: - git+https://gitlab.cern.ch/cburr/fts-rest.git@python3 # Doesn't support Python 3.9 # - rucio-clients >=1.25.6 - - git+https://github.com/rucio/rucio.git@master + - git+https://github.com/rucio/rucio.git@eafde85bfbfe51fb22c7774e2af8deda3fb05e81 - -e . From 42e47d1e7c05798e8e0eaa9cd2098227536781a5 Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Thu, 17 Jun 2021 09:36:24 +0200 Subject: [PATCH 04/12] Prevent using m2crypto 0.38.0 to avoid https://github.com/DIRACGrid/DIRAC/issues/5204 --- environment-py3.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment-py3.yml b/environment-py3.yml index 47ae78486b6..f1c7f97aed3 100644 --- a/environment-py3.yml +++ b/environment-py3.yml @@ -19,7 +19,7 @@ dependencies: - elasticsearch-dsl - future - gitpython >=2.1.0 - - m2crypto >=0.36 + - m2crypto >=0.36,!=0.38.0 - matplotlib - numpy - pexpect >=4.0.1 From d2deb160d32ebf3d7ce1ffd872dd5a749c4a4603 Mon Sep 17 00:00:00 2001 From: fstagni Date: Thu, 17 Jun 2021 08:22:28 +0200 Subject: [PATCH 05/12] @@ --- environment-py3.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment-py3.yml b/environment-py3.yml index e753c00f490..47ae78486b6 100644 --- a/environment-py3.yml +++ b/environment-py3.yml @@ -93,5 +93,5 @@ dependencies: - git+https://gitlab.cern.ch/cburr/fts-rest.git@python3 # Doesn't support Python 3.9 # - rucio-clients >=1.25.6 - - git+https://github.com/rucio/rucio.git@master + - git+https://github.com/rucio/rucio.git@eafde85bfbfe51fb22c7774e2af8deda3fb05e81 - -e . From 3642cabba688483dd99938c32a3a4b90fb17c6ff Mon Sep 17 00:00:00 2001 From: fstagni Date: Thu, 17 Jun 2021 10:52:04 +0200 Subject: [PATCH 06/12] if no entrypoint, at least DIRAC --- src/DIRAC/Core/Utilities/Extensions.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/DIRAC/Core/Utilities/Extensions.py b/src/DIRAC/Core/Utilities/Extensions.py index 9ed27fbeec1..695030f2850 100644 --- a/src/DIRAC/Core/Utilities/Extensions.py +++ b/src/DIRAC/Core/Utilities/Extensions.py @@ -158,10 +158,13 @@ def _extensionsByPriorityPy3(): from importlib import metadata # pylint: disable=no-name-in-module priorties = defaultdict(list) - for entrypoint in set(metadata.entry_points()['dirac']): - extensionName = entrypointToExtension(entrypoint) - extension_metadata = entrypoint.load()() - priorties[extension_metadata["priority"]].append(extensionName) + try: + for entrypoint in set(metadata.entry_points()['dirac']): + extensionName = entrypointToExtension(entrypoint) + extension_metadata = entrypoint.load()() + priorties[extension_metadata["priority"]].append(extensionName) + except KeyError: + return ['DIRAC'] extensions = [] for priority, extensionNames in sorted(priorties.items()): From 7361bd8c1b769a093791ce47b2b12f5149f9296b Mon Sep 17 00:00:00 2001 From: fstagni Date: Thu, 17 Jun 2021 10:52:16 +0200 Subject: [PATCH 07/12] exclude rucio agents --- tests/Jenkins/dirac_ci.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Jenkins/dirac_ci.sh b/tests/Jenkins/dirac_ci.sh index 8beec165e0d..70b20b1365e 100644 --- a/tests/Jenkins/dirac_ci.sh +++ b/tests/Jenkins/dirac_ci.sh @@ -366,7 +366,7 @@ fullInstallDIRAC() { dirac-admin-allow-se SE-1 SE-2 S3-DIRECT S3-INDIRECT --All #agents - findAgents + findAgents 'exclude' 'Rucio' if ! diracAgents; then echo "ERROR: diracAgents failed" exit 1 From b524968905d2c87b8e6fe8984f60c65bde7f0fca Mon Sep 17 00:00:00 2001 From: fstagni Date: Thu, 17 Jun 2021 13:52:37 +0200 Subject: [PATCH 08/12] just better order --- .../Agent/JobAgent.py | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/src/DIRAC/WorkloadManagementSystem/Agent/JobAgent.py b/src/DIRAC/WorkloadManagementSystem/Agent/JobAgent.py index 6b2e9586003..c8bbf78e2cf 100755 --- a/src/DIRAC/WorkloadManagementSystem/Agent/JobAgent.py +++ b/src/DIRAC/WorkloadManagementSystem/Agent/JobAgent.py @@ -367,6 +367,18 @@ def execute(self): return self._rescheduleFailedJob( jobID, errorMsg, self.stopOnApplicationFailure) + gridCE = gConfig.getValue('/LocalSite/GridCE', '') + if gridCE: + jobReport.setJobParameter(par_name='GridCE', + par_value=gridCE, + sendFlag=False) + + queue = gConfig.getValue('/LocalSite/CEQueue', '') + if queue: + jobReport.setJobParameter(par_name='CEQueue', + par_value=queue, + sendFlag=False) + self.log.debug('Before self._submitJob() (%sCE)' % (self.ceName)) result_submitJob = self._submitJob( jobID=jobID, @@ -573,15 +585,6 @@ def _submitJob(self, jobID, jobParams, resourceParams, optimizerParams, jobReport.setJobStatus(status=JobStatus.MATCHED, minorStatus='Submitting To CE') - gridCE = gConfig.getValue('/LocalSite/GridCE', '') - queue = gConfig.getValue('/LocalSite/CEQueue', '') - jobReport.setJobParameter(par_name='GridCE', - par_value=gridCE, - sendFlag=False) - jobReport.setJobParameter(par_name='CEQueue', - par_value=queue, - sendFlag=False) - self.log.info('Submitting JobWrapper', '%s to %sCE' % (os.path.basename(wrapperFile), self.ceName)) From f80290e24ce38422530a1f43e34b51e089be082a Mon Sep 17 00:00:00 2001 From: fstagni Date: Thu, 17 Jun 2021 14:59:15 +0200 Subject: [PATCH 09/12] fix SiteInspectorAgent --- .../Agent/ElementInspectorAgent.py | 2 +- .../Agent/SiteInspectorAgent.py | 32 +++++++++---------- .../ResourceStatusSystem/Client/SiteStatus.py | 1 + .../Command/PropagationCommand.py | 4 +-- 4 files changed, 19 insertions(+), 20 deletions(-) diff --git a/src/DIRAC/ResourceStatusSystem/Agent/ElementInspectorAgent.py b/src/DIRAC/ResourceStatusSystem/Agent/ElementInspectorAgent.py index d5d9a71357d..92b584838e3 100644 --- a/src/DIRAC/ResourceStatusSystem/Agent/ElementInspectorAgent.py +++ b/src/DIRAC/ResourceStatusSystem/Agent/ElementInspectorAgent.py @@ -147,7 +147,7 @@ def execute(self): try: future.result() except Exception as exc: - self.log.error('%d generated an exception: %s' % (transID, exc)) + self.log.exception('%s generated an exception: %s' % (transID, exc)) else: self.log.info('Processed', transID) diff --git a/src/DIRAC/ResourceStatusSystem/Agent/SiteInspectorAgent.py b/src/DIRAC/ResourceStatusSystem/Agent/SiteInspectorAgent.py index 6adafc2007b..16a74591746 100644 --- a/src/DIRAC/ResourceStatusSystem/Agent/SiteInspectorAgent.py +++ b/src/DIRAC/ResourceStatusSystem/Agent/SiteInspectorAgent.py @@ -51,27 +51,27 @@ def __init__(self, *args, **kwargs): AgentModule.__init__(self, *args, **kwargs) - self.siteClient = None + self.rsClient = None self.clients = {} def initialize(self): """ Standard initialize. """ - res = ObjectLoader().loadObject('DIRAC.ResourceStatusSystem.Client.SiteStatus') - if not res['OK']: - self.log.error('Failed to load SiteStatus class: %s' % res['Message']) - return res - siteStatusClass = res['Value'] - res = ObjectLoader().loadObject('DIRAC.ResourceStatusSystem.Client.ResourceManagementClient') if not res['OK']: self.log.error('Failed to load ResourceManagementClient class: %s' % res['Message']) return res rmClass = res['Value'] - self.siteClient = siteStatusClass() - self.clients['SiteStatus'] = siteStatusClass() + res = ObjectLoader().loadObject('DIRAC.ResourceStatusSystem.Client.ResourceStatusClient') + if not res['OK']: + self.log.error('Failed to load ResourceStatusClient class: %s' % res['Message']) + return res + rsClass = res['Value'] + + self.rsClient = rsClass() + self.clients['ResourceStatusClient'] = rsClass() self.clients['ResourceManagementClient'] = rmClass() maxNumberOfThreads = self.am_getOption('maxNumberOfThreads', 15) @@ -85,15 +85,11 @@ def execute(self): It gets the sites from the Database which are eligible to be re-checked. """ - res = self.siteClient.getSites('All') - if not res['OK']: - return res - utcnow = datetime.datetime.utcnow().replace(microsecond=0) future_to_element = {} # get the current status - res = self.siteClient.getSiteStatuses(res['Value']) + res = self.rsClient.selectStatusElement('Site', 'Status') if not res['OK']: return res @@ -119,7 +115,8 @@ def execute(self): self.log.verbose('"%s" # %s # %s' % (siteDict['Name'], siteDict['Status'], siteDict['LastCheckTime'])) - lowerElementDict = {} + + lowerElementDict = {'element': 'Site'} for key, value in siteDict.items(): if len(key) >= 2: # VO ! lowerElementDict[key[0].lower() + key[1:]] = value @@ -132,7 +129,7 @@ def execute(self): try: future.result() except Exception as exc: - self.log.error('%d generated an exception: %s' % (transID, exc)) + self.log.exception('%s generated an exception: %s' % (transID, exc)) else: self.log.info('Processed', transID) @@ -147,8 +144,9 @@ def _execute(self, site): pep = PEP(clients=self.clients) self.log.verbose( - '%s ( status=%s / statusType=%s ) being processed' % ( + '%s ( VO=%s / status=%s / statusType=%s ) being processed' % ( site['name'], + site['vO'], site['status'], site['statusType'])) diff --git a/src/DIRAC/ResourceStatusSystem/Client/SiteStatus.py b/src/DIRAC/ResourceStatusSystem/Client/SiteStatus.py index 17799469445..79f64930e0e 100644 --- a/src/DIRAC/ResourceStatusSystem/Client/SiteStatus.py +++ b/src/DIRAC/ResourceStatusSystem/Client/SiteStatus.py @@ -41,6 +41,7 @@ class SiteStatus(object): * getUsableSites * getSites """ + def __init__(self): """ Constructor, initializes the rssClient. diff --git a/src/DIRAC/ResourceStatusSystem/Command/PropagationCommand.py b/src/DIRAC/ResourceStatusSystem/Command/PropagationCommand.py index 63eb38c6722..d4d278619cb 100644 --- a/src/DIRAC/ResourceStatusSystem/Command/PropagationCommand.py +++ b/src/DIRAC/ResourceStatusSystem/Command/PropagationCommand.py @@ -28,10 +28,10 @@ def doNew(self, masterParams=None): def doCache(self): - if not self.args['site']: + if not self.args['name']: return S_ERROR('site was not found in args') - site = self.args['site'] + site = self.args['name'] elements = CSHelpers.getSiteElements(site) From d4a2858e8c686f23033c78965fb257399900ee0e Mon Sep 17 00:00:00 2001 From: fstagni Date: Thu, 17 Jun 2021 15:47:47 +0200 Subject: [PATCH 10/12] fix test SiteInspectorAgent --- .../Agent/test/Test_Agent_SiteInspectorAgent.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/DIRAC/ResourceStatusSystem/Agent/test/Test_Agent_SiteInspectorAgent.py b/src/DIRAC/ResourceStatusSystem/Agent/test/Test_Agent_SiteInspectorAgent.py index 6629a10fa43..d17aa56f9c7 100644 --- a/src/DIRAC/ResourceStatusSystem/Agent/test/Test_Agent_SiteInspectorAgent.py +++ b/src/DIRAC/ResourceStatusSystem/Agent/test/Test_Agent_SiteInspectorAgent.py @@ -22,6 +22,7 @@ site = { 'status': 'status', 'name': 'site', + 'vO': 'some_vo', 'site': 'site', 'element': 'Site', 'statusType': 'all', From 83da2728592f9c0c83688d7d4e36f19c1b48a295 Mon Sep 17 00:00:00 2001 From: fstagni Date: Tue, 15 Jun 2021 11:17:35 +0200 Subject: [PATCH 11/12] fix test --- tests/Integration/WorkloadManagementSystem/Test_Client_WMS.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Integration/WorkloadManagementSystem/Test_Client_WMS.py b/tests/Integration/WorkloadManagementSystem/Test_Client_WMS.py index 070bc8755d2..658c39bae9a 100644 --- a/tests/Integration/WorkloadManagementSystem/Test_Client_WMS.py +++ b/tests/Integration/WorkloadManagementSystem/Test_Client_WMS.py @@ -356,7 +356,7 @@ def test_JobStateUpdateAndJobMonitoringMultuple(self): resJG_olderThanOneYear) res = jobMonitor.getStates() self.assertTrue(res['OK'], res.get('Message')) - self.assertTrue(sorted(res['Value']) in [[JobStatus.RECEIVED], sorted([JobStatus.RECEIVED, JobStatus.WAITING])], + self.assertTrue(sorted(res['Value']) in [[JobStatus.RECEIVED], sorted([JobStatus.RECEIVED, JobStatus.KILLED])], res['Value']) res = jobMonitor.getMinorStates() self.assertTrue(res['OK'], res.get('Message')) From 757f751d27e92ccdbd78d42c2fa49366dcb9ea6e Mon Sep 17 00:00:00 2001 From: fstagni Date: Thu, 17 Jun 2021 16:34:14 +0200 Subject: [PATCH 12/12] Revert "if no entrypoint, at least DIRAC" This reverts commit 3642cabba688483dd99938c32a3a4b90fb17c6ff. --- src/DIRAC/Core/Utilities/Extensions.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/DIRAC/Core/Utilities/Extensions.py b/src/DIRAC/Core/Utilities/Extensions.py index 695030f2850..9ed27fbeec1 100644 --- a/src/DIRAC/Core/Utilities/Extensions.py +++ b/src/DIRAC/Core/Utilities/Extensions.py @@ -158,13 +158,10 @@ def _extensionsByPriorityPy3(): from importlib import metadata # pylint: disable=no-name-in-module priorties = defaultdict(list) - try: - for entrypoint in set(metadata.entry_points()['dirac']): - extensionName = entrypointToExtension(entrypoint) - extension_metadata = entrypoint.load()() - priorties[extension_metadata["priority"]].append(extensionName) - except KeyError: - return ['DIRAC'] + for entrypoint in set(metadata.entry_points()['dirac']): + extensionName = entrypointToExtension(entrypoint) + extension_metadata = entrypoint.load()() + priorties[extension_metadata["priority"]].append(extensionName) extensions = [] for priority, extensionNames in sorted(priorties.items()):