From 9de62784bd2ec3eef5891f00a52e5b55383a9b33 Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Thu, 9 Dec 2021 18:15:44 +0100 Subject: [PATCH 1/4] feat: SingularityComputingElement: install python3 version --- .../Resources/Computing/index.rst | 2 - src/DIRAC/Resources/Cloud/cloudinit.template | 1 - .../Computing/SingularityComputingElement.py | 109 +++++------------- 3 files changed, 32 insertions(+), 80 deletions(-) diff --git a/docs/source/AdministratorGuide/Configuration/ConfReference/Resources/Computing/index.rst b/docs/source/AdministratorGuide/Configuration/ConfReference/Resources/Computing/index.rst index 88ec609205f..688d2b21f5f 100644 --- a/docs/source/AdministratorGuide/Configuration/ConfReference/Resources/Computing/index.rst +++ b/docs/source/AdministratorGuide/Configuration/ConfReference/Resources/Computing/index.rst @@ -126,8 +126,6 @@ Singularity CE Parameters +-------------------------+-------------------------------------------------------------------+------------------------------------------------------------------------------+ | InstallDIRACInContainer | Flag for re-installing, or not, DIRAC in the container | False (default: True) | +-------------------------+-------------------------------------------------------------------+------------------------------------------------------------------------------+ -| ContainerExtraOpts | Extra options for dirac-install (py2) within the container. | -u 'http://other.host/instdir' | -+-------------------------+-------------------------------------------------------------------+------------------------------------------------------------------------------+ | KeepWorkArea | If set to True container work area won't be deleted at end of job | True (Default: False) | +-------------------------+-------------------------------------------------------------------+------------------------------------------------------------------------------+ diff --git a/src/DIRAC/Resources/Cloud/cloudinit.template b/src/DIRAC/Resources/Cloud/cloudinit.template index ca49327b20d..6faad96aeeb 100644 --- a/src/DIRAC/Resources/Cloud/cloudinit.template +++ b/src/DIRAC/Resources/Cloud/cloudinit.template @@ -111,7 +111,6 @@ write_files: -o /LocalSite/LocalCE=%(ce-type)s \ -o /Resources/Computing/CEDefaults/VirtualOrganization=%(vo)s \ -o /Resources/Computing/CEDefaults/WholeNode=%(whole-node)s \ - -o /Resources/Computing/CEDefaults/ContainerExtraOpts="%(extraopts)s" \ -o /Resources/Computing/CEDefaults/ContainerRoot=%(user-root)s - path: /root/run_monitor.sh permissions: '0755' diff --git a/src/DIRAC/Resources/Computing/SingularityComputingElement.py b/src/DIRAC/Resources/Computing/SingularityComputingElement.py index 3d9a9a60b77..c1dd87e4073 100644 --- a/src/DIRAC/Resources/Computing/SingularityComputingElement.py +++ b/src/DIRAC/Resources/Computing/SingularityComputingElement.py @@ -6,9 +6,7 @@ The goal of this CE is to start the job in the container set by the "ContainerRoot" config option. - DIRAC can be re-installed within the container, extra flags can - be given to the dirac-install command with the "ContainerExtraOpts" - option. + DIRAC can be re-installed within the container. See the Configuration/Resources/Computing documention for details on where to set the option parameters. @@ -25,12 +23,9 @@ import sys import tempfile -from six.moves.urllib.request import urlopen - import DIRAC from DIRAC import S_OK, S_ERROR, gConfig, gLogger from DIRAC.Core.Utilities.Subprocess import systemCall -from DIRAC.ConfigurationSystem.Client.Helpers import CSGlobals from DIRAC.ConfigurationSystem.Client.Helpers import Operations from DIRAC.Core.Utilities.ThreadScheduler import gThreadScheduler from DIRAC.Resources.Computing.ComputingElement import ComputingElement @@ -48,18 +43,23 @@ CONTAINER_WRAPPER_INSTALL = """#!/bin/bash -echo "Starting inner container wrapper scripts (uses dirac-install) at `date`." -set -x +echo "Starting inner container wrapper scripts at `date`." +set -ex cd /tmp -# Avoid using the host's DIRAC(OS) installation -unset DIRAC -unset DIRACOS # Install DIRAC -./dirac-install.py %(install_args)s -source bashrc +installer_name="DIRACOS-Linux-$(uname -m).sh" +if [[ -d /cvmfs/dirac.egi.eu/installSource/ ]]; then + bash /cvmfs/dirac.egi.eu/installSource/"${installer_name}" +else + curl -LO "https://github.com/DIRACGrid/DIRACOS2/releases/latest/download/${installer_name}" + bash "${installer_name}" + rm "${installer_name}" +fi +source diracos/diracosrc +pip install %(dirac_project)s==%(version)s dirac-configure -F %(config_args)s -I # Add compatibility with pilot3 where config is in pilot.cfg -ln -s etc/dirac.cfg pilot.cfg +ln -s diracos/etc/dirac.cfg pilot.cfg # Run next wrapper (to start actual job) bash %(next_wrapper)s # Write the payload errorcode to a file for the outer scripts @@ -105,7 +105,7 @@ def __init__(self, ceUniqueID): self.__workdir = CONTAINER_WORKDIR self.__innerdir = CONTAINER_INNERDIR self.__singularityBin = "singularity" - self.__installDIRACInContainer = self.ceParameters.get("InstallDIRACInContainer", six.PY2) + self.__installDIRACInContainer = self.ceParameters.get("InstallDIRACInContainer", True) if isinstance(self.__installDIRACInContainer, six.string_types) and self.__installDIRACInContainer.lower() in ( "false", "no", @@ -155,24 +155,12 @@ def __hasSingularity(self): # No suitable binaries found return False - @staticmethod - def __findInstallBaseDir(): - """Find the path to root of the current DIRAC installation""" - if six.PY3: - return os.path.realpath(sys.base_prefix) # pylint: disable=no-member - else: - candidate = os.path.join(DIRAC.rootPath, "bashrc") - return os.path.dirname(os.path.realpath(candidate)) - def __getInstallFlags(self, infoDict=None): - """Get the flags to pass to dirac-install.py inside the container. - Returns a string containing the command line flags. - """ + """Get the flags for installing inside the container.""" + if not infoDict: infoDict = {} - instOpts = [] - setup = infoDict.get("DefaultSetup") if not setup: setup = list(infoDict.get("Setups"))[0] @@ -180,35 +168,20 @@ def __getInstallFlags(self, infoDict=None): setup = gConfig.getValue("/DIRAC/Setup", "unknown") setup = str(setup) - installationName = str(infoDict.get("Installation")) - if not installationName or installationName == "None": - installationName = Operations.Operations(setup=setup).getValue("Pilot/Installation", "") - if installationName: - instOpts.append("-V %s" % installationName) + diracProject = "DIRAC" + + project = str(infoDict.get("Project")) + if not project or project == "None": + diracProject = Operations.Operations(setup=setup).getValue("Pilot/Project", "") + project diracVersions = str(infoDict["Setups"][setup].get("Version")).split(",") if not diracVersions: diracVersions = str(infoDict["Setups"]["Defaults"].get("Version")).split(",") if not diracVersions: diracVersions = Operations.Operations(setup=setup).getValue("Pilot/Version", []) - instOpts.append("-r '%s'" % diracVersions[0].strip()) - - pilotExtensionsList = str(infoDict["Setups"][setup].get("CommandExtensions")).split(",") - if not pilotExtensionsList: - pilotExtensionsList = str(infoDict["Setups"]["Defaults"].get("CommandExtensions")).split(",") - if not pilotExtensionsList: - pilotExtensionsList = Operations.Operations(setup=setup).getValue("Pilot/Extensions", []) - extensionsList = [] - if pilotExtensionsList: - if pilotExtensionsList[0] != "None": - extensionsList = pilotExtensionsList - else: - extensionsList = CSGlobals.getCSExtensions() - if extensionsList: - instOpts.append("-e '%s'" % ",".join([ext for ext in extensionsList if "Web" not in ext])) - if "ContainerExtraOpts" in self.ceParameters: - instOpts.append(self.ceParameters["ContainerExtraOpts"]) - return " ".join(instOpts) + version = diracVersions[0].strip() + + return diracProject, version @staticmethod def __getConfigFlags(infoDict=None): @@ -292,23 +265,10 @@ def __createWorkArea(self, jobDesc=None, log=None, logLevel="INFO", proxy=None): wrapperPath = result["Value"] if self.__installDIRACInContainer: - if six.PY3: - result = S_ERROR("InstallDIRACInContainer is not supported with Python 3") + if six.PY2: + result = S_ERROR("InstallDIRACInContainer is not supported with Python 2") result["ReschedulePayload"] = True return result - # dirac-install.py - - # Download dirac-install.py - response = urlopen("https://raw.githubusercontent.com/DIRACGrid/management/master/dirac-install.py") - code = response.getcode() - if code > 200 or code >= 300: - return S_ERROR("Failed to download dirac-install.py with code %s" % code) - with open("dirac-install.py", "wb") as fp: - fp.write(response.read()) - - install_loc = os.path.join(tmpDir, "dirac-install.py") - shutil.copyfile("dirac-install.py", install_loc) - os.chmod(install_loc, 0o755) infoDict = None if os.path.isfile("pilot.json"): # if this is a pilot 3 this file should be found @@ -316,9 +276,11 @@ def __createWorkArea(self, jobDesc=None, log=None, logLevel="INFO", proxy=None): infoDict = json.load(pj) # Extra Wrapper (Container DIRAC installer) + installFlags = self.__getInstallFlags(infoDict) wrapSubs = { "next_wrapper": wrapperPath, - "install_args": self.__getInstallFlags(infoDict), + "dirac_project": installFlags[0], + "version": installFlags[1], "config_args": self.__getConfigFlags(infoDict), } CONTAINER_WRAPPER = CONTAINER_WRAPPER_INSTALL @@ -329,14 +291,7 @@ def __createWorkArea(self, jobDesc=None, log=None, logLevel="INFO", proxy=None): "dirac_env_var": os.environ.get("DIRAC", ""), "diracos_env_var": os.environ.get("DIRACOS", ""), } - if six.PY2: - shutil.copyfile( - os.path.join(self.__findInstallBaseDir(), "bashrc"), - os.path.join(tmpDir, "bashrc"), - ) - wrapSubs["rc_script"] = "bashrc" - else: - wrapSubs["rc_script"] = os.path.join(self.__findInstallBaseDir(), "diracosrc") + wrapSubs["rc_script"] = os.path.join(os.path.realpath(sys.base_prefix), "diracosrc") shutil.copyfile("pilot.cfg", os.path.join(tmpDir, "pilot.cfg")) CONTAINER_WRAPPER = CONTAINER_WRAPPER_NO_INSTALL @@ -450,7 +405,7 @@ def submitJob(self, executableFile, proxy=None, **kwargs): if withCVMFS: cmd.extend(["--bind", "/cvmfs"]) if not self.__installDIRACInContainer: - cmd.extend(["--bind", "{0}:{0}:ro".format(self.__findInstallBaseDir())]) + cmd.extend(["--bind", "{0}:{0}:ro".format(os.path.realpath(sys.base_prefix))]) if "ContainerBind" in self.ceParameters: bindPaths = self.ceParameters["ContainerBind"].split(",") for bindPath in bindPaths: From 51fd0022d1c84a0d0fad5f6d9a188f8d56663d78 Mon Sep 17 00:00:00 2001 From: fstagni Date: Tue, 14 Dec 2021 12:53:35 +0100 Subject: [PATCH 2/4] test: moved Test_SingualityCE from server to client --- tests/Integration/all_integration_client_tests.sh | 4 ++++ tests/Integration/all_integration_server_tests.sh | 1 - 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/Integration/all_integration_client_tests.sh b/tests/Integration/all_integration_client_tests.sh index 94c62990b6a..cbd3dae6a43 100644 --- a/tests/Integration/all_integration_client_tests.sh +++ b/tests/Integration/all_integration_client_tests.sh @@ -79,6 +79,10 @@ echo -e "*** $(date -u) **** PS TESTS ****\n" pytest "${THIS_DIR}/ProductionSystem/Test_Client_Production.py" |& tee -a clientTestOutputs.txt; (( ERR |= "${?}" )) pytest "${THIS_DIR}/ProductionSystem/Test_Client_TS_Prod.py" |& tee -a clientTestOutputs.txt; (( ERR |= "${?}" )) +#-------------------------------------------------------------------------------# +echo -e "*** $(date -u) **** Resources TESTS ****\n" +pytest "${THIS_DIR}/Resources/Computing/Test_SingularityCE.py" |& tee -a "${SERVER_TEST_OUTPUT}"; (( ERR |= "${?}" )) + #-------------------------------------------------------------------------------# echo -e "*** $(date -u) **** DataManager TESTS ****\n" diff --git a/tests/Integration/all_integration_server_tests.sh b/tests/Integration/all_integration_server_tests.sh index d98f978a193..01cfd527ae8 100644 --- a/tests/Integration/all_integration_server_tests.sh +++ b/tests/Integration/all_integration_server_tests.sh @@ -100,7 +100,6 @@ pytest "${THIS_DIR}/RequestManagementSystem/Test_ReqDB.py" |& tee -a "${SERVER_T echo -e "*** $(date -u) **** Resources TESTS ****\n" python "${THIS_DIR}/Resources/Storage/Test_Resources_GFAL2StorageBase.py" ProductionSandboxSE |& tee -a "${SERVER_TEST_OUTPUT}"; (( ERR |= "${?}" )) -pytest "${THIS_DIR}/Resources/Computing/Test_SingularityCE.py" |& tee -a "${SERVER_TEST_OUTPUT}"; (( ERR |= "${?}" )) pytest "${THIS_DIR}/Resources/ProxyProvider/Test_DIRACCAProxyProvider.py" |& tee -a "${SERVER_TEST_OUTPUT}"; (( ERR |= "${?}" )) # Can only run if there's a Stomp MQ local... From 96de9277f8520f62d9dfbf14ba46dc5e38d4f12d Mon Sep 17 00:00:00 2001 From: fstagni Date: Tue, 14 Dec 2021 12:56:55 +0100 Subject: [PATCH 3/4] test: fix when running with py2 --- .../Computing/SingularityComputingElement.py | 46 +++++++++++++------ .../Resources/Computing/Test_SingularityCE.py | 11 ++++- .../all_integration_client_tests.sh | 2 +- 3 files changed, 41 insertions(+), 18 deletions(-) diff --git a/src/DIRAC/Resources/Computing/SingularityComputingElement.py b/src/DIRAC/Resources/Computing/SingularityComputingElement.py index c1dd87e4073..d7d53a66a52 100644 --- a/src/DIRAC/Resources/Computing/SingularityComputingElement.py +++ b/src/DIRAC/Resources/Computing/SingularityComputingElement.py @@ -105,7 +105,7 @@ def __init__(self, ceUniqueID): self.__workdir = CONTAINER_WORKDIR self.__innerdir = CONTAINER_INNERDIR self.__singularityBin = "singularity" - self.__installDIRACInContainer = self.ceParameters.get("InstallDIRACInContainer", True) + self.__installDIRACInContainer = self.ceParameters.get("InstallDIRACInContainer", True) if isinstance(self.__installDIRACInContainer, six.string_types) and self.__installDIRACInContainer.lower() in ( "false", "no", @@ -155,8 +155,17 @@ def __hasSingularity(self): # No suitable binaries found return False + @staticmethod + def __findInstallBaseDir(): + """Find the path to root of the current DIRAC installation""" + if six.PY3: + return os.path.realpath(sys.base_prefix) # pylint: disable=no-member + else: + candidate = os.path.join(DIRAC.rootPath, "bashrc") + return os.path.dirname(os.path.realpath(candidate)) + def __getInstallFlags(self, infoDict=None): - """Get the flags for installing inside the container.""" + """Get the flags for installing inside the container.""" if not infoDict: infoDict = {} @@ -168,20 +177,20 @@ def __getInstallFlags(self, infoDict=None): setup = gConfig.getValue("/DIRAC/Setup", "unknown") setup = str(setup) - diracProject = "DIRAC" + diracProject = "DIRAC" - project = str(infoDict.get("Project")) - if not project or project == "None": - diracProject = Operations.Operations(setup=setup).getValue("Pilot/Project", "") + project + project = str(infoDict.get("Project")) + if not project or project == "None": + diracProject = Operations.Operations(setup=setup).getValue("Pilot/Project", "") + project diracVersions = str(infoDict["Setups"][setup].get("Version")).split(",") if not diracVersions: diracVersions = str(infoDict["Setups"]["Defaults"].get("Version")).split(",") if not diracVersions: diracVersions = Operations.Operations(setup=setup).getValue("Pilot/Version", []) - version = diracVersions[0].strip() + version = diracVersions[0].strip() - return diracProject, version + return diracProject, version @staticmethod def __getConfigFlags(infoDict=None): @@ -265,8 +274,8 @@ def __createWorkArea(self, jobDesc=None, log=None, logLevel="INFO", proxy=None): wrapperPath = result["Value"] if self.__installDIRACInContainer: - if six.PY2: - result = S_ERROR("InstallDIRACInContainer is not supported with Python 2") + if six.PY2: + result = S_ERROR("InstallDIRACInContainer is not supported with Python 2") result["ReschedulePayload"] = True return result @@ -276,11 +285,11 @@ def __createWorkArea(self, jobDesc=None, log=None, logLevel="INFO", proxy=None): infoDict = json.load(pj) # Extra Wrapper (Container DIRAC installer) - installFlags = self.__getInstallFlags(infoDict) + installFlags = self.__getInstallFlags(infoDict) wrapSubs = { "next_wrapper": wrapperPath, - "dirac_project": installFlags[0], - "version": installFlags[1], + "dirac_project": installFlags[0], + "version": installFlags[1], "config_args": self.__getConfigFlags(infoDict), } CONTAINER_WRAPPER = CONTAINER_WRAPPER_INSTALL @@ -291,7 +300,14 @@ def __createWorkArea(self, jobDesc=None, log=None, logLevel="INFO", proxy=None): "dirac_env_var": os.environ.get("DIRAC", ""), "diracos_env_var": os.environ.get("DIRACOS", ""), } - wrapSubs["rc_script"] = os.path.join(os.path.realpath(sys.base_prefix), "diracosrc") + if six.PY2: + shutil.copyfile( + os.path.join(self.__findInstallBaseDir(), "bashrc"), + os.path.join(tmpDir, "bashrc"), + ) + wrapSubs["rc_script"] = "bashrc" + else: + wrapSubs["rc_script"] = os.path.join(self.__findInstallBaseDir(), "diracosrc") shutil.copyfile("pilot.cfg", os.path.join(tmpDir, "pilot.cfg")) CONTAINER_WRAPPER = CONTAINER_WRAPPER_NO_INSTALL @@ -405,7 +421,7 @@ def submitJob(self, executableFile, proxy=None, **kwargs): if withCVMFS: cmd.extend(["--bind", "/cvmfs"]) if not self.__installDIRACInContainer: - cmd.extend(["--bind", "{0}:{0}:ro".format(os.path.realpath(sys.base_prefix))]) + cmd.extend(["--bind", "{0}:{0}:ro".format(self.__findInstallBaseDir())]) if "ContainerBind" in self.ceParameters: bindPaths = self.ceParameters["ContainerBind"].split(",") for bindPath in bindPaths: diff --git a/tests/Integration/Resources/Computing/Test_SingularityCE.py b/tests/Integration/Resources/Computing/Test_SingularityCE.py index d026a9d25b9..60296c6d6ee 100644 --- a/tests/Integration/Resources/Computing/Test_SingularityCE.py +++ b/tests/Integration/Resources/Computing/Test_SingularityCE.py @@ -4,6 +4,7 @@ This test is here and not in the unit tests because it requires singularity to be installed. """ +import six import os import shutil @@ -35,7 +36,10 @@ def test_submitJob(): assert res["ReschedulePayload"] is True res = ce.getCEStatus() assert res["OK"] is True - assert res["SubmittedJobs"] == 1 + if six.PY2: + assert res["SubmittedJobs"] == 0 + else: + assert res["SubmittedJobs"] == 1 _stopJob(1) for ff in ["testJob.py", "pilot.json"]: if os.path.isfile(ff): @@ -73,7 +77,10 @@ def test_submitJobWrapper(): res = ce.getCEStatus() assert res["OK"] is True - assert res["SubmittedJobs"] == 1 + if six.PY2: + assert res["SubmittedJobs"] == 0 + else: + assert res["SubmittedJobs"] == 1 _stopJob(2) for ff in ["testJob.py", "stop_job_2", "job.info", "std.out", "pilot.json"]: diff --git a/tests/Integration/all_integration_client_tests.sh b/tests/Integration/all_integration_client_tests.sh index cbd3dae6a43..72dd97d0fc4 100644 --- a/tests/Integration/all_integration_client_tests.sh +++ b/tests/Integration/all_integration_client_tests.sh @@ -81,7 +81,7 @@ pytest "${THIS_DIR}/ProductionSystem/Test_Client_TS_Prod.py" |& tee -a clientTes #-------------------------------------------------------------------------------# echo -e "*** $(date -u) **** Resources TESTS ****\n" -pytest "${THIS_DIR}/Resources/Computing/Test_SingularityCE.py" |& tee -a "${SERVER_TEST_OUTPUT}"; (( ERR |= "${?}" )) +pytest "${THIS_DIR}/Resources/Computing/Test_SingularityCE.py" |& tee -a clientTestOutputs.txt; (( ERR |= "${?}" )) #-------------------------------------------------------------------------------# echo -e "*** $(date -u) **** DataManager TESTS ****\n" From d7105730fa428ff3d4e6addfeb227cf22c83d2e5 Mon Sep 17 00:00:00 2001 From: fstagni Date: Wed, 15 Dec 2021 18:03:17 +0100 Subject: [PATCH 4/4] fix: diracProject, not project --- src/DIRAC/Resources/Computing/SingularityComputingElement.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/DIRAC/Resources/Computing/SingularityComputingElement.py b/src/DIRAC/Resources/Computing/SingularityComputingElement.py index d7d53a66a52..57d1a001e5c 100644 --- a/src/DIRAC/Resources/Computing/SingularityComputingElement.py +++ b/src/DIRAC/Resources/Computing/SingularityComputingElement.py @@ -181,7 +181,7 @@ def __getInstallFlags(self, infoDict=None): project = str(infoDict.get("Project")) if not project or project == "None": - diracProject = Operations.Operations(setup=setup).getValue("Pilot/Project", "") + project + diracProject = Operations.Operations(setup=setup).getValue("Pilot/Project", "") + diracProject diracVersions = str(infoDict["Setups"][setup].get("Version")).split(",") if not diracVersions: