Merge pull request #5716 from fstagni/v7r3-fixes38

[v7r3] SingularityComputingElement: install python3 version

docs/source/AdministratorGuide/Configuration/ConfReference/Resources/Computing/index.rst

@@ -126,8 +126,6 @@ Singularity CE Parameters
 +-------------------------+-------------------------------------------------------------------+------------------------------------------------------------------------------+
 | InstallDIRACInContainer | Flag for re-installing, or not, DIRAC in the container            |  False (default: True)                                                       |
 +-------------------------+-------------------------------------------------------------------+------------------------------------------------------------------------------+
-| ContainerExtraOpts      | Extra options for dirac-install (py2) within the container.       |  -u 'http://other.host/instdir'                                              |
-+-------------------------+-------------------------------------------------------------------+------------------------------------------------------------------------------+
 | KeepWorkArea            | If set to True container work area won't be deleted at end of job |  True (Default: False)                                                       |
 +-------------------------+-------------------------------------------------------------------+------------------------------------------------------------------------------+
 

src/DIRAC/Resources/Cloud/cloudinit.template

@@ -111,7 +111,6 @@ write_files:
             -o /LocalSite/LocalCE=%(ce-type)s \
             -o /Resources/Computing/CEDefaults/VirtualOrganization=%(vo)s \
             -o /Resources/Computing/CEDefaults/WholeNode=%(whole-node)s \
-            -o /Resources/Computing/CEDefaults/ContainerExtraOpts="%(extraopts)s" \
             -o /Resources/Computing/CEDefaults/ContainerRoot=%(user-root)s
  - path: /root/run_monitor.sh
    permissions: '0755'

src/DIRAC/Resources/Computing/SingularityComputingElement.py

@@ -6,9 +6,7 @@
     The goal of this CE is to start the job in the container set by
     the "ContainerRoot" config option.
 
-    DIRAC can be re-installed within the container, extra flags can
-    be given to the dirac-install command with the "ContainerExtraOpts"
-    option.
+    DIRAC can be re-installed within the container.
 
     See the Configuration/Resources/Computing documention for details on
     where to set the option parameters.
@@ -25,12 +23,9 @@
 import sys
 import tempfile
 
-from six.moves.urllib.request import urlopen
-
 import DIRAC
 from DIRAC import S_OK, S_ERROR, gConfig, gLogger
 from DIRAC.Core.Utilities.Subprocess import systemCall
-from DIRAC.ConfigurationSystem.Client.Helpers import CSGlobals
 from DIRAC.ConfigurationSystem.Client.Helpers import Operations
 from DIRAC.Core.Utilities.ThreadScheduler import gThreadScheduler
 from DIRAC.Resources.Computing.ComputingElement import ComputingElement
@@ -48,18 +43,23 @@
 
 CONTAINER_WRAPPER_INSTALL = """#!/bin/bash
 
-echo "Starting inner container wrapper scripts (uses dirac-install) at `date`."
-set -x
+echo "Starting inner container wrapper scripts at `date`."
+set -ex
 cd /tmp
-# Avoid using the host's DIRAC(OS) installation
-unset DIRAC
-unset DIRACOS
 # Install DIRAC
-./dirac-install.py %(install_args)s
-source bashrc
+installer_name="DIRACOS-Linux-$(uname -m).sh"
+if [[ -d /cvmfs/dirac.egi.eu/installSource/ ]]; then
+  bash /cvmfs/dirac.egi.eu/installSource/"${installer_name}"
+else
+  curl -LO "https://github.com/DIRACGrid/DIRACOS2/releases/latest/download/${installer_name}"
+  bash "${installer_name}"
+  rm "${installer_name}"
+fi
+source diracos/diracosrc
+pip install %(dirac_project)s==%(version)s
 dirac-configure -F %(config_args)s -I
 # Add compatibility with pilot3 where config is in pilot.cfg
-ln -s etc/dirac.cfg pilot.cfg
+ln -s diracos/etc/dirac.cfg pilot.cfg
 # Run next wrapper (to start actual job)
 bash %(next_wrapper)s
 # Write the payload errorcode to a file for the outer scripts
@@ -105,7 +105,7 @@ def __init__(self, ceUniqueID):
         self.__workdir = CONTAINER_WORKDIR
         self.__innerdir = CONTAINER_INNERDIR
         self.__singularityBin = "singularity"
-        self.__installDIRACInContainer = self.ceParameters.get("InstallDIRACInContainer", six.PY2)
+        self.__installDIRACInContainer = self.ceParameters.get("InstallDIRACInContainer", True)
         if isinstance(self.__installDIRACInContainer, six.string_types) and self.__installDIRACInContainer.lower() in (
             "false",
             "no",
@@ -165,50 +165,32 @@ def __findInstallBaseDir():
             return os.path.dirname(os.path.realpath(candidate))
 
     def __getInstallFlags(self, infoDict=None):
-        """Get the flags to pass to dirac-install.py inside the container.
-        Returns a string containing the command line flags.
-        """
+        """Get the flags for installing inside the container."""
+
         if not infoDict:
             infoDict = {}
 
-        instOpts = []
-
         setup = infoDict.get("DefaultSetup")
         if not setup:
             setup = list(infoDict.get("Setups"))[0]
         if not setup:
             setup = gConfig.getValue("/DIRAC/Setup", "unknown")
         setup = str(setup)
 
-        installationName = str(infoDict.get("Installation"))
-        if not installationName or installationName == "None":
-            installationName = Operations.Operations(setup=setup).getValue("Pilot/Installation", "")
-        if installationName:
-            instOpts.append("-V %s" % installationName)
+        diracProject = "DIRAC"
+
+        project = str(infoDict.get("Project"))
+        if not project or project == "None":
+            diracProject = Operations.Operations(setup=setup).getValue("Pilot/Project", "") + diracProject
 
         diracVersions = str(infoDict["Setups"][setup].get("Version")).split(",")
         if not diracVersions:
             diracVersions = str(infoDict["Setups"]["Defaults"].get("Version")).split(",")
         if not diracVersions:
             diracVersions = Operations.Operations(setup=setup).getValue("Pilot/Version", [])
-        instOpts.append("-r '%s'" % diracVersions[0].strip())
-
-        pilotExtensionsList = str(infoDict["Setups"][setup].get("CommandExtensions")).split(",")
-        if not pilotExtensionsList:
-            pilotExtensionsList = str(infoDict["Setups"]["Defaults"].get("CommandExtensions")).split(",")
-        if not pilotExtensionsList:
-            pilotExtensionsList = Operations.Operations(setup=setup).getValue("Pilot/Extensions", [])
-        extensionsList = []
-        if pilotExtensionsList:
-            if pilotExtensionsList[0] != "None":
-                extensionsList = pilotExtensionsList
-        else:
-            extensionsList = CSGlobals.getCSExtensions()
-        if extensionsList:
-            instOpts.append("-e '%s'" % ",".join([ext for ext in extensionsList if "Web" not in ext]))
-        if "ContainerExtraOpts" in self.ceParameters:
-            instOpts.append(self.ceParameters["ContainerExtraOpts"])
-        return " ".join(instOpts)
+        version = diracVersions[0].strip()
+
+        return diracProject, version
 
     @staticmethod
     def __getConfigFlags(infoDict=None):
@@ -292,33 +274,22 @@ def __createWorkArea(self, jobDesc=None, log=None, logLevel="INFO", proxy=None):
         wrapperPath = result["Value"]
 
         if self.__installDIRACInContainer:
-            if six.PY3:
-                result = S_ERROR("InstallDIRACInContainer is not supported with Python 3")
+            if six.PY2:
+                result = S_ERROR("InstallDIRACInContainer is not supported with Python 2")
                 result["ReschedulePayload"] = True
                 return result
-            # dirac-install.py
-
-            # Download dirac-install.py
-            response = urlopen("https://raw.githubusercontent.com/DIRACGrid/management/master/dirac-install.py")
-            code = response.getcode()
-            if code > 200 or code >= 300:
-                return S_ERROR("Failed to download dirac-install.py with code %s" % code)
-            with open("dirac-install.py", "wb") as fp:
-                fp.write(response.read())
-
-            install_loc = os.path.join(tmpDir, "dirac-install.py")
-            shutil.copyfile("dirac-install.py", install_loc)
-            os.chmod(install_loc, 0o755)
 
             infoDict = None
             if os.path.isfile("pilot.json"):  # if this is a pilot 3 this file should be found
                 with io.open("pilot.json") as pj:
                     infoDict = json.load(pj)
 
             # Extra Wrapper (Container DIRAC installer)
+            installFlags = self.__getInstallFlags(infoDict)
             wrapSubs = {
                 "next_wrapper": wrapperPath,
-                "install_args": self.__getInstallFlags(infoDict),
+                "dirac_project": installFlags[0],
+                "version": installFlags[1],
                 "config_args": self.__getConfigFlags(infoDict),
             }
             CONTAINER_WRAPPER = CONTAINER_WRAPPER_INSTALL

tests/Integration/Resources/Computing/Test_SingularityCE.py

@@ -4,6 +4,7 @@
     This test is here and not in the unit tests because it requires singularity to be installed.
 """
 
+import six
 import os
 import shutil
 
@@ -35,7 +36,10 @@ def test_submitJob():
     assert res["ReschedulePayload"] is True
     res = ce.getCEStatus()
     assert res["OK"] is True
-    assert res["SubmittedJobs"] == 1
+    if six.PY2:
+        assert res["SubmittedJobs"] == 0
+    else:
+        assert res["SubmittedJobs"] == 1
     _stopJob(1)
     for ff in ["testJob.py", "pilot.json"]:
         if os.path.isfile(ff):
@@ -73,7 +77,10 @@ def test_submitJobWrapper():
 
     res = ce.getCEStatus()
     assert res["OK"] is True
-    assert res["SubmittedJobs"] == 1
+    if six.PY2:
+        assert res["SubmittedJobs"] == 0
+    else:
+        assert res["SubmittedJobs"] == 1
 
     _stopJob(2)
     for ff in ["testJob.py", "stop_job_2", "job.info", "std.out", "pilot.json"]:

tests/Integration/all_integration_client_tests.sh

@@ -79,6 +79,10 @@ echo -e "*** $(date -u)  **** PS TESTS ****\n"
 pytest "${THIS_DIR}/ProductionSystem/Test_Client_Production.py" |& tee -a clientTestOutputs.txt; (( ERR |= "${?}" ))
 pytest "${THIS_DIR}/ProductionSystem/Test_Client_TS_Prod.py" |& tee -a clientTestOutputs.txt; (( ERR |= "${?}" ))
 
+#-------------------------------------------------------------------------------#
+echo -e "*** $(date -u)  **** Resources TESTS ****\n"
+pytest "${THIS_DIR}/Resources/Computing/Test_SingularityCE.py" |& tee -a clientTestOutputs.txt; (( ERR |= "${?}" ))
+
 #-------------------------------------------------------------------------------#
 echo -e "*** $(date -u) **** DataManager TESTS ****\n"
 

tests/Integration/all_integration_server_tests.sh

@@ -100,7 +100,6 @@ pytest "${THIS_DIR}/RequestManagementSystem/Test_ReqDB.py" |& tee -a "${SERVER_T
 echo -e "*** $(date -u)  **** Resources TESTS ****\n"
 
 python "${THIS_DIR}/Resources/Storage/Test_Resources_GFAL2StorageBase.py" ProductionSandboxSE |& tee -a "${SERVER_TEST_OUTPUT}"; (( ERR |= "${?}" ))
-pytest "${THIS_DIR}/Resources/Computing/Test_SingularityCE.py" |& tee -a "${SERVER_TEST_OUTPUT}"; (( ERR |= "${?}" ))
 pytest "${THIS_DIR}/Resources/ProxyProvider/Test_DIRACCAProxyProvider.py" |& tee -a "${SERVER_TEST_OUTPUT}"; (( ERR |= "${?}" ))
 
 # Can only run if there's a Stomp MQ local...