languages: add python support

Author: jkloetzke

Diff for: pym/bob/input.py

@@ -5,7 +5,7 @@
 
 from . import BOB_VERSION, BOB_INPUT_HASH, DEBUG
 from .errors import ParseError, BobError
-from .languages import getLanguage, ScriptLanguage, BashLanguage, PwshLanguage
+from .languages import getLanguage, ScriptLanguage, BashLanguage, PwshLanguage, PythonLanguage
 from .pathspec import PackageSet
 from .scm import CvsScm, GitScm, ImportScm, SvnScm, UrlScm, ScmOverride, \
     auditFromDir, getScm, SYNTHETIC_SCM_PROPS
@@ -115,9 +115,11 @@ def fetchFingerprintScripts(recipe):
             recipe.get("fingerprintScript")),
         ScriptLanguage.PWSH : recipe.get("fingerprintScriptPwsh",
             recipe.get("fingerprintScript")),
+        ScriptLanguage.PYTHON : recipe.get("fingerprintScriptPython",
+            recipe.get("fingerprintScript")),
     }
 
-def fetchScripts(recipe, prefix, resolveBash, resolvePwsh):
+def fetchScripts(recipe, prefix, resolveBash, resolvePwsh, resolvePython):
     return {
         ScriptLanguage.BASH : (
             resolveBash(recipe.get(prefix + "SetupBash", recipe.get(prefix + "Setup")),
@@ -130,7 +132,13 @@ def fetchScripts(recipe, prefix, resolveBash, resolvePwsh):
                         prefix + "Setup[Pwsh]"),
             resolvePwsh(recipe.get(prefix + "ScriptPwsh", recipe.get(prefix + "Script")),
                         prefix + "Script[Pwsh]"),
-        )
+        ),
+        ScriptLanguage.PYTHON : (
+            resolvePython(recipe.get(prefix + "SetupPython", recipe.get(prefix + "Setup")),
+                          prefix + "Setup[Python]"),
+            resolvePython(recipe.get(prefix + "ScriptPython", recipe.get(prefix + "Script")),
+                          prefix + "Script[Python]"),
+        ),
     }
 
 def mergeScripts(fragments, glue):
@@ -2187,9 +2195,11 @@ def __init__(self, recipeSet, recipe, layer, sourceFile, baseDir, packageName, b
                                       baseDir, packageName, sourceName).resolve
         incHelperPwsh = IncludeHelper(PwshLanguage, recipeSet.loadBinary,
                                       baseDir, packageName, sourceName).resolve
+        incHelperPython = IncludeHelper(PythonLanguage, recipeSet.loadBinary,
+                                        baseDir, packageName, sourceName).resolve
 
         self.__scriptLanguage = recipe.get("scriptLanguage")
-        self.__checkout = fetchScripts(recipe, "checkout", incHelperBash, incHelperPwsh)
+        self.__checkout = fetchScripts(recipe, "checkout", incHelperBash, incHelperPwsh, incHelperPython)
         self.__checkoutSCMs = recipe.get("checkoutSCM", [])
         for scm in self.__checkoutSCMs:
             scm["__source"] = sourceName
@@ -2199,8 +2209,8 @@ def __init__(self, recipeSet, recipe, layer, sourceFile, baseDir, packageName, b
         for a in self.__checkoutAsserts:
             a["__source"] = sourceName + ", checkoutAssert #{}".format(i)
             i += 1
-        self.__build = fetchScripts(recipe, "build", incHelperBash, incHelperPwsh)
-        self.__package = fetchScripts(recipe, "package", incHelperBash, incHelperPwsh)
+        self.__build = fetchScripts(recipe, "build", incHelperBash, incHelperPwsh, incHelperPython)
+        self.__package = fetchScripts(recipe, "package", incHelperBash, incHelperPwsh, incHelperPython)
         self.__fingerprintScriptList = fetchFingerprintScripts(recipe)
         self.__fingerprintIf = recipe.get("fingerprintIf")
         self.__fingerprintVarsList = set(recipe.get("fingerprintVars", []))
@@ -3055,7 +3065,7 @@ class RecipeSet:
         ),
         schema.Optional('layers') : [str],
         schema.Optional('scriptLanguage',
-                        default=ScriptLanguage.BASH) : schema.And(schema.Or("bash", "PowerShell"),
+                        default=ScriptLanguage.BASH) : schema.And(schema.Or("bash", "PowerShell", "python"),
                                                                   schema.Use(ScriptLanguage)),
     })
 
@@ -3727,7 +3737,7 @@ def __createSchemas(self):
             schema.Optional('fingerprintScriptPwsh', default="") : str,
             schema.Optional('fingerprintIf') : schema.Or(None, str, bool, IfExpression),
             schema.Optional('fingerprintVars') : [ varNameUseSchema ],
-            schema.Optional('scriptLanguage') : schema.And(schema.Or("bash", "PowerShell"),
+            schema.Optional('scriptLanguage') : schema.And(schema.Or("bash", "PowerShell", "python"),
                                                            schema.Use(ScriptLanguage)),
             schema.Optional('jobServer') : bool,
         }

Diff for: pym/bob/languages.py

@@ -139,6 +139,7 @@
 class ScriptLanguage(Enum):
     BASH = 'bash'
     PWSH = 'PowerShell'
+    PYTHON = 'python'
 
 
 class IncludeResolver:
@@ -185,7 +186,7 @@ def _includeLiteral(self, content):
         raise NotImplementedError()
 
     def _resolveContent(self, result):
-        raise NotImplementedError()
+        return result
 
     def resolve(self, result):
         return (self._resolveContent(result), "\n".join(self.__incDigests), self.__incFiles)
@@ -397,20 +398,12 @@ def setupFingerprint(spec, env):
 
 
 class PwshResolver(IncludeResolver):
-    def __init__(self, fileLoader, baseDir, origText, sourceName, varBase):
-        super().__init__(fileLoader, baseDir, origText, sourceName, varBase)
-        self.prolog = []
-        self.count = 0
-
     def _includeFile(self, name):
         return '"$_BOB_TMP_BASE/' + escapePwsh(name) + '"'
 
     def _includeLiteral(self, content):
         return quotePwsh(content.decode('utf8'))
 
-    def _resolveContent(self, result):
-        return "\n".join(self.prolog + [result])
-
 
 class PwshLanguage:
     index = ScriptLanguage.PWSH
@@ -574,9 +567,150 @@ def setupFingerprint(spec, env):
         return [interpreter, "-c", spec.fingerprintScript]
 
 
+class PythonResolver(IncludeResolver):
+    def _includeFile(self, name):
+        return 'os.path.join(_BOB_TMP_BASE, ' + repr(name) + ')'
+
+    def _includeLiteral(self, content):
+        return repr(content.decode('utf8'))
+
+
+class PythonLanguage:
+    index = ScriptLanguage.PYTHON
+    glue = "\nos.chdir(os.environ['BOB_CWD'])\n"
+    Resolver = PythonResolver
+
+    HELPERS = dedent("""\
+        from subprocess import run, call, check_call, check_output
+        import os, os.path, sys
+        """)
+
+    @staticmethod
+    def __formatProlog(spec, tmpDir):
+        pathSep = ";" if sys.platform == "win32" else ":"
+        env = { key : repr(value) for (key, value) in spec.env.items() }
+        env.update({
+            "PATH": '"' + pathSep + '".join([' + ", ".join(
+                [repr(os.path.abspath(p)) for p in spec.paths] +
+                (['os.environ["PATH"]'] if not spec.hasSandbox else
+                 [repr(p) for p in spec.sandboxPaths])
+            ) + '])',
+            "LD_LIBRARY_PATH": '"' + pathSep + '".join(' +
+                repr([ os.path.abspath(p) for p in spec.libraryPaths ]) + ')',
+            "BOB_CWD": repr(os.path.abspath(spec.workspaceExecPath)),
+        })
+
+        ret = [
+            "# Convenience helpers",
+            PythonLanguage.HELPERS,
+            "",
+            "# Special Bob array variables:",
+            "BOB_ALL_PATHS = dict({})".format(repr(sorted(
+                [ (name, os.path.abspath(path)) for name, path in spec.allPaths ]
+            ))),
+            "BOB_DEP_PATHS = dict({})".format(repr(sorted(
+                [ (name, os.path.abspath(path)) for name,path in spec.depPaths ]
+            ))),
+            "BOB_TOOL_PATHS = dict({})".format(repr(sorted(
+                [ (name, os.path.abspath(path)) for name,path in spec.toolPaths ]
+            ))),
+            '_BOB_TMP_BASE = ' + repr("/tmp" if spec.hasSandbox else os.path.join(tmpDir, "tmp")),
+            "",
+            "# Environment:",
+            "\n".join('os.environ["{}"] = {}'.format(k, v) for (k,v) in sorted(env.items())),
+        ]
+        return "\n".join(ret)
+
+    @staticmethod
+    def __formatSetup(spec):
+        return "\n".join([
+            "",
+            "# Recipe setup script",
+            spec.setupScript,
+            "os.chdir(os.environ['BOB_CWD'])",
+        ])
+
+    @staticmethod
+    def __formatScript(spec, tmpDir, trace):
+        if spec.envFile:
+            envFile = "/bob/env" if spec.hasSandbox else os.path.abspath(spec.envFile)
+        else:
+            envFile = None
+        ret = [
+            PythonLanguage.__formatProlog(spec, tmpDir),
+            "",
+            "# Setup",
+            dedent("""\
+                with open({ENV_FILE}, "w") as f:
+                    f.write(repr({{ "env" : dict(os.environ), "globals" : globals() }}))
+                """.format(ENV_FILE=repr(envFile))) if envFile else "",
+            "os.chdir(os.environ['BOB_CWD'])",
+            "",
+            PythonLanguage.__formatSetup(spec),
+            "",
+            "# Recipe main script",
+            spec.mainScript,
+        ]
+        return "\n".join(ret)
+
+    @staticmethod
+    def __scriptFilePaths(spec, tmpDir):
+        if spec.hasSandbox:
+            execScriptFile = "/.script.py"
+            realScriptFile = (spec.scriptHint or os.path.join(tmpDir, ".script")) + ".py"
+        else:
+            execScriptFile = (spec.scriptHint or os.path.join(tmpDir, "script")) + ".py"
+            realScriptFile = execScriptFile
+        return (os.path.abspath(realScriptFile), os.path.abspath(execScriptFile))
+
+    @staticmethod
+    def setupShell(spec, tmpDir, keepEnv):
+        realScriptFile, execScriptFile = PythonLanguage.__scriptFilePaths(spec, tmpDir)
+        with open(realScriptFile, "w") as f:
+            f.write(PythonLanguage.__formatProlog(spec, tmpDir))
+            f.write(PythonLanguage.__formatSetup(spec))
+
+        args = [sys.executable, "-i", execScriptFile]
+        args.extend(os.path.abspath(a) for a in spec.args)
+
+        return (realScriptFile, execScriptFile, args)
+
+    @staticmethod
+    def setupCall(spec, tmpDir, keepEnv, trace):
+        realScriptFile, execScriptFile = PythonLanguage.__scriptFilePaths(spec, tmpDir)
+        with open(realScriptFile, "w") as f:
+            f.write(PythonLanguage.__formatScript(spec, tmpDir, trace))
+
+        args = [sys.executable, execScriptFile]
+        args.extend(os.path.abspath(a) for a in spec.args)
+
+        return (realScriptFile, execScriptFile, args)
+
+    @staticmethod
+    def mangleFingerprints(scriptFragments, env):
+        # join the script fragments first
+        script = joinScripts(scriptFragments, PythonLanguage.glue)
+
+        # do not add preamble for empty scripts
+        if not script: return ""
+
+        # Add snippets as they match and a default settings preamble
+        ret = [script]
+        for n,v in sorted(env.items()):
+            ret.append('os.environ["{}"] = {}'.format(k, repr(v)))
+        ret.append(PythonLanguage.HELPERS)
+
+        return "\n".join(reversed(ret))
+
+    @staticmethod
+    def setupFingerprint(spec, env):
+        return [sys.executable, "-c", spec.fingerprintScript]
+
+
 LANG = {
     ScriptLanguage.BASH : BashLanguage,
     ScriptLanguage.PWSH : PwshLanguage,
+    ScriptLanguage.PYTHON : PythonLanguage,
 }
 
 def getLanguage(language):