Skip to content

Commit 1920662

Browse files
jkloetzkejkloetzke
committed
languages: add python support
1 parent 2b13716 commit 1920662

File tree

2 files changed

+168
-17
lines changed

2 files changed

+168
-17
lines changed

Diff for: ‎pym/bob/input.py

+26-8
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55

66
from . import BOB_VERSION, BOB_INPUT_HASH, DEBUG
77
from .errors import ParseError, BobError
8-
from .languages import getLanguage, ScriptLanguage, BashLanguage, PwshLanguage
8+
from .languages import getLanguage, ScriptLanguage, BashLanguage, PwshLanguage, PythonLanguage
99
from .pathspec import PackageSet
1010
from .scm import CvsScm, GitScm, ImportScm, SvnScm, UrlScm, ScmOverride, \
1111
auditFromDir, getScm, SYNTHETIC_SCM_PROPS
@@ -115,9 +115,11 @@ def fetchFingerprintScripts(recipe):
115115
recipe.get("fingerprintScript")),
116116
ScriptLanguage.PWSH : recipe.get("fingerprintScriptPwsh",
117117
recipe.get("fingerprintScript")),
118+
ScriptLanguage.PYTHON : recipe.get("fingerprintScriptPython",
119+
recipe.get("fingerprintScript")),
118120
}
119121

120-
def fetchScripts(recipe, prefix, resolveBash, resolvePwsh):
122+
def fetchScripts(recipe, prefix, resolveBash, resolvePwsh, resolvePython):
121123
return {
122124
ScriptLanguage.BASH : (
123125
resolveBash(recipe.get(prefix + "SetupBash", recipe.get(prefix + "Setup")),
@@ -130,7 +132,13 @@ def fetchScripts(recipe, prefix, resolveBash, resolvePwsh):
130132
prefix + "Setup[Pwsh]"),
131133
resolvePwsh(recipe.get(prefix + "ScriptPwsh", recipe.get(prefix + "Script")),
132134
prefix + "Script[Pwsh]"),
133-
)
135+
),
136+
ScriptLanguage.PYTHON : (
137+
resolvePython(recipe.get(prefix + "SetupPython", recipe.get(prefix + "Setup")),
138+
prefix + "Setup[Python]"),
139+
resolvePython(recipe.get(prefix + "ScriptPython", recipe.get(prefix + "Script")),
140+
prefix + "Script[Python]"),
141+
),
134142
}
135143

136144
def mergeScripts(fragments, glue):
@@ -2187,9 +2195,11 @@ def __init__(self, recipeSet, recipe, layer, sourceFile, baseDir, packageName, b
21872195
baseDir, packageName, sourceName).resolve
21882196
incHelperPwsh = IncludeHelper(PwshLanguage, recipeSet.loadBinary,
21892197
baseDir, packageName, sourceName).resolve
2198+
incHelperPython = IncludeHelper(PythonLanguage, recipeSet.loadBinary,
2199+
baseDir, packageName, sourceName).resolve
21902200

21912201
self.__scriptLanguage = recipe.get("scriptLanguage")
2192-
self.__checkout = fetchScripts(recipe, "checkout", incHelperBash, incHelperPwsh)
2202+
self.__checkout = fetchScripts(recipe, "checkout", incHelperBash, incHelperPwsh, incHelperPython)
21932203
self.__checkoutSCMs = recipe.get("checkoutSCM", [])
21942204
for scm in self.__checkoutSCMs:
21952205
scm["__source"] = sourceName
@@ -2199,8 +2209,8 @@ def __init__(self, recipeSet, recipe, layer, sourceFile, baseDir, packageName, b
21992209
for a in self.__checkoutAsserts:
22002210
a["__source"] = sourceName + ", checkoutAssert #{}".format(i)
22012211
i += 1
2202-
self.__build = fetchScripts(recipe, "build", incHelperBash, incHelperPwsh)
2203-
self.__package = fetchScripts(recipe, "package", incHelperBash, incHelperPwsh)
2212+
self.__build = fetchScripts(recipe, "build", incHelperBash, incHelperPwsh, incHelperPython)
2213+
self.__package = fetchScripts(recipe, "package", incHelperBash, incHelperPwsh, incHelperPython)
22042214
self.__fingerprintScriptList = fetchFingerprintScripts(recipe)
22052215
self.__fingerprintIf = recipe.get("fingerprintIf")
22062216
self.__fingerprintVarsList = set(recipe.get("fingerprintVars", []))
@@ -3055,7 +3065,7 @@ class RecipeSet:
30553065
),
30563066
schema.Optional('layers') : [str],
30573067
schema.Optional('scriptLanguage',
3058-
default=ScriptLanguage.BASH) : schema.And(schema.Or("bash", "PowerShell"),
3068+
default=ScriptLanguage.BASH) : schema.And(schema.Or("bash", "PowerShell", "python"),
30593069
schema.Use(ScriptLanguage)),
30603070
})
30613071

@@ -3653,21 +3663,27 @@ def __createSchemas(self):
36533663
schema.Optional('checkoutScript') : str,
36543664
schema.Optional('checkoutScriptBash') : str,
36553665
schema.Optional('checkoutScriptPwsh') : str,
3666+
schema.Optional('checkoutScriptPython') : str,
36563667
schema.Optional('checkoutSetup') : str,
36573668
schema.Optional('checkoutSetupBash') : str,
36583669
schema.Optional('checkoutSetupPwsh') : str,
3670+
schema.Optional('checkoutSetupPython') : str,
36593671
schema.Optional('buildScript') : str,
36603672
schema.Optional('buildScriptBash') : str,
36613673
schema.Optional('buildScriptPwsh') : str,
3674+
schema.Optional('buildScriptPython') : str,
36623675
schema.Optional('buildSetup') : str,
36633676
schema.Optional('buildSetupBash') : str,
36643677
schema.Optional('buildSetupPwsh') : str,
3678+
schema.Optional('buildSetupPython') : str,
36653679
schema.Optional('packageScript') : str,
36663680
schema.Optional('packageScriptBash') : str,
36673681
schema.Optional('packageScriptPwsh') : str,
3682+
schema.Optional('packageScriptPython') : str,
36683683
schema.Optional('packageSetup') : str,
36693684
schema.Optional('packageSetupBash') : str,
36703685
schema.Optional('packageSetupPwsh') : str,
3686+
schema.Optional('packageSetupPython') : str,
36713687
schema.Optional('checkoutTools') : [ toolNameSchema ],
36723688
schema.Optional('buildTools') : [ toolNameSchema ],
36733689
schema.Optional('packageTools') : [ toolNameSchema ],
@@ -3705,6 +3721,7 @@ def __createSchemas(self):
37053721
schema.Optional('fingerprintScript', default="") : str,
37063722
schema.Optional('fingerprintScriptBash') : str,
37073723
schema.Optional('fingerprintScriptPwsh', default="") : str,
3724+
schema.Optional('fingerprintScriptPython', default="") : str,
37083725
schema.Optional('fingerprintIf') : schema.Or(None, str, bool, IfExpression),
37093726
schema.Optional('fingerprintVars') : [ varNameUseSchema ],
37103727
})
@@ -3725,9 +3742,10 @@ def __createSchemas(self):
37253742
schema.Optional('fingerprintScript', default="") : str,
37263743
schema.Optional('fingerprintScriptBash') : str,
37273744
schema.Optional('fingerprintScriptPwsh', default="") : str,
3745+
schema.Optional('fingerprintScriptPython', default="") : str,
37283746
schema.Optional('fingerprintIf') : schema.Or(None, str, bool, IfExpression),
37293747
schema.Optional('fingerprintVars') : [ varNameUseSchema ],
3730-
schema.Optional('scriptLanguage') : schema.And(schema.Or("bash", "PowerShell"),
3748+
schema.Optional('scriptLanguage') : schema.And(schema.Or("bash", "PowerShell", "python"),
37313749
schema.Use(ScriptLanguage)),
37323750
schema.Optional('jobServer') : bool,
37333751
}

Diff for: ‎pym/bob/languages.py

+142-9
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,7 @@
139139
class ScriptLanguage(Enum):
140140
BASH = 'bash'
141141
PWSH = 'PowerShell'
142+
PYTHON = 'python'
142143

143144

144145
class IncludeResolver:
@@ -185,7 +186,7 @@ def _includeLiteral(self, content):
185186
raise NotImplementedError()
186187

187188
def _resolveContent(self, result):
188-
raise NotImplementedError()
189+
return result
189190

190191
def resolve(self, result):
191192
return (self._resolveContent(result), "\n".join(self.__incDigests), self.__incFiles)
@@ -397,20 +398,12 @@ def setupFingerprint(spec, env):
397398

398399

399400
class PwshResolver(IncludeResolver):
400-
def __init__(self, fileLoader, baseDir, origText, sourceName, varBase):
401-
super().__init__(fileLoader, baseDir, origText, sourceName, varBase)
402-
self.prolog = []
403-
self.count = 0
404-
405401
def _includeFile(self, name):
406402
return '"$_BOB_TMP_BASE/' + escapePwsh(name) + '"'
407403

408404
def _includeLiteral(self, content):
409405
return quotePwsh(content.decode('utf8'))
410406

411-
def _resolveContent(self, result):
412-
return "\n".join(self.prolog + [result])
413-
414407

415408
class PwshLanguage:
416409
index = ScriptLanguage.PWSH
@@ -574,9 +567,149 @@ def setupFingerprint(spec, env):
574567
return [interpreter, "-c", spec.fingerprintScript]
575568

576569

570+
class PythonResolver(IncludeResolver):
571+
def _includeFile(self, name):
572+
return 'os.path.join(_BOB_TMP_BASE, ' + repr(name) + ')'
573+
574+
def _includeLiteral(self, content):
575+
return repr(content.decode('utf8'))
576+
577+
578+
class PythonLanguage:
579+
index = ScriptLanguage.PYTHON
580+
glue = "\nos.chdir(os.environ['BOB_CWD'])\n"
581+
Resolver = PythonResolver
582+
583+
PROLOGUE = dedent("""\
584+
import os, os.path, sys
585+
""")
586+
587+
@staticmethod
588+
def __formatProlog(spec, tmpDir):
589+
pathSep = ";" if sys.platform == "win32" else ":"
590+
env = { key : repr(value) for (key, value) in spec.env.items() }
591+
env.update({
592+
"PATH": '"' + pathSep + '".join([' + ", ".join(
593+
[repr(os.path.abspath(p)) for p in spec.paths] +
594+
(['os.environ["PATH"]'] if not spec.hasSandbox else
595+
[repr(p) for p in spec.sandboxPaths])
596+
) + '])',
597+
"LD_LIBRARY_PATH": '"' + pathSep + '".join(' +
598+
repr([ os.path.abspath(p) for p in spec.libraryPaths ]) + ')',
599+
"BOB_CWD": repr(os.path.abspath(spec.workspaceExecPath)),
600+
})
601+
602+
ret = [
603+
"# Convenience helpers",
604+
PythonLanguage.PROLOGUE,
605+
"",
606+
"# Special Bob array variables:",
607+
"BOB_ALL_PATHS = dict({})".format(repr(sorted(
608+
[ (name, os.path.abspath(path)) for name, path in spec.allPaths ]
609+
))),
610+
"BOB_DEP_PATHS = dict({})".format(repr(sorted(
611+
[ (name, os.path.abspath(path)) for name,path in spec.depPaths ]
612+
))),
613+
"BOB_TOOL_PATHS = dict({})".format(repr(sorted(
614+
[ (name, os.path.abspath(path)) for name,path in spec.toolPaths ]
615+
))),
616+
'_BOB_TMP_BASE = ' + repr("/tmp" if spec.hasSandbox else os.path.join(tmpDir, "tmp")),
617+
"",
618+
"# Environment:",
619+
"\n".join('os.environ["{}"] = {}'.format(k, v) for (k,v) in sorted(env.items())),
620+
]
621+
return "\n".join(ret)
622+
623+
@staticmethod
624+
def __formatSetup(spec):
625+
return "\n".join([
626+
"",
627+
"# Recipe setup script",
628+
spec.setupScript,
629+
"os.chdir(os.environ['BOB_CWD'])",
630+
])
631+
632+
@staticmethod
633+
def __formatScript(spec, tmpDir, trace):
634+
if spec.envFile:
635+
envFile = "/bob/env" if spec.hasSandbox else os.path.abspath(spec.envFile)
636+
else:
637+
envFile = None
638+
ret = [
639+
PythonLanguage.__formatProlog(spec, tmpDir),
640+
"",
641+
"# Setup",
642+
dedent("""\
643+
with open({ENV_FILE}, "w") as f:
644+
f.write(repr({{ "env" : dict(os.environ), "globals" : globals() }}))
645+
""".format(ENV_FILE=repr(envFile))) if envFile else "",
646+
"os.chdir(os.environ['BOB_CWD'])",
647+
"",
648+
PythonLanguage.__formatSetup(spec),
649+
"",
650+
"# Recipe main script",
651+
spec.mainScript,
652+
]
653+
return "\n".join(ret)
654+
655+
@staticmethod
656+
def __scriptFilePaths(spec, tmpDir):
657+
if spec.hasSandbox:
658+
execScriptFile = "/.script.py"
659+
realScriptFile = (spec.scriptHint or os.path.join(tmpDir, ".script")) + ".py"
660+
else:
661+
execScriptFile = (spec.scriptHint or os.path.join(tmpDir, "script")) + ".py"
662+
realScriptFile = execScriptFile
663+
return (os.path.abspath(realScriptFile), os.path.abspath(execScriptFile))
664+
665+
@staticmethod
666+
def setupShell(spec, tmpDir, keepEnv):
667+
realScriptFile, execScriptFile = PythonLanguage.__scriptFilePaths(spec, tmpDir)
668+
with open(realScriptFile, "w") as f:
669+
f.write(PythonLanguage.__formatProlog(spec, tmpDir))
670+
f.write(PythonLanguage.__formatSetup(spec))
671+
672+
args = [sys.executable, "-i", execScriptFile]
673+
args.extend(os.path.abspath(a) for a in spec.args)
674+
675+
return (realScriptFile, execScriptFile, args)
676+
677+
@staticmethod
678+
def setupCall(spec, tmpDir, keepEnv, trace):
679+
realScriptFile, execScriptFile = PythonLanguage.__scriptFilePaths(spec, tmpDir)
680+
with open(realScriptFile, "w") as f:
681+
f.write(PythonLanguage.__formatScript(spec, tmpDir, trace))
682+
683+
args = [sys.executable, execScriptFile]
684+
args.extend(os.path.abspath(a) for a in spec.args)
685+
686+
return (realScriptFile, execScriptFile, args)
687+
688+
@staticmethod
689+
def mangleFingerprints(scriptFragments, env):
690+
# join the script fragments first
691+
script = joinScripts(scriptFragments, PythonLanguage.glue)
692+
693+
# do not add preamble for empty scripts
694+
if not script: return ""
695+
696+
# Add snippets as they match and a default settings preamble
697+
ret = [script]
698+
for n,v in sorted(env.items()):
699+
ret.append('os.environ["{}"] = {}'.format(k, repr(v)))
700+
ret.append(PythonLanguage.PROLOGUE)
701+
702+
return "\n".join(reversed(ret))
703+
704+
@staticmethod
705+
def setupFingerprint(spec, env):
706+
return [sys.executable, "-c", spec.fingerprintScript]
707+
708+
577709
LANG = {
578710
ScriptLanguage.BASH : BashLanguage,
579711
ScriptLanguage.PWSH : PwshLanguage,
712+
ScriptLanguage.PYTHON : PythonLanguage,
580713
}
581714

582715
def getLanguage(language):

0 commit comments

Comments
 (0)