From c8cb600ac75c59188f96b893a17aa45e04a87c6e Mon Sep 17 00:00:00 2001 From: Nikola Jokic <97525037+nikola-jokic@users.noreply.github.com> Date: Mon, 11 Apr 2022 14:43:24 +0200 Subject: [PATCH] Use StepHost when evaluating inputs to actions (#1762) * composite action github.action_path set based on the StepHost * in progress on updating github context for input template * Fixed updating the context data for evaluation * refactored logic so it is a little cleaner * removed resolving the action_path in CompositeActionHandler * removed added DeepClone * added feature flag and modified the dict in place * refactored step host to change context data. Added L0 * repaired spaces * moved logic from step host to execution context, added recursive translation * removed empty lines * moved to extension methods * Update src/Test/L0/Worker/StepHostL0.cs Co-authored-by: Ferenc Hammerl <31069338+fhammerl@users.noreply.github.com> --- src/Runner.Common/Constants.cs | 1 + src/Runner.Worker/ActionRunner.cs | 23 +++- src/Runner.Worker/ExecutionContext.cs | 68 +++++++++-- src/Runner.Worker/Handlers/StepHost.cs | 7 +- src/Test/L0/Worker/ExecutionContextL0.cs | 148 ++++++++++++++++++++++- src/Test/L0/Worker/StepHostL0.cs | 3 + 6 files changed, 233 insertions(+), 17 deletions(-) diff --git a/src/Runner.Common/Constants.cs b/src/Runner.Common/Constants.cs index 42fb9dcbe60..1a53e0b4d2e 100644 --- a/src/Runner.Common/Constants.cs +++ b/src/Runner.Common/Constants.cs @@ -150,6 +150,7 @@ public static class Features { public static readonly string DiskSpaceWarning = "runner.diskspace.warning"; public static readonly string Node12Warning = "DistributedTask.AddWarningToNode12Action"; + public static readonly string UseContainerPathForTemplate = "DistributedTask.UseContainerPathForTemplate"; } public static readonly string InternalTelemetryIssueDataKey = "_internal_telemetry"; diff --git a/src/Runner.Worker/ActionRunner.cs b/src/Runner.Worker/ActionRunner.cs index a4b82de51d3..18b266e2c90 100644 --- a/src/Runner.Worker/ActionRunner.cs +++ b/src/Runner.Worker/ActionRunner.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Threading.Tasks; using GitHub.DistributedTask.ObjectTemplating; using GitHub.DistributedTask.ObjectTemplating.Tokens; @@ -9,7 +10,6 @@ using GitHub.Runner.Common; using GitHub.Runner.Common.Util; using GitHub.Runner.Sdk; -using GitHub.Runner.Worker; using GitHub.Runner.Worker.Handlers; using Pipelines = GitHub.DistributedTask.Pipelines; @@ -171,8 +171,16 @@ Action.Reference is Pipelines.RepositoryPathReference repoAction && // Load the inputs. ExecutionContext.Debug("Loading inputs"); - var templateEvaluator = ExecutionContext.ToPipelineTemplateEvaluator(); - var inputs = templateEvaluator.EvaluateStepInputs(Action.Inputs, ExecutionContext.ExpressionValues, ExecutionContext.ExpressionFunctions); + Dictionary inputs; + if (ExecutionContext.Global.Variables.GetBoolean(Constants.Runner.Features.UseContainerPathForTemplate) ?? false) + { + inputs = EvaluateStepInputs(stepHost); + } + else + { + var templateEvaluator = ExecutionContext.ToPipelineTemplateEvaluator(); + inputs = templateEvaluator.EvaluateStepInputs(Action.Inputs, ExecutionContext.ExpressionValues, ExecutionContext.ExpressionFunctions); + } var userInputs = new HashSet(StringComparer.OrdinalIgnoreCase); foreach (KeyValuePair input in inputs) @@ -299,6 +307,15 @@ public bool TryEvaluateDisplayName(DictionaryContextData contextData, IExecution return didFullyEvaluate; } + private Dictionary EvaluateStepInputs(IStepHost stepHost) + { + DictionaryContextData expressionValues = ExecutionContext.GetExpressionValues(stepHost); + var templateEvaluator = ExecutionContext.ToPipelineTemplateEvaluator(); + var inputs = templateEvaluator.EvaluateStepInputs(Action.Inputs, expressionValues, ExecutionContext.ExpressionFunctions); + + return inputs; + } + private string GenerateDisplayName(ActionStep action, DictionaryContextData contextData, IExecutionContext context, out bool didFullyEvaluate) { ArgUtil.NotNull(context, nameof(context)); diff --git a/src/Runner.Worker/ExecutionContext.cs b/src/Runner.Worker/ExecutionContext.cs index a2703370994..4fc869c074b 100644 --- a/src/Runner.Worker/ExecutionContext.cs +++ b/src/Runner.Worker/ExecutionContext.cs @@ -1,18 +1,13 @@ using System; -using System.Collections; using System.Collections.Generic; -using System.Collections.Specialized; using System.Globalization; using System.IO; using System.Linq; using System.Text; -using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; -using System.Web; using GitHub.DistributedTask.Expressions2; using GitHub.DistributedTask.ObjectTemplating.Tokens; -using GitHub.DistributedTask.Pipelines; using GitHub.DistributedTask.Pipelines.ContextData; using GitHub.DistributedTask.Pipelines.ObjectTemplating; using GitHub.DistributedTask.WebApi; @@ -20,7 +15,7 @@ using GitHub.Runner.Common.Util; using GitHub.Runner.Sdk; using GitHub.Runner.Worker.Container; -using GitHub.Services.WebApi; +using GitHub.Runner.Worker.Handlers; using Newtonsoft.Json; using ObjectTemplating = GitHub.DistributedTask.ObjectTemplating; using Pipelines = GitHub.DistributedTask.Pipelines; @@ -115,7 +110,6 @@ public interface IExecutionContext : IRunnerService void UpdateGlobalStepsContext(); void WriteWebhookPayload(); - } public sealed class ExecutionContext : RunnerService, IExecutionContext @@ -1199,6 +1193,66 @@ public static ObjectTemplating.ITraceWriter ToTemplateTraceWriter(this IExecutio { return new TemplateTraceWriter(context); } + + public static DictionaryContextData GetExpressionValues(this IExecutionContext context, IStepHost stepHost) + { + if (stepHost is ContainerStepHost) + { + + var expressionValues = context.ExpressionValues.Clone() as DictionaryContextData; + context.UpdatePathsInExpressionValues("github", expressionValues, stepHost); + context.UpdatePathsInExpressionValues("runner", expressionValues, stepHost); + return expressionValues; + } + else + { + return context.ExpressionValues.Clone() as DictionaryContextData; + } + } + + private static void UpdatePathsInExpressionValues(this IExecutionContext context, string contextName, DictionaryContextData expressionValues, IStepHost stepHost) + { + var dict = expressionValues[contextName].AssertDictionary($"expected context {contextName} to be a dictionary"); + context.ResolvePathsInExpressionValuesDictionary(dict, stepHost); + expressionValues[contextName] = dict; + } + + private static void ResolvePathsInExpressionValuesDictionary(this IExecutionContext context, DictionaryContextData dict, IStepHost stepHost) + { + foreach (var key in dict.Keys.ToList()) + { + if (dict[key] is StringContextData) + { + var value = dict[key].ToString(); + if (!string.IsNullOrEmpty(value)) + { + dict[key] = new StringContextData(stepHost.ResolvePathForStepHost(value)); + } + } + else if (dict[key] is DictionaryContextData) + { + var innerDict = dict[key].AssertDictionary("expected dictionary"); + context.ResolvePathsInExpressionValuesDictionary(innerDict, stepHost); + var updatedDict = new DictionaryContextData(); + foreach (var k in innerDict.Keys.ToList()) + { + updatedDict[k] = innerDict[k]; + } + dict[key] = updatedDict; + } + else if (dict[key] is CaseSensitiveDictionaryContextData) + { + var innerDict = dict[key].AssertDictionary("expected dictionary"); + context.ResolvePathsInExpressionValuesDictionary(innerDict, stepHost); + var updatedDict = new CaseSensitiveDictionaryContextData(); + foreach (var k in innerDict.Keys.ToList()) + { + updatedDict[k] = innerDict[k]; + } + dict[key] = updatedDict; + } + } + } } internal sealed class TemplateTraceWriter : ObjectTemplating.ITraceWriter diff --git a/src/Runner.Worker/Handlers/StepHost.cs b/src/Runner.Worker/Handlers/StepHost.cs index 8741c22bbe5..2db55913f72 100644 --- a/src/Runner.Worker/Handlers/StepHost.cs +++ b/src/Runner.Worker/Handlers/StepHost.cs @@ -1,18 +1,13 @@ using System; using System.Collections.Generic; -using System.IO; using System.Text; using System.Threading; -using System.Threading.Channels; using System.Threading.Tasks; -using GitHub.DistributedTask.WebApi; -using GitHub.Runner.Common.Util; using GitHub.Runner.Worker.Container; -using GitHub.Services.WebApi; -using Newtonsoft.Json; using GitHub.Runner.Common; using GitHub.Runner.Sdk; using System.Linq; +using GitHub.DistributedTask.Pipelines.ContextData; namespace GitHub.Runner.Worker.Handlers { diff --git a/src/Test/L0/Worker/ExecutionContextL0.cs b/src/Test/L0/Worker/ExecutionContextL0.cs index 4ae7cc0d58c..d20fe3e7244 100644 --- a/src/Test/L0/Worker/ExecutionContextL0.cs +++ b/src/Test/L0/Worker/ExecutionContextL0.cs @@ -6,6 +6,8 @@ using GitHub.DistributedTask.Pipelines.ContextData; using GitHub.DistributedTask.WebApi; using GitHub.Runner.Worker; +using GitHub.Runner.Worker.Container; +using GitHub.Runner.Worker.Handlers; using Moq; using Xunit; using GitHub.DistributedTask.ObjectTemplating.Tokens; @@ -724,5 +726,149 @@ private TestHostContext CreateTestContext([CallerMemberName] String testName = " return hc; } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void GetExpressionValues_ContainerStepHost() + { + using (TestHostContext hc = CreateTestContext()) + { + const string source = "/home/username/Projects/work/runner/_layout"; + var containerInfo = new ContainerInfo(); + containerInfo.ContainerId = "test"; + + containerInfo.AddPathTranslateMapping($"{source}/_work", "/__w"); + containerInfo.AddPathTranslateMapping($"{source}/_temp", "/__t"); + containerInfo.AddPathTranslateMapping($"{source}/externals", "/__e"); + + containerInfo.AddPathTranslateMapping($"{source}/_work/_temp/_github_home", "/github/home"); + containerInfo.AddPathTranslateMapping($"{source}/_work/_temp/_github_workflow", "/github/workflow"); + + foreach (var v in new List() { + $"{source}/_work", + $"{source}/externals", + $"{source}/_work/_temp", + $"{source}/_work/_actions", + $"{source}/_work/_tool", + }) + { + containerInfo.MountVolumes.Add(new MountVolume(v, containerInfo.TranslateToContainerPath(v))); + }; + + var stepHost = new ContainerStepHost(); + stepHost.Container = containerInfo; + + var ec = new Runner.Worker.ExecutionContext(); + ec.Initialize(hc); + + var inputGithubContext = new GitHubContext(); + var inputeRunnerContext = new RunnerContext(); + + // string context data + inputGithubContext["action_path"] = new StringContextData("/home/username/Projects/work/runner/_layout/_work/_actions/owner/composite/main"); + inputGithubContext["action"] = new StringContextData("__owner_composite"); + inputGithubContext["api_url"] = new StringContextData("https://api.github.com/custom/path"); + inputGithubContext["env"] = new StringContextData("/home/username/Projects/work/runner/_layout/_work/_temp/_runner_file_commands/set_env_265698aa-7f38-40f5-9316-5c01a3153672"); + inputGithubContext["path"] = new StringContextData("/home/username/Projects/work/runner/_layout/_work/_temp/_runner_file_commands/add_path_265698aa-7f38-40f5-9316-5c01a3153672"); + inputGithubContext["event_path"] = new StringContextData("/home/username/Projects/work/runner/_layout/_work/_temp/_github_workflow/event.json"); + inputGithubContext["repository"] = new StringContextData("owner/repo-name"); + inputGithubContext["run_id"] = new StringContextData("2033211332"); + inputGithubContext["workflow"] = new StringContextData("Name of Workflow"); + inputGithubContext["workspace"] = new StringContextData("/home/username/Projects/work/runner/_layout/_work/step-order/step-order"); + inputeRunnerContext["temp"] = new StringContextData("/home/username/Projects/work/runner/_layout/_work/_temp"); + inputeRunnerContext["tool_cache"] = new StringContextData("/home/username/Projects/work/runner/_layout/_work/_tool"); + + // dictionary context data + var githubEvent = new DictionaryContextData(); + githubEvent["inputs"] = null; + githubEvent["ref"] = new StringContextData("refs/heads/main"); + githubEvent["repository"] = new DictionaryContextData(); + githubEvent["sender"] = new DictionaryContextData(); + githubEvent["workflow"] = new StringContextData(".github/workflows/composite_step_host_translate.yaml"); + + inputGithubContext["event"] = githubEvent; + + ec.ExpressionValues["github"] = inputGithubContext; + ec.ExpressionValues["runner"] = inputeRunnerContext; + + var ecExpect = new Runner.Worker.ExecutionContext(); + ecExpect.Initialize(hc); + + var expectedGithubEvent = new DictionaryContextData(); + expectedGithubEvent["inputs"] = null; + expectedGithubEvent["ref"] = new StringContextData("refs/heads/main"); + expectedGithubEvent["repository"] = new DictionaryContextData(); + expectedGithubEvent["sender"] = new DictionaryContextData(); + expectedGithubEvent["workflow"] = new StringContextData(".github/workflows/composite_step_host_translate.yaml"); + var expectedGithubContext = new GitHubContext(); + var expectedRunnerContext = new RunnerContext(); + expectedGithubContext["action_path"] = new StringContextData("/__w/_actions/owner/composite/main"); + expectedGithubContext["action"] = new StringContextData("__owner_composite"); + expectedGithubContext["api_url"] = new StringContextData("https://api.github.com/custom/path"); + expectedGithubContext["env"] = new StringContextData("/__w/_temp/_runner_file_commands/set_env_265698aa-7f38-40f5-9316-5c01a3153672"); + expectedGithubContext["path"] = new StringContextData("/__w/_temp/_runner_file_commands/add_path_265698aa-7f38-40f5-9316-5c01a3153672"); + expectedGithubContext["event_path"] = new StringContextData("/github/workflow/event.json"); + expectedGithubContext["repository"] = new StringContextData("owner/repo-name"); + expectedGithubContext["run_id"] = new StringContextData("2033211332"); + expectedGithubContext["workflow"] = new StringContextData("Name of Workflow"); + expectedGithubContext["workspace"] = new StringContextData("/__w/step-order/step-order"); + expectedGithubContext["event"] = expectedGithubEvent; + expectedRunnerContext["temp"] = new StringContextData("/__w/_temp"); + expectedRunnerContext["tool_cache"] = new StringContextData("/__w/_tool"); + + ecExpect.ExpressionValues["github"] = expectedGithubContext; + ecExpect.ExpressionValues["runner"] = expectedRunnerContext; + + var translatedExpressionValues = ec.GetExpressionValues(stepHost); + + foreach (var contextName in new string[] { "github", "runner" }) + { + var dict = translatedExpressionValues[contextName].AssertDictionary($"expected context github to be a dictionary"); + var expectedExpressionValues = ecExpect.ExpressionValues[contextName].AssertDictionary("expect dict"); + foreach (var key in dict.Keys.ToList()) + { + if (dict[key] is StringContextData) + { + var expect = dict[key].AssertString("expect string"); + var outcome = expectedExpressionValues[key].AssertString("expect string"); + Assert.Equal(expect.Value, outcome.Value); + } + else if (dict[key] is DictionaryContextData || dict[key] is CaseSensitiveDictionaryContextData) + { + var expectDict = dict[key].AssertDictionary("expect dict"); + var actualDict = expectedExpressionValues[key].AssertDictionary("expect dict"); + Assert.True(ExpressionValuesAssertEqual(expectDict, actualDict)); + } + } + } + } + } + + private bool ExpressionValuesAssertEqual(DictionaryContextData expect, DictionaryContextData actual) + { + foreach (var key in expect.Keys.ToList()) + { + if (expect[key] is StringContextData) + { + var expectValue = expect[key].AssertString("expect string"); + var actualValue = actual[key].AssertString("expect string"); + if (expectValue.Equals(actualValue)) + { + return false; + } + } + else if (expect[key] is DictionaryContextData || expect[key] is CaseSensitiveDictionaryContextData) + { + var expectDict = expect[key].AssertDictionary("expect dict"); + var actualDict = actual[key].AssertDictionary("expect dict"); + if (!ExpressionValuesAssertEqual(expectDict, actualDict)) + { + return false; + } + } + } + return true; + } } -} +} \ No newline at end of file diff --git a/src/Test/L0/Worker/StepHostL0.cs b/src/Test/L0/Worker/StepHostL0.cs index 8d0a102efc6..8666e2e0af5 100644 --- a/src/Test/L0/Worker/StepHostL0.cs +++ b/src/Test/L0/Worker/StepHostL0.cs @@ -7,6 +7,9 @@ using GitHub.Runner.Worker; using GitHub.Runner.Worker.Handlers; using GitHub.Runner.Worker.Container; +using GitHub.DistributedTask.Pipelines.ContextData; +using System.Linq; +using GitHub.DistributedTask.Pipelines; namespace GitHub.Runner.Common.Tests.Worker {