From 887c88296e0b3d508864c1dcba361e4e15394335 Mon Sep 17 00:00:00 2001 From: Francisco-Gamino Date: Thu, 6 Oct 2022 11:18:13 -0700 Subject: [PATCH] Add support for the New PowerShell Programming Model --- src/DependencyManagement/DependencyManager.cs | 12 ++-- src/RequestProcessor.cs | 63 +++++++++++++--- src/WorkerIndexing/BindingInformation.cs | 59 +++++++++++++++ src/WorkerIndexing/FunctionInformation.cs | 40 +++++++++++ src/WorkerIndexing/WorkerIndexingHelper.cs | 71 +++++++++++++++++++ src/resources/PowerShellWorkerStrings.resx | 6 ++ 6 files changed, 235 insertions(+), 16 deletions(-) create mode 100644 src/WorkerIndexing/BindingInformation.cs create mode 100644 src/WorkerIndexing/FunctionInformation.cs create mode 100644 src/WorkerIndexing/WorkerIndexingHelper.cs diff --git a/src/DependencyManagement/DependencyManager.cs b/src/DependencyManagement/DependencyManager.cs index 28f4c59b..c0917b3b 100644 --- a/src/DependencyManagement/DependencyManager.cs +++ b/src/DependencyManagement/DependencyManager.cs @@ -44,7 +44,7 @@ internal class DependencyManager : IDisposable #endregion public DependencyManager( - string requestMetadataDirectory = null, + string functionAppRootPath = null, IModuleProvider moduleProvider = null, IDependencyManagerStorage storage = null, IInstalledDependenciesLocator installedDependenciesLocator = null, @@ -54,7 +54,7 @@ public DependencyManager( IBackgroundDependencySnapshotContentLogger currentSnapshotContentLogger = null, ILogger logger = null) { - _storage = storage ?? new DependencyManagerStorage(GetFunctionAppRootPath(requestMetadataDirectory)); + _storage = storage ?? new DependencyManagerStorage(GetFunctionAppRootPath(functionAppRootPath)); _installedDependenciesLocator = installedDependenciesLocator ?? new InstalledDependenciesLocator(_storage, logger); var snapshotContentLogger = new PowerShellModuleSnapshotLogger(); _installer = installer ?? new DependencySnapshotInstaller( @@ -252,14 +252,14 @@ private bool AreAcceptableDependenciesAlreadyInstalled() return _storage.SnapshotExists(_currentSnapshotPath); } - private static string GetFunctionAppRootPath(string requestMetadataDirectory) + private static string GetFunctionAppRootPath(string functionAppRootPath) { - if (string.IsNullOrWhiteSpace(requestMetadataDirectory)) + if (string.IsNullOrWhiteSpace(functionAppRootPath)) { - throw new ArgumentException("Empty request metadata directory path", nameof(requestMetadataDirectory)); + throw new ArgumentException("Empty function app root path", nameof(functionAppRootPath)); } - return Path.GetFullPath(Path.Join(requestMetadataDirectory, "..")); + return functionAppRootPath; } #endregion diff --git a/src/RequestProcessor.cs b/src/RequestProcessor.cs index c9c10d2f..cb7fc057 100644 --- a/src/RequestProcessor.cs +++ b/src/RequestProcessor.cs @@ -18,7 +18,10 @@ namespace Microsoft.Azure.Functions.PowerShellWorker { + using Microsoft.Azure.Functions.PowerShellWorker.WorkerIndexing; + using Microsoft.PowerShell; using System.Diagnostics; + using System.Text.Json; using LogLevel = Microsoft.Azure.WebJobs.Script.Grpc.Messages.RpcLog.Types.Level; internal class RequestProcessor @@ -66,6 +69,8 @@ internal RequestProcessor(MessagingStream msgStream, System.Management.Automatio // If an invocation is cancelled, host will receive an invocation response with status cancelled. _requestHandlers.Add(StreamingMessage.ContentOneofCase.InvocationCancel, ProcessInvocationCancelRequest); + _requestHandlers.Add(StreamingMessage.ContentOneofCase.FunctionsMetadataRequest, ProcessFunctionMetadataRequest); + _requestHandlers.Add(StreamingMessage.ContentOneofCase.FunctionEnvironmentReloadRequest, ProcessFunctionEnvironmentReloadRequest); } @@ -95,6 +100,9 @@ internal async Task ProcessRequestLoop() internal StreamingMessage ProcessWorkerInitRequest(StreamingMessage request) { + var stopwatch = new Stopwatch(); + stopwatch.Start(); + var workerInitRequest = request.WorkerInitRequest; Environment.SetEnvironmentVariable("AZUREPS_HOST_ENVIRONMENT", $"AzureFunctions/{workerInitRequest.HostVersion}"); Environment.SetEnvironmentVariable("POWERSHELL_DISTRIBUTION_CHANNEL", $"Azure-Functions:{workerInitRequest.HostVersion}"); @@ -117,6 +125,32 @@ internal StreamingMessage ProcessWorkerInitRequest(StreamingMessage request) RemoteSessionNamedPipeServer.CreateCustomNamedPipeServer(pipeName); } + // Previously, this half of the dependency management would happen just prior to the dependency download in the + // first function load request. Now that we have the FunctionAppDirectory in the WorkerInitRequest, + // we can do the setup of these variables in the function load request. We need these variables initialized + // for the FunctionMetadataRequest, should it be sent. + try + { + var rpcLogger = new RpcLogger(_msgStream); + rpcLogger.SetContext(request.RequestId, null); + + _dependencyManager = new DependencyManager(request.WorkerInitRequest.FunctionAppDirectory, logger: rpcLogger); + + _powershellPool.Initialize(_firstPwshInstance); + + rpcLogger.Log(isUserOnlyLog: false, LogLevel.Trace, string.Format(PowerShellWorkerStrings.FirstFunctionLoadCompleted, stopwatch.ElapsedMilliseconds)); + } + catch (Exception e) + { + // This is a terminating failure: we will need to return a failure response to + // all subsequent 'FunctionLoadRequest'. Cache the exception so we can reuse it in future calls. + _initTerminatingError = e; + + status.Status = StatusResult.Types.Status.Failure; + status.Exception = e.ToRpcException(); + return response; + } + return response; } @@ -189,26 +223,20 @@ internal StreamingMessage ProcessFunctionLoadRequest(StreamingMessage request) { try { - _isFunctionAppInitialized = true; - var rpcLogger = new RpcLogger(_msgStream); rpcLogger.SetContext(request.RequestId, null); - _dependencyManager = new DependencyManager(request.FunctionLoadRequest.Metadata.Directory, logger: rpcLogger); - var managedDependenciesPath = _dependencyManager.Initialize(request, rpcLogger); - - SetupAppRootPathAndModulePath(functionLoadRequest, managedDependenciesPath); + _isFunctionAppInitialized = true; - _powershellPool.Initialize(_firstPwshInstance); + var managedDependenciesPath = _dependencyManager.Initialize(request, rpcLogger); + SetupAppRootPathAndModulePath(request.FunctionLoadRequest, managedDependenciesPath); // Start the download asynchronously if needed. _dependencyManager.StartDependencyInstallationIfNeeded(request, _firstPwshInstance, rpcLogger); - - rpcLogger.Log(isUserOnlyLog: false, LogLevel.Trace, string.Format(PowerShellWorkerStrings.FirstFunctionLoadCompleted, stopwatch.ElapsedMilliseconds)); } catch (Exception e) { - // Failure that happens during this step is terminating and we will need to return a failure response to + // This is a terminating failure: we will need to return a failure response to // all subsequent 'FunctionLoadRequest'. Cache the exception so we can reuse it in future calls. _initTerminatingError = e; @@ -341,6 +369,18 @@ internal StreamingMessage ProcessInvocationCancelRequest(StreamingMessage reques return null; } + private StreamingMessage ProcessFunctionMetadataRequest(StreamingMessage request) + { + StreamingMessage response = NewStreamingMessageTemplate( + request.RequestId, + StreamingMessage.ContentOneofCase.FunctionMetadataResponse, + out StatusResult status); + + response.FunctionMetadataResponse.FunctionMetadataResults.AddRange(WorkerIndexingHelper.IndexFunctions(request.FunctionsMetadataRequest.FunctionAppDirectory)); + + return response; + } + internal StreamingMessage ProcessFunctionEnvironmentReloadRequest(StreamingMessage request) { var stopwatch = new Stopwatch(); @@ -394,6 +434,9 @@ private StreamingMessage NewStreamingMessageTemplate(string requestId, Streaming case StreamingMessage.ContentOneofCase.FunctionEnvironmentReloadResponse: response.FunctionEnvironmentReloadResponse = new FunctionEnvironmentReloadResponse() { Result = status }; break; + case StreamingMessage.ContentOneofCase.FunctionMetadataResponse: + response.FunctionMetadataResponse = new FunctionMetadataResponse() { Result = status }; + break; default: throw new InvalidOperationException("Unreachable code."); } diff --git a/src/WorkerIndexing/BindingInformation.cs b/src/WorkerIndexing/BindingInformation.cs new file mode 100644 index 00000000..309adf3d --- /dev/null +++ b/src/WorkerIndexing/BindingInformation.cs @@ -0,0 +1,59 @@ +// +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// + +using Microsoft.Azure.WebJobs.Script.Grpc.Messages; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using System; +using System.Collections.Generic; + +namespace Microsoft.Azure.Functions.PowerShellWorker.WorkerIndexing +{ + internal class BindingInformation + { + private const string BindingNameKey = "name"; + private const string BindingDirectionKey = "direction"; + private const string BindingTypeKey = "type"; + public enum Directions + { + Unknown = -1, + In = 0, + Out = 1, + Inout = 2 + } + + public Directions Direction { get; set; } = Directions.Unknown; + public string Type { get; set; } = ""; + public string Name { get; set; } = ""; + public Dictionary otherInformation { get; set; } = new Dictionary(); + + internal string ConvertToRpcRawBinding(out BindingInfo bindingInfo) + { + string rawBinding = string.Empty; + JObject rawBindingObject = new JObject(); + rawBindingObject.Add(BindingNameKey, Name); + BindingInfo outInfo = new BindingInfo(); + + + if (Direction == Directions.Unknown) + { + throw new Exception(string.Format(PowerShellWorkerStrings.InvalidBindingInfoDirection, Name)); + } + outInfo.Direction = (BindingInfo.Types.Direction)Direction; + rawBindingObject.Add(BindingDirectionKey, Enum.GetName(typeof(BindingInfo.Types.Direction), outInfo.Direction).ToLower()); + outInfo.Type = Type; + rawBindingObject.Add(BindingTypeKey, Type); + + foreach (KeyValuePair pair in otherInformation) + { + rawBindingObject.Add(pair.Key, JToken.FromObject(pair.Value)); + } + + rawBinding = JsonConvert.SerializeObject(rawBindingObject); + bindingInfo = outInfo; + return rawBinding; + } + } +} diff --git a/src/WorkerIndexing/FunctionInformation.cs b/src/WorkerIndexing/FunctionInformation.cs new file mode 100644 index 00000000..2408225b --- /dev/null +++ b/src/WorkerIndexing/FunctionInformation.cs @@ -0,0 +1,40 @@ +// +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// + +using Microsoft.Azure.WebJobs.Script.Grpc.Messages; +using System.Collections.Generic; + +namespace Microsoft.Azure.Functions.PowerShellWorker.WorkerIndexing +{ + internal class FunctionInformation + { + private const string FunctionLanguagePowerShell = "powershell"; + + public string Directory { get; set; } = ""; + public string ScriptFile { get; set; } = ""; + public string Name { get; set; } = ""; + public string EntryPoint { get; set; } = ""; + public string FunctionId { get; set; } = ""; + public List Bindings { get; set; } = new List(); + + internal RpcFunctionMetadata ConvertToRpc() + { + RpcFunctionMetadata returnMetadata = new RpcFunctionMetadata(); + returnMetadata.FunctionId = FunctionId; + returnMetadata.Directory = Directory; + returnMetadata.EntryPoint = EntryPoint; + returnMetadata.Name = Name; + returnMetadata.ScriptFile = ScriptFile; + returnMetadata.Language = FunctionLanguagePowerShell; + foreach(BindingInformation binding in Bindings) + { + string rawBinding = binding.ConvertToRpcRawBinding(out BindingInfo bindingInfo); + returnMetadata.Bindings.Add(binding.Name, bindingInfo); + returnMetadata.RawBindings.Add(rawBinding); + } + return returnMetadata; + } + } +} diff --git a/src/WorkerIndexing/WorkerIndexingHelper.cs b/src/WorkerIndexing/WorkerIndexingHelper.cs new file mode 100644 index 00000000..74304c3c --- /dev/null +++ b/src/WorkerIndexing/WorkerIndexingHelper.cs @@ -0,0 +1,71 @@ +// +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// + +using Microsoft.Azure.WebJobs.Script.Grpc.Messages; +using Newtonsoft.Json; +using System; +using System.Collections.Generic; +using System.Management.Automation; +using System.Management.Automation.Runspaces; + +namespace Microsoft.Azure.Functions.PowerShellWorker.WorkerIndexing +{ + internal class WorkerIndexingHelper + { + const string GetFunctionsMetadataCmdletName = "AzureFunctions.PowerShell.SDK\\Get-FunctionsMetadata"; + internal static IEnumerable IndexFunctions(string baseDir) + { + List indexedFunctions = new List(); + + // This is not the correct way to deal with getting a runspace for the cmdlet. + + // Firstly, creating a runspace is expensive. If we are going to generate a runspace, it should be done on + // the function load request so that it can be created while the host is processing. + + // Secondly, this assumes that the AzureFunctions.PowerShell.SDK module is present on the machine/VM's + // PSModulePath. On an Azure instance, it will not be. What we need to do here is move the call + // to SetupAppRootPathAndModulePath in RequestProcessor to the init request, and then use the + // _firstPwshInstance to invoke the Get-FunctionsMetadata command. The only issue with this is that + // SetupAppRootPathAndModulePath needs the initial function init request in order to know if managed + // dependencies are enabled in this function app. + + // Proposed solutions: + // 1. Pass ManagedDependencyEnabled flag in the worker init request + // 2. Change the flow, so that _firstPwshInstance is initialized in worker init with the PSModulePath + // assuming that managed dependencies are enabled, and then revert the PSModulePath in the first function + // init request should the managed dependencies not be enabled. + // 3. Continue using a new runspace for invoking Get-FunctionsMetadata, but initialize it in worker init and + // point the PsModulePath to the module path bundled with the worker. + + + InitialSessionState initial = InitialSessionState.CreateDefault(); + Runspace runspace = RunspaceFactory.CreateRunspace(initial); + runspace.Open(); + System.Management.Automation.PowerShell _powershell = System.Management.Automation.PowerShell.Create(); + _powershell.Runspace = runspace; + + _powershell.AddCommand(GetFunctionsMetadataCmdletName).AddArgument(baseDir); + string outputString = string.Empty; + foreach (PSObject rawMetadata in _powershell.Invoke()) + { + if (outputString != string.Empty) + { + throw new Exception(PowerShellWorkerStrings.GetFunctionsMetadataMultipleResultsError); + } + outputString = rawMetadata.ToString(); + } + _powershell.Commands.Clear(); + + List functionInformations = JsonConvert.DeserializeObject>(outputString); + + foreach(FunctionInformation fi in functionInformations) + { + indexedFunctions.Add(fi.ConvertToRpc()); + } + + return indexedFunctions; + } + } +} diff --git a/src/resources/PowerShellWorkerStrings.resx b/src/resources/PowerShellWorkerStrings.resx index 2d0228d7..561e5456 100644 --- a/src/resources/PowerShellWorkerStrings.resx +++ b/src/resources/PowerShellWorkerStrings.resx @@ -352,4 +352,10 @@ Dependency snapshot '{0}' does not contain acceptable module versions. + + Multiple results from metadata cmdlet. + + + Invalid binding direction. Binding name: {0} + \ No newline at end of file