From f32160a3969fd858afacc318f3abd7562eeb5991 Mon Sep 17 00:00:00 2001 From: Shyju Krishnankutty Date: Tue, 10 Oct 2023 13:54:09 -0700 Subject: [PATCH 1/8] Handling env reload response from native placeholder for failure case. --- .../Channel/GrpcWorkerChannel.cs | 53 +++++++++++++++++-- .../Workers/Rpc/IRpcWorkerChannel.cs | 2 +- .../Rpc/WebHostRpcWorkerChannelManager.cs | 6 ++- .../Workers/Rpc/TestRpcWorkerChannel.cs | 4 +- 4 files changed, 55 insertions(+), 10 deletions(-) diff --git a/src/WebJobs.Script.Grpc/Channel/GrpcWorkerChannel.cs b/src/WebJobs.Script.Grpc/Channel/GrpcWorkerChannel.cs index 045f0ae3ff..767b1f669a 100644 --- a/src/WebJobs.Script.Grpc/Channel/GrpcWorkerChannel.cs +++ b/src/WebJobs.Script.Grpc/Channel/GrpcWorkerChannel.cs @@ -376,13 +376,55 @@ internal void FunctionEnvironmentReloadResponse(FunctionEnvironmentReloadRespons if (res.Result.IsFailure(out Exception reloadEnvironmentVariablesException)) { - _workerChannelLogger.LogError(reloadEnvironmentVariablesException, "Failed to reload environment variables"); - _reloadTask.SetException(reloadEnvironmentVariablesException); + if (IsFatalFailure(res)) + { + _workerChannelLogger.LogError(reloadEnvironmentVariablesException, "Failed to reload environment variables"); + _reloadTask.SetException(reloadEnvironmentVariablesException); + } + else + { + _reloadTask.SetResult(false); + } + } + else + { + _reloadTask.SetResult(true); } - _reloadTask.SetResult(true); latencyEvent.Dispose(); } + /// + /// Determines whether the environment reload failure should be considered as a fatal error or should be ignored. + /// + internal bool IsFatalFailure(FunctionEnvironmentReloadResponse response) + { + var workerRuntime = _environment.GetFunctionsWorkerRuntime(); + + // Currently we support non fatal reload errors for dotnet isolated worker only. + if (workerRuntime != RpcWorkerConstants.DotNetIsolatedLanguageWorkerName) + { + return true; + } + + var rpcException = response.Result.Exception; + + // Currently the below 2 errors are considered as non fatal. + if (rpcException.Message.StartsWith("Microsoft.Azure.Functions.Worker.EnvironmentReloadUnsupportedException")) + { + _workerChannelLogger.LogDebug(new EventId(422, ScriptConstants.PlaceholderMissDueToBitnessEventName), + " This app is not using the latest version of Microsoft.Azure.Functions.Worker SDK and therefore does not leverage all performance optimizations. See https://aka.ms/azure-functions/dotnet/placeholders for more information."); + return false; + } + if (rpcException.Message.StartsWith("Microsoft.Azure.Functions.Worker.FunctionAppPayloadNotFoundException")) + { + // If no app payload present, we can ignore the error because + // it is possible that the app was created (instance assigned), but no deployment has happened yet. + return false; + } + + return true; + } + internal void WorkerInitResponse(GrpcEvent initEvent) { _startLatencyMetric?.Dispose(); @@ -546,7 +588,7 @@ internal FunctionLoadRequestCollection GetFunctionLoadRequestCollection(IEnumera return functionLoadRequestCollection; } - public Task SendFunctionEnvironmentReloadRequest() + public Task SendFunctionEnvironmentReloadRequest() { _functionsIndexingTask = new TaskCompletionSource>(TaskCreationOptions.RunContinuationsAsynchronously); _functionMetadataRequestSent = false; @@ -566,7 +608,8 @@ public Task SendFunctionEnvironmentReloadRequest() FunctionEnvironmentReloadRequest = request }); - return _reloadTask.Task; + Task t = _reloadTask.Task; + return t; } public void SendWorkerWarmupRequest() diff --git a/src/WebJobs.Script/Workers/Rpc/IRpcWorkerChannel.cs b/src/WebJobs.Script/Workers/Rpc/IRpcWorkerChannel.cs index df9bde5278..d9d3e6eb9c 100644 --- a/src/WebJobs.Script/Workers/Rpc/IRpcWorkerChannel.cs +++ b/src/WebJobs.Script/Workers/Rpc/IRpcWorkerChannel.cs @@ -22,7 +22,7 @@ public interface IRpcWorkerChannel : IWorkerChannel void SendFunctionLoadRequests(ManagedDependencyOptions managedDependencyOptions, TimeSpan? functionTimeout); - Task SendFunctionEnvironmentReloadRequest(); + Task SendFunctionEnvironmentReloadRequest(); void SendWorkerWarmupRequest(); diff --git a/src/WebJobs.Script/Workers/Rpc/WebHostRpcWorkerChannelManager.cs b/src/WebJobs.Script/Workers/Rpc/WebHostRpcWorkerChannelManager.cs index c02c23354a..c2dcd91b39 100644 --- a/src/WebJobs.Script/Workers/Rpc/WebHostRpcWorkerChannelManager.cs +++ b/src/WebJobs.Script/Workers/Rpc/WebHostRpcWorkerChannelManager.cs @@ -123,12 +123,14 @@ public async Task SpecializeAsync() if (_workerRuntime != null && rpcWorkerChannel != null) { + bool envReloadRequestResult = false; if (UsePlaceholderChannel(rpcWorkerChannel)) { _logger.LogDebug("Loading environment variables for runtime: {runtime}", _workerRuntime); - await rpcWorkerChannel.SendFunctionEnvironmentReloadRequest(); + envReloadRequestResult = await rpcWorkerChannel.SendFunctionEnvironmentReloadRequest(); } - else + + if (envReloadRequestResult == false) { _logger.LogDebug("Shutting down placeholder worker. Worker is not compatible for runtime: {runtime}", _workerRuntime); // If we need to allow file edits, we should shutdown the webhost channel on specialization. diff --git a/test/WebJobs.Script.Tests/Workers/Rpc/TestRpcWorkerChannel.cs b/test/WebJobs.Script.Tests/Workers/Rpc/TestRpcWorkerChannel.cs index c65e15d7fa..c7b6145565 100644 --- a/test/WebJobs.Script.Tests/Workers/Rpc/TestRpcWorkerChannel.cs +++ b/test/WebJobs.Script.Tests/Workers/Rpc/TestRpcWorkerChannel.cs @@ -74,10 +74,10 @@ public void SendFunctionLoadRequests(ManagedDependencyOptions managedDependencie _testLogger.LogInformation("RegisterFunctions called"); } - public Task SendFunctionEnvironmentReloadRequest() + public Task SendFunctionEnvironmentReloadRequest() { _testLogger.LogInformation("SendFunctionEnvironmentReloadRequest called"); - return Task.CompletedTask; + return Task.FromResult(true); } public void SendInvocationRequest(ScriptInvocationContext context) From 22c2cdb31230d34cc76294200d5f2c34d53b44c6 Mon Sep 17 00:00:00 2001 From: Shyju Krishnankutty Date: Wed, 11 Oct 2023 17:06:43 -0700 Subject: [PATCH 2/8] Almost working except one test needs cleanup --- .../Channel/GrpcWorkerChannel.cs | 4 +- src/WebJobs.Script/WebJobs.Script.csproj | 2 +- test/DotNetIsolated60/DotNetIsolated60.csproj | 6 +- test/DotNetIsolated60/DotNetIsolated60.sln | 6 ++ test/DotNetIsolated60/Program.cs | 6 +- .../DotNetIsolatedUnsupportedWorker.csproj | 31 +++++++++ .../HttpRequestDataFunction.cs | 30 +++++++++ .../HttpRequestFunction.cs | 23 +++++++ .../Program.cs | 23 +++++++ .../QueueFunction.cs | 21 ++++++ .../DotNetIsolatedUnsupportedWorker/host.json | 12 ++++ test/EmptyScriptRoot/host.json | 7 ++ .../WebHostEndToEnd/SpecializationE2ETests.cs | 64 +++++++++++++++---- 13 files changed, 214 insertions(+), 21 deletions(-) create mode 100644 test/DotNetIsolatedUnsupportedWorker/DotNetIsolatedUnsupportedWorker.csproj create mode 100644 test/DotNetIsolatedUnsupportedWorker/HttpRequestDataFunction.cs create mode 100644 test/DotNetIsolatedUnsupportedWorker/HttpRequestFunction.cs create mode 100644 test/DotNetIsolatedUnsupportedWorker/Program.cs create mode 100644 test/DotNetIsolatedUnsupportedWorker/QueueFunction.cs create mode 100644 test/DotNetIsolatedUnsupportedWorker/host.json create mode 100644 test/EmptyScriptRoot/host.json diff --git a/src/WebJobs.Script.Grpc/Channel/GrpcWorkerChannel.cs b/src/WebJobs.Script.Grpc/Channel/GrpcWorkerChannel.cs index 767b1f669a..291aa7d9f9 100644 --- a/src/WebJobs.Script.Grpc/Channel/GrpcWorkerChannel.cs +++ b/src/WebJobs.Script.Grpc/Channel/GrpcWorkerChannel.cs @@ -409,7 +409,7 @@ internal bool IsFatalFailure(FunctionEnvironmentReloadResponse response) var rpcException = response.Result.Exception; // Currently the below 2 errors are considered as non fatal. - if (rpcException.Message.StartsWith("Microsoft.Azure.Functions.Worker.EnvironmentReloadUnsupportedException")) + if (rpcException.Message.StartsWith("Microsoft.Azure.Functions.Worker.EnvironmentReloadNotSupportedException")) { _workerChannelLogger.LogDebug(new EventId(422, ScriptConstants.PlaceholderMissDueToBitnessEventName), " This app is not using the latest version of Microsoft.Azure.Functions.Worker SDK and therefore does not leverage all performance optimizations. See https://aka.ms/azure-functions/dotnet/placeholders for more information."); @@ -418,7 +418,7 @@ internal bool IsFatalFailure(FunctionEnvironmentReloadResponse response) if (rpcException.Message.StartsWith("Microsoft.Azure.Functions.Worker.FunctionAppPayloadNotFoundException")) { // If no app payload present, we can ignore the error because - // it is possible that the app was created (instance assigned), but no deployment has happened yet. + // it is possible that the app was created (instance assigned from platform), but no deployment has happened yet. return false; } diff --git a/src/WebJobs.Script/WebJobs.Script.csproj b/src/WebJobs.Script/WebJobs.Script.csproj index a76dd6d946..e5df2f43a1 100644 --- a/src/WebJobs.Script/WebJobs.Script.csproj +++ b/src/WebJobs.Script/WebJobs.Script.csproj @@ -47,7 +47,7 @@ - + diff --git a/test/DotNetIsolated60/DotNetIsolated60.csproj b/test/DotNetIsolated60/DotNetIsolated60.csproj index 8e602a3cd7..a6d802816b 100644 --- a/test/DotNetIsolated60/DotNetIsolated60.csproj +++ b/test/DotNetIsolated60/DotNetIsolated60.csproj @@ -6,13 +6,15 @@ enable enable True + True + True - + - + diff --git a/test/DotNetIsolated60/DotNetIsolated60.sln b/test/DotNetIsolated60/DotNetIsolated60.sln index 1f839fdf73..286c56b973 100644 --- a/test/DotNetIsolated60/DotNetIsolated60.sln +++ b/test/DotNetIsolated60/DotNetIsolated60.sln @@ -5,6 +5,8 @@ VisualStudioVersion = 17.5.33627.172 MinimumVisualStudioVersion = 10.0.40219.1 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DotNetIsolated60", "DotNetIsolated60.csproj", "{1DA92227-F28E-408D-96B1-20C72571E4AE}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DotNetIsolatedUnsupportedWorker", "..\DotNetIsolatedUnsupportedWorker\DotNetIsolatedUnsupportedWorker.csproj", "{3F15B936-6365-447E-9EC6-4E996B30C55F}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -15,6 +17,10 @@ Global {1DA92227-F28E-408D-96B1-20C72571E4AE}.Debug|Any CPU.Build.0 = Debug|Any CPU {1DA92227-F28E-408D-96B1-20C72571E4AE}.Release|Any CPU.ActiveCfg = Release|Any CPU {1DA92227-F28E-408D-96B1-20C72571E4AE}.Release|Any CPU.Build.0 = Release|Any CPU + {3F15B936-6365-447E-9EC6-4E996B30C55F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {3F15B936-6365-447E-9EC6-4E996B30C55F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {3F15B936-6365-447E-9EC6-4E996B30C55F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {3F15B936-6365-447E-9EC6-4E996B30C55F}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/test/DotNetIsolated60/Program.cs b/test/DotNetIsolated60/Program.cs index 2d2691d7d4..29dd670851 100644 --- a/test/DotNetIsolated60/Program.cs +++ b/test/DotNetIsolated60/Program.cs @@ -11,14 +11,12 @@ if (useProxy) { hostBuilder - .ConfigureFunctionsWebApplication() - .ConfigureGeneratedFunctionMetadataProvider(); + .ConfigureFunctionsWebApplication(); } else { hostBuilder - .ConfigureFunctionsWorkerDefaults() - .ConfigureGeneratedFunctionMetadataProvider(); + .ConfigureFunctionsWorkerDefaults(); } var host = hostBuilder.Build(); diff --git a/test/DotNetIsolatedUnsupportedWorker/DotNetIsolatedUnsupportedWorker.csproj b/test/DotNetIsolatedUnsupportedWorker/DotNetIsolatedUnsupportedWorker.csproj new file mode 100644 index 0000000000..8cb09f29e9 --- /dev/null +++ b/test/DotNetIsolatedUnsupportedWorker/DotNetIsolatedUnsupportedWorker.csproj @@ -0,0 +1,31 @@ + + + net6.0 + v4 + Exe + enable + enable + True + True + True + + + + + + + + + + + PreserveNewest + + + PreserveNewest + Never + + + + + + \ No newline at end of file diff --git a/test/DotNetIsolatedUnsupportedWorker/HttpRequestDataFunction.cs b/test/DotNetIsolatedUnsupportedWorker/HttpRequestDataFunction.cs new file mode 100644 index 0000000000..f494114440 --- /dev/null +++ b/test/DotNetIsolatedUnsupportedWorker/HttpRequestDataFunction.cs @@ -0,0 +1,30 @@ +using System.Net; +using Microsoft.Azure.Functions.Worker; +using Microsoft.Azure.Functions.Worker.Http; +using Microsoft.Extensions.Logging; + +namespace DotNetIsolatedUnsupportedWorker +{ + public class HttpRequestDataFunction + { + private readonly ILogger _logger; + + public HttpRequestDataFunction(ILoggerFactory loggerFactory) + { + _logger = loggerFactory.CreateLogger(); + } + + [Function("HttpRequestDataFunction")] + public HttpResponseData Run([HttpTrigger(AuthorizationLevel.Anonymous, "get", "post")] HttpRequestData req) + { + _logger.LogInformation("C# HTTP trigger function processed a request."); + + var response = req.CreateResponse(HttpStatusCode.OK); + response.Headers.Add("Content-Type", "text/plain; charset=utf-8"); + + response.WriteString("Welcome to Azure Functions!"); + + return response; + } + } +} diff --git a/test/DotNetIsolatedUnsupportedWorker/HttpRequestFunction.cs b/test/DotNetIsolatedUnsupportedWorker/HttpRequestFunction.cs new file mode 100644 index 0000000000..ba6df7f15b --- /dev/null +++ b/test/DotNetIsolatedUnsupportedWorker/HttpRequestFunction.cs @@ -0,0 +1,23 @@ +using Microsoft.AspNetCore.Http; +using Microsoft.Azure.Functions.Worker; +using Microsoft.Extensions.Logging; + +namespace DotNetIsolatedUnsupportedWorker +{ + public class HttpRequestFunction + { + private readonly ILogger _logger; + + public HttpRequestFunction(ILogger logger) + { + _logger = logger; + } + + [Function(nameof(HttpRequestFunction))] + public Task Run([HttpTrigger(AuthorizationLevel.Anonymous, "get", "post")] HttpRequest req) + { + _logger.LogInformation("C# HTTP trigger function processed a request."); + return req.HttpContext.Response.WriteAsync("Welcome to Azure Functions!"); + } + } +} diff --git a/test/DotNetIsolatedUnsupportedWorker/Program.cs b/test/DotNetIsolatedUnsupportedWorker/Program.cs new file mode 100644 index 0000000000..29dd670851 --- /dev/null +++ b/test/DotNetIsolatedUnsupportedWorker/Program.cs @@ -0,0 +1,23 @@ +using Microsoft.Azure.Functions.Worker; +using Microsoft.Extensions.Hosting; + +//Debugger.Launch(); + +// Tests can set an env var that will swap this to use the proxy +bool useProxy = Environment.GetEnvironmentVariable("UseProxyInTest")?.Contains("1") ?? false; + +var hostBuilder = new HostBuilder(); + +if (useProxy) +{ + hostBuilder + .ConfigureFunctionsWebApplication(); +} +else +{ + hostBuilder + .ConfigureFunctionsWorkerDefaults(); +} + +var host = hostBuilder.Build(); +host.Run(); diff --git a/test/DotNetIsolatedUnsupportedWorker/QueueFunction.cs b/test/DotNetIsolatedUnsupportedWorker/QueueFunction.cs new file mode 100644 index 0000000000..69dd183369 --- /dev/null +++ b/test/DotNetIsolatedUnsupportedWorker/QueueFunction.cs @@ -0,0 +1,21 @@ +using Microsoft.Azure.Functions.Worker; +using Microsoft.Extensions.Logging; + +namespace DotNetIsolatedUnsupportedWorker +{ + public class QueueFunction + { + private readonly ILogger _logger; + + public QueueFunction(ILoggerFactory loggerFactory) + { + _logger = loggerFactory.CreateLogger(); + } + + [Function("QueueFunction")] + public void Run([QueueTrigger("myqueue-items", Connection = "AzureWebJobsStorage")] string myQueueItem) + { + _logger.LogInformation($"C# Queue trigger function processed: {myQueueItem}"); + } + } +} diff --git a/test/DotNetIsolatedUnsupportedWorker/host.json b/test/DotNetIsolatedUnsupportedWorker/host.json new file mode 100644 index 0000000000..ee5cf5f83f --- /dev/null +++ b/test/DotNetIsolatedUnsupportedWorker/host.json @@ -0,0 +1,12 @@ +{ + "version": "2.0", + "logging": { + "applicationInsights": { + "samplingSettings": { + "isEnabled": true, + "excludedTypes": "Request" + }, + "enableLiveMetricsFilters": true + } + } +} \ No newline at end of file diff --git a/test/EmptyScriptRoot/host.json b/test/EmptyScriptRoot/host.json new file mode 100644 index 0000000000..55d16424d6 --- /dev/null +++ b/test/EmptyScriptRoot/host.json @@ -0,0 +1,7 @@ +{ + "version": "2.0", + "extensionBundle": { + "id": "Microsoft.Azure.Functions.ExtensionBundle", + "version": "[4.*, 5.0.0)" + } +} \ No newline at end of file diff --git a/test/WebJobs.Script.Tests.Integration/WebHostEndToEnd/SpecializationE2ETests.cs b/test/WebJobs.Script.Tests.Integration/WebHostEndToEnd/SpecializationE2ETests.cs index 27e343fc73..044c4d1e84 100644 --- a/test/WebJobs.Script.Tests.Integration/WebHostEndToEnd/SpecializationE2ETests.cs +++ b/test/WebJobs.Script.Tests.Integration/WebHostEndToEnd/SpecializationE2ETests.cs @@ -51,6 +51,8 @@ public class SpecializationE2ETests private static readonly string _scriptRootConfigPath = ConfigurationPath.Combine(ConfigurationSectionNames.WebHost, nameof(ScriptApplicationHostOptions.ScriptPath)); private static readonly string _dotnetIsolated60Path = Path.GetFullPath(@"..\..\..\..\DotNetIsolated60\bin\Debug\net6.0"); + private static readonly string _dotnetIsolatedUnsuppportedPath = Path.GetFullPath(@"..\..\..\..\DotNetIsolatedUnsupportedWorker\bin\Debug\net6.0"); + private static readonly string _dotnetIsolatedEmptyScriptRoot = Path.GetFullPath(@"..\..\..\..\EmptyScriptRoot"); private const string _specializedScriptRoot = @"TestScripts\CSharp"; @@ -799,7 +801,7 @@ public async Task Specialization_JobHostInternalStorageOptionsUpdatesWithActiveH [Fact] public async Task DotNetIsolated_PlaceholderHit() { - var builder = InitializeDotNetIsolatedPlaceholderBuilder("HttpRequestDataFunction"); + var builder = InitializeDotNetIsolatedPlaceholderBuilder(_dotnetIsolated60Path, "HttpRequestDataFunction"); using var testServer = new TestServer(builder); @@ -839,7 +841,7 @@ public async Task DotNetIsolated_PlaceholderHit_WithProxies() { // This test ensures that capabilities are correctly applied in EnvironmentReload during // specialization - var builder = InitializeDotNetIsolatedPlaceholderBuilder("HttpRequestFunction"); + var builder = InitializeDotNetIsolatedPlaceholderBuilder(_dotnetIsolated60Path, "HttpRequestFunction"); using var testServer = new TestServer(builder); @@ -888,7 +890,7 @@ public async Task DotNetIsolated_PlaceholderHit_WithProxies() public async Task DotNetIsolated_PlaceholderMiss_EnvVar() { // Placeholder miss if the WEBSITE_USE_PLACEHOLDER_DOTNETISOLATED env var is not set - await DotNetIsolatedPlaceholderMiss(); + await DotNetIsolatedPlaceholderMiss(_dotnetIsolated60Path); var log = _loggerProvider.GetLog(); Assert.Contains("UsePlaceholderDotNetIsolated: False", log); @@ -901,7 +903,7 @@ public async Task DotNetIsolated_PlaceholderMiss_Not64Bit() _environment.SetProcessBitness(is64Bitness: false); // We only specialize when host process is 64 bit process. - await DotNetIsolatedPlaceholderMiss(() => + await DotNetIsolatedPlaceholderMiss(_dotnetIsolated60Path , () => { _environment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebsiteUsePlaceholderDotNetIsolated, "1"); _environment.SetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeVersionSettingName, "6.0"); @@ -918,7 +920,7 @@ public async Task DotNetIsolated_PlaceholderMiss_DotNetVer() { // Even with placeholders enabled via the WEBSITE_USE_PLACEHOLDER_DOTNETISOLATED env var, // if the dotnet version does not match, we should not use the placeholder - await DotNetIsolatedPlaceholderMiss(() => + await DotNetIsolatedPlaceholderMiss(_dotnetIsolated60Path, () => { _environment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebsiteUsePlaceholderDotNetIsolated, "1"); _environment.SetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeVersionSettingName, "7.0"); @@ -930,6 +932,36 @@ await DotNetIsolatedPlaceholderMiss(() => Assert.Contains("Shutting down placeholder worker. Worker is not compatible for runtime: dotnet-isolated", log); } + [Fact] + public async Task DotNetIsolated_PlaceholderMiss_UnsupportedWorkerPackage() + { + await DotNetIsolatedPlaceholderMiss(_dotnetIsolatedUnsuppportedPath, () => + { + _environment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebsiteUsePlaceholderDotNetIsolated, "1"); + _environment.SetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeVersionSettingName, "6.0"); + }); + + var log = _loggerProvider.GetLog(); + Assert.Contains("UsePlaceholderDotNetIsolated: True", log); + Assert.Contains("Placeholder runtime version: '6.0'. Site runtime version: '6.0'. Match: True", log); + Assert.Contains("Shutting down placeholder worker. Worker is not compatible for runtime: dotnet-isolated", log); + } + + [Fact] + public async Task DotNetIsolated_PlaceholderMiss_EmptyScriptRoot() + { + await DotNetIsolatedPlaceholderMiss(_dotnetIsolatedEmptyScriptRoot, () => + { + _environment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebsiteUsePlaceholderDotNetIsolated, "1"); + _environment.SetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeVersionSettingName, "6.0"); + }); + + var log = _loggerProvider.GetLog(); + Assert.Contains("UsePlaceholderDotNetIsolated: True", log); + Assert.Contains("Placeholder runtime version: '6.0'. Site runtime version: '6.0'. Match: True", log); + Assert.Contains("Shutting down placeholder worker. Worker is not compatible for runtime: dotnet-isolated", log); + } + [Fact] // Fix for https://github.com/Azure/azure-functions-host/issues/9288 public async Task SpecializedSite_StopsHostBeforeWorker() @@ -943,7 +975,7 @@ public async Task SpecializedSite_StopsHostBeforeWorker() await queue.CreateIfNotExistsAsync(); await queue.ClearAsync(); - var builder = InitializeDotNetIsolatedPlaceholderBuilder("HttpRequestDataFunction", "QueueFunction"); + var builder = InitializeDotNetIsolatedPlaceholderBuilder(_dotnetIsolated60Path, "HttpRequestDataFunction", "QueueFunction"); using var testServer = new TestServer(builder); @@ -1005,9 +1037,9 @@ await TestHelpers.Await(() => Assert.Empty(completedLogs.Where(p => p.Level == LogLevel.Error)); } - private async Task DotNetIsolatedPlaceholderMiss(Action additionalSpecializedSetup = null) + private async Task DotNetIsolatedPlaceholderMiss(string scriptRootPath, Action additionalSpecializedSetup = null) { - var builder = InitializeDotNetIsolatedPlaceholderBuilder("HttpRequestDataFunction"); + var builder = InitializeDotNetIsolatedPlaceholderBuilder(scriptRootPath, "HttpRequestDataFunction"); // remove WEBSITE_USE_PLACEHOLDER_DOTNETISOLATED _environment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebsiteUsePlaceholderDotNetIsolated, null); @@ -1033,12 +1065,20 @@ private async Task DotNetIsolatedPlaceholderMiss(Action additionalSpecializedSet additionalSpecializedSetup?.Invoke(); response = await client.GetAsync("api/HttpRequestDataFunction"); - response.EnsureSuccessStatusCode(); + if (scriptRootPath == _dotnetIsolatedEmptyScriptRoot) + { + Assert.Equal(HttpStatusCode.NotFound, response.StatusCode); + } + else + { + response.EnsureSuccessStatusCode(); + } + var expectedProcessName = scriptRootPath == _dotnetIsolated60Path ? "DotNetIsolated60" : "DotNetIsolatedUnsupported"; // Placeholder miss; new channel should be started using the deployed worker directly var specializedChannel = await webChannelManager.GetChannels("dotnet-isolated").Single().Value.Task; Assert.Contains("dotnet.exe", specializedChannel.WorkerProcess.Process.StartInfo.FileName); - Assert.Contains("DotNetIsolated60", specializedChannel.WorkerProcess.Process.StartInfo.Arguments); + Assert.Contains(expectedProcessName, specializedChannel.WorkerProcess.Process.StartInfo.Arguments); runningProcess = Process.GetProcessById(specializedChannel.WorkerProcess.Id); Assert.Contains(runningProcess.ProcessName, "dotnet"); @@ -1053,7 +1093,7 @@ private static void BuildDotnetIsolated60() p.WaitForExit(); } - private IWebHostBuilder InitializeDotNetIsolatedPlaceholderBuilder(params string[] functions) + private IWebHostBuilder InitializeDotNetIsolatedPlaceholderBuilder(string scriptRootPath, params string[] functions) { BuildDotnetIsolated60(); @@ -1068,7 +1108,7 @@ private IWebHostBuilder InitializeDotNetIsolatedPlaceholderBuilder(params string { config.AddInMemoryCollection(new Dictionary { - { _scriptRootConfigPath, _dotnetIsolated60Path }, + { _scriptRootConfigPath, scriptRootPath }, }); }); From f924ebc192c788dc4911aa2a0fe5eff72cfe6cc0 Mon Sep 17 00:00:00 2001 From: Shyju Krishnankutty Date: Wed, 11 Oct 2023 17:25:43 -0700 Subject: [PATCH 3/8] Cleanup --- .../Rpc/WebHostRpcWorkerChannelManager.cs | 6 ++--- .../WebHostEndToEnd/SpecializationE2ETests.cs | 26 +++++++++---------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/WebJobs.Script/Workers/Rpc/WebHostRpcWorkerChannelManager.cs b/src/WebJobs.Script/Workers/Rpc/WebHostRpcWorkerChannelManager.cs index c2dcd91b39..8e0ce86d50 100644 --- a/src/WebJobs.Script/Workers/Rpc/WebHostRpcWorkerChannelManager.cs +++ b/src/WebJobs.Script/Workers/Rpc/WebHostRpcWorkerChannelManager.cs @@ -123,14 +123,14 @@ public async Task SpecializeAsync() if (_workerRuntime != null && rpcWorkerChannel != null) { - bool envReloadRequestResult = false; + bool envReloadRequestResultSuccessful = false; if (UsePlaceholderChannel(rpcWorkerChannel)) { _logger.LogDebug("Loading environment variables for runtime: {runtime}", _workerRuntime); - envReloadRequestResult = await rpcWorkerChannel.SendFunctionEnvironmentReloadRequest(); + envReloadRequestResultSuccessful = await rpcWorkerChannel.SendFunctionEnvironmentReloadRequest(); } - if (envReloadRequestResult == false) + if (envReloadRequestResultSuccessful == false) { _logger.LogDebug("Shutting down placeholder worker. Worker is not compatible for runtime: {runtime}", _workerRuntime); // If we need to allow file edits, we should shutdown the webhost channel on specialization. diff --git a/test/WebJobs.Script.Tests.Integration/WebHostEndToEnd/SpecializationE2ETests.cs b/test/WebJobs.Script.Tests.Integration/WebHostEndToEnd/SpecializationE2ETests.cs index 044c4d1e84..79ca16577d 100644 --- a/test/WebJobs.Script.Tests.Integration/WebHostEndToEnd/SpecializationE2ETests.cs +++ b/test/WebJobs.Script.Tests.Integration/WebHostEndToEnd/SpecializationE2ETests.cs @@ -903,7 +903,7 @@ public async Task DotNetIsolated_PlaceholderMiss_Not64Bit() _environment.SetProcessBitness(is64Bitness: false); // We only specialize when host process is 64 bit process. - await DotNetIsolatedPlaceholderMiss(_dotnetIsolated60Path , () => + await DotNetIsolatedPlaceholderMiss(_dotnetIsolated60Path, () => { _environment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebsiteUsePlaceholderDotNetIsolated, "1"); _environment.SetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeVersionSettingName, "6.0"); @@ -1072,19 +1072,19 @@ private async Task DotNetIsolatedPlaceholderMiss(string scriptRootPath, Action a else { response.EnsureSuccessStatusCode(); - } - var expectedProcessName = scriptRootPath == _dotnetIsolated60Path ? "DotNetIsolated60" : "DotNetIsolatedUnsupported"; - // Placeholder miss; new channel should be started using the deployed worker directly - var specializedChannel = await webChannelManager.GetChannels("dotnet-isolated").Single().Value.Task; - Assert.Contains("dotnet.exe", specializedChannel.WorkerProcess.Process.StartInfo.FileName); - Assert.Contains(expectedProcessName, specializedChannel.WorkerProcess.Process.StartInfo.Arguments); - runningProcess = Process.GetProcessById(specializedChannel.WorkerProcess.Id); - Assert.Contains(runningProcess.ProcessName, "dotnet"); - - // Ensure other process is gone. - Assert.DoesNotContain(Process.GetProcesses(), p => p.ProcessName.Contains("FunctionsNetHost")); - Assert.Throws(() => placeholderChannel.WorkerProcess.Process.Id); + var expectedProcessName = scriptRootPath == _dotnetIsolated60Path ? "DotNetIsolated60" : "DotNetIsolatedUnsupported"; + // Placeholder miss; new channel should be started using the deployed worker directly + var specializedChannel = await webChannelManager.GetChannels("dotnet-isolated").Single().Value.Task; + Assert.Contains("dotnet.exe", specializedChannel.WorkerProcess.Process.StartInfo.FileName); + Assert.Contains(expectedProcessName, specializedChannel.WorkerProcess.Process.StartInfo.Arguments); + runningProcess = Process.GetProcessById(specializedChannel.WorkerProcess.Id); + Assert.Contains(runningProcess.ProcessName, "dotnet"); + + // Ensure other process is gone. + Assert.DoesNotContain(Process.GetProcesses(), p => p.ProcessName.Contains("FunctionsNetHost")); + Assert.Throws(() => placeholderChannel.WorkerProcess.Process.Id); + } } private static void BuildDotnetIsolated60() From 22ab9d1f953a3b88755f509b44f1f7a2b71bf92f Mon Sep 17 00:00:00 2001 From: Shyju Krishnankutty Date: Wed, 11 Oct 2023 17:37:37 -0700 Subject: [PATCH 4/8] Cleanup --- src/WebJobs.Script.Grpc/Channel/GrpcWorkerChannel.cs | 11 ++++++----- .../DotNetIsolatedUnsupportedWorker.csproj | 6 ++---- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/WebJobs.Script.Grpc/Channel/GrpcWorkerChannel.cs b/src/WebJobs.Script.Grpc/Channel/GrpcWorkerChannel.cs index 291aa7d9f9..4695e32860 100644 --- a/src/WebJobs.Script.Grpc/Channel/GrpcWorkerChannel.cs +++ b/src/WebJobs.Script.Grpc/Channel/GrpcWorkerChannel.cs @@ -376,7 +376,7 @@ internal void FunctionEnvironmentReloadResponse(FunctionEnvironmentReloadRespons if (res.Result.IsFailure(out Exception reloadEnvironmentVariablesException)) { - if (IsFatalFailure(res)) + if (IsEnvironmentReloadFailureFatalError(res)) { _workerChannelLogger.LogError(reloadEnvironmentVariablesException, "Failed to reload environment variables"); _reloadTask.SetException(reloadEnvironmentVariablesException); @@ -396,11 +396,11 @@ internal void FunctionEnvironmentReloadResponse(FunctionEnvironmentReloadRespons /// /// Determines whether the environment reload failure should be considered as a fatal error or should be ignored. /// - internal bool IsFatalFailure(FunctionEnvironmentReloadResponse response) + internal bool IsEnvironmentReloadFailureFatalError(FunctionEnvironmentReloadResponse response) { var workerRuntime = _environment.GetFunctionsWorkerRuntime(); - // Currently we support non fatal reload errors for dotnet isolated worker only. + // Currently we support non fatal environment reload errors for dotnet isolated worker only. if (workerRuntime != RpcWorkerConstants.DotNetIsolatedLanguageWorkerName) { return true; @@ -608,8 +608,9 @@ public Task SendFunctionEnvironmentReloadRequest() FunctionEnvironmentReloadRequest = request }); - Task t = _reloadTask.Task; - return t; + var environmentReloadRequestResult = _reloadTask.Task; + + return environmentReloadRequestResult; } public void SendWorkerWarmupRequest() diff --git a/test/DotNetIsolatedUnsupportedWorker/DotNetIsolatedUnsupportedWorker.csproj b/test/DotNetIsolatedUnsupportedWorker/DotNetIsolatedUnsupportedWorker.csproj index 8cb09f29e9..e9f73aeb48 100644 --- a/test/DotNetIsolatedUnsupportedWorker/DotNetIsolatedUnsupportedWorker.csproj +++ b/test/DotNetIsolatedUnsupportedWorker/DotNetIsolatedUnsupportedWorker.csproj @@ -6,15 +6,13 @@ enable enable True - True - True - + - + From 2d06452c82569bd3decdfb83d2a6c6d4e4c5fba7 Mon Sep 17 00:00:00 2001 From: Shyju Krishnankutty Date: Wed, 11 Oct 2023 19:07:17 -0700 Subject: [PATCH 5/8] Fixing Worker.Extensions.Http package version to align with Http.AspNetCore package. --- test/DotNetIsolated60/DotNetIsolated60.csproj | 4 ++-- .../DotNetIsolatedUnsupportedWorker.csproj | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/DotNetIsolated60/DotNetIsolated60.csproj b/test/DotNetIsolated60/DotNetIsolated60.csproj index a6d802816b..f6a818fb17 100644 --- a/test/DotNetIsolated60/DotNetIsolated60.csproj +++ b/test/DotNetIsolated60/DotNetIsolated60.csproj @@ -11,8 +11,8 @@ - - + + diff --git a/test/DotNetIsolatedUnsupportedWorker/DotNetIsolatedUnsupportedWorker.csproj b/test/DotNetIsolatedUnsupportedWorker/DotNetIsolatedUnsupportedWorker.csproj index e9f73aeb48..81c3461eaf 100644 --- a/test/DotNetIsolatedUnsupportedWorker/DotNetIsolatedUnsupportedWorker.csproj +++ b/test/DotNetIsolatedUnsupportedWorker/DotNetIsolatedUnsupportedWorker.csproj @@ -1,4 +1,4 @@ - + net6.0 v4 @@ -9,7 +9,7 @@ - + From 3c0c784cd42f38e1ce5478f76223d1ee2f09ffee Mon Sep 17 00:00:00 2001 From: Shyju Krishnankutty Date: Thu, 12 Oct 2023 10:27:24 -0700 Subject: [PATCH 6/8] Logging error as it is received from the worker. --- .../Channel/GrpcWorkerChannel.cs | 55 +++++-------------- .../StatusResultExtensions.cs | 13 +++-- 2 files changed, 22 insertions(+), 46 deletions(-) diff --git a/src/WebJobs.Script.Grpc/Channel/GrpcWorkerChannel.cs b/src/WebJobs.Script.Grpc/Channel/GrpcWorkerChannel.cs index 4695e32860..64afbf6522 100644 --- a/src/WebJobs.Script.Grpc/Channel/GrpcWorkerChannel.cs +++ b/src/WebJobs.Script.Grpc/Channel/GrpcWorkerChannel.cs @@ -374,17 +374,13 @@ internal void FunctionEnvironmentReloadResponse(FunctionEnvironmentReloadRespons ApplyCapabilities(res.Capabilities, res.CapabilitiesUpdateStrategy.ToGrpcCapabilitiesUpdateStrategy()); - if (res.Result.IsFailure(out Exception reloadEnvironmentVariablesException)) + if (res.Result.IsFailure(IsUserCodeExceptionCapabilityEnabled(), out var reloadEnvironmentVariablesException)) { - if (IsEnvironmentReloadFailureFatalError(res)) + if (res.Result.Exception is not null && reloadEnvironmentVariablesException is not null) { - _workerChannelLogger.LogError(reloadEnvironmentVariablesException, "Failed to reload environment variables"); - _reloadTask.SetException(reloadEnvironmentVariablesException); - } - else - { - _reloadTask.SetResult(false); + _workerChannelLogger.LogWarning(reloadEnvironmentVariablesException, reloadEnvironmentVariablesException.Message); } + _reloadTask.SetResult(false); } else { @@ -393,38 +389,6 @@ internal void FunctionEnvironmentReloadResponse(FunctionEnvironmentReloadRespons latencyEvent.Dispose(); } - /// - /// Determines whether the environment reload failure should be considered as a fatal error or should be ignored. - /// - internal bool IsEnvironmentReloadFailureFatalError(FunctionEnvironmentReloadResponse response) - { - var workerRuntime = _environment.GetFunctionsWorkerRuntime(); - - // Currently we support non fatal environment reload errors for dotnet isolated worker only. - if (workerRuntime != RpcWorkerConstants.DotNetIsolatedLanguageWorkerName) - { - return true; - } - - var rpcException = response.Result.Exception; - - // Currently the below 2 errors are considered as non fatal. - if (rpcException.Message.StartsWith("Microsoft.Azure.Functions.Worker.EnvironmentReloadNotSupportedException")) - { - _workerChannelLogger.LogDebug(new EventId(422, ScriptConstants.PlaceholderMissDueToBitnessEventName), - " This app is not using the latest version of Microsoft.Azure.Functions.Worker SDK and therefore does not leverage all performance optimizations. See https://aka.ms/azure-functions/dotnet/placeholders for more information."); - return false; - } - if (rpcException.Message.StartsWith("Microsoft.Azure.Functions.Worker.FunctionAppPayloadNotFoundException")) - { - // If no app payload present, we can ignore the error because - // it is possible that the app was created (instance assigned from platform), but no deployment has happened yet. - return false; - } - - return true; - } - internal void WorkerInitResponse(GrpcEvent initEvent) { _startLatencyMetric?.Dispose(); @@ -456,6 +420,15 @@ internal void WorkerInitResponse(GrpcEvent initEvent) _workerInitTask.TrySetResult(true); } + private bool IsUserCodeExceptionCapabilityEnabled() + { + var enableUserCodeExceptionCapability = string.Equals( + _workerCapabilities.GetCapabilityState(RpcWorkerConstants.EnableUserCodeException), bool.TrueString, + StringComparison.OrdinalIgnoreCase); + + return enableUserCodeExceptionCapability; + } + private void LogWorkerMetadata(WorkerMetadata workerMetadata) { if (workerMetadata == null) @@ -1628,4 +1601,4 @@ private void OnTimeout() } } } -} +} \ No newline at end of file diff --git a/src/WebJobs.Script.Grpc/MessageExtensions/StatusResultExtensions.cs b/src/WebJobs.Script.Grpc/MessageExtensions/StatusResultExtensions.cs index b555e6c192..077a0185e3 100644 --- a/src/WebJobs.Script.Grpc/MessageExtensions/StatusResultExtensions.cs +++ b/src/WebJobs.Script.Grpc/MessageExtensions/StatusResultExtensions.cs @@ -3,20 +3,18 @@ using System; using System.Threading.Tasks; -using Grpc.Core; -using Microsoft.Azure.WebJobs.Script.Config; using Microsoft.Azure.WebJobs.Script.Grpc.Messages; namespace Microsoft.Azure.WebJobs.Script.Grpc { internal static class StatusResultExtensions { - public static bool IsFailure(this StatusResult statusResult, out Exception exception) + public static bool IsFailure(this StatusResult statusResult, bool enableUserCodeExceptionCapability, out Exception exception) { switch (statusResult.Status) { case StatusResult.Types.Status.Failure: - exception = GetRpcException(statusResult); + exception = GetRpcException(statusResult, enableUserCodeExceptionCapability); return true; case StatusResult.Types.Status.Cancelled: @@ -29,6 +27,11 @@ public static bool IsFailure(this StatusResult statusResult, out Exception excep } } + public static bool IsFailure(this StatusResult statusResult, out Exception exception) + { + return IsFailure(statusResult, false, out exception); + } + /// /// This method is only hit on the invocation code path. /// enableUserCodeExceptionCapability = feature flag exposed as a capability that is set by the worker. @@ -68,4 +71,4 @@ public static Workers.Rpc.RpcException GetRpcException(StatusResult statusResult return new Workers.Rpc.RpcException(status, string.Empty, string.Empty); } } -} +} \ No newline at end of file From 5eb863e9c03605d6d230c1971da2b2607865957d Mon Sep 17 00:00:00 2001 From: Shyju Krishnankutty Date: Thu, 12 Oct 2023 10:29:54 -0700 Subject: [PATCH 7/8] DotNetIsolatedNativeHost package to 1.0.2 --- src/WebJobs.Script/WebJobs.Script.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/WebJobs.Script/WebJobs.Script.csproj b/src/WebJobs.Script/WebJobs.Script.csproj index e5df2f43a1..7c00814776 100644 --- a/src/WebJobs.Script/WebJobs.Script.csproj +++ b/src/WebJobs.Script/WebJobs.Script.csproj @@ -47,7 +47,7 @@ - + From 42e4df33fd85f582d3375000a0466fdc2bfadade Mon Sep 17 00:00:00 2001 From: Shyju Krishnankutty Date: Thu, 12 Oct 2023 10:44:41 -0700 Subject: [PATCH 8/8] Inlining a variable (Reverting to previous version) --- src/WebJobs.Script.Grpc/Channel/GrpcWorkerChannel.cs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/WebJobs.Script.Grpc/Channel/GrpcWorkerChannel.cs b/src/WebJobs.Script.Grpc/Channel/GrpcWorkerChannel.cs index 64afbf6522..2b1fb894c6 100644 --- a/src/WebJobs.Script.Grpc/Channel/GrpcWorkerChannel.cs +++ b/src/WebJobs.Script.Grpc/Channel/GrpcWorkerChannel.cs @@ -581,9 +581,7 @@ public Task SendFunctionEnvironmentReloadRequest() FunctionEnvironmentReloadRequest = request }); - var environmentReloadRequestResult = _reloadTask.Task; - - return environmentReloadRequestResult; + return _reloadTask.Task; } public void SendWorkerWarmupRequest()