Add github action logs to pipeline witness

.github/CODEOWNERS

@@ -39,7 +39,7 @@
 /tools/mock-service-host/              @raych1 @tadelesh
 /tools/perf-automation/                @mikeharder @benbp
 /tools/pipeline-generator/             @weshaggard @benbp
-/tools/pipeline-witness/               @praveenkuttappan @weshaggard
+/tools/pipeline-witness/               @hallipr
 /tools/sdk-ai-bots/                    @raych1
 /tools/sdk-testgen/                    @raych1 @tadelesh
 /tools/test-proxy/                     @scbedd @mikeharder @benbp

global.json

@@ -3,7 +3,7 @@
     "Microsoft.Build.Traversal": "3.2.0"
   },
   "sdk": {
-    "version": "7.0.102",
+    "version": "8.0.303",
     "rollForward": "feature"
   }
 }

tools/pipeline-witness/.gitignore

@@ -0,0 +1 @@
+appsettings.local.json

tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness.Tests/Azure.Sdk.Tools.PipelineWitness.Tests.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <IsPackable>false</IsPackable>
   </PropertyGroup>
 

tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Azure.Sdk.Tools.PipelineWitness.csproj

@@ -1,11 +1,12 @@
 <Project Sdk="Microsoft.NET.Sdk.Web">
 
   <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <UserSecretsId>bc5587e8-3503-4e1a-816c-1e219e4047f6</UserSecretsId>
   </PropertyGroup>
 
   <ItemGroup>
+    <PackageReference Include="Azure.Security.KeyVault.Secrets" Version="4.6.0" />
     <PackageReference Include="Azure.Storage.Blobs" Version="12.13.0" />
     <PackageReference Include="Azure.Storage.Queues" Version="12.11.0" />
     <PackageReference Include="Microsoft.ApplicationInsights.AspNetCore" Version="2.21.0" />
@@ -15,6 +16,7 @@
     <PackageReference Include="Microsoft.TeamFoundationServer.Client" Version="19.225.1" />
     <PackageReference Include="Microsoft.VisualStudio.Services.InteractiveClient" Version="19.225.1" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
+    <PackageReference Include="Octokit" Version="12.0.0" />
     <PackageReference Include="Swashbuckle.AspNetCore" Version="6.4.0" />
   </ItemGroup>
 

tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Configuration/ISecretClientProvider.cs

@@ -0,0 +1,9 @@
+using System;
+using Azure.Security.KeyVault.Secrets;
+
+namespace Azure.Sdk.Tools.PipelineWitness.Configuration;
+
+public interface ISecretClientProvider
+{
+    SecretClient GetSecretClient(Uri vaultUri);
+}

tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Configuration/PipelineWitnessSettings.cs

@@ -29,6 +29,37 @@ public class PipelineWitnessSettings
         /// </summary>
         public string BuildCompleteQueueName { get; set; }
 
+        /// <summary>
+        /// Gets or sets the number of concurrent build complete queue workers to register
+        /// </summary>
+        public int BuildCompleteWorkerCount { get; set; } = 1;
+
+        /// <summary>
+        /// Gets or sets whether the build definition worker is enabled
+        /// </summary>
+        public bool BuildDefinitionWorkerEnabled { get; set; } = true;
+
+        /// <summary>
+        /// Gets or sets the name of the GitHub actions queue
+        /// </summary>
+        public string GitHubActionRunsQueueName { get; set; }
+
+        /// <summary>
+        /// Gets or sets the name of the GitHub action queue workers to register
+        /// </summary>
+        public int GitHubActionRunsWorkerCount { get; set; } = 1;
+
+        /// <summary>
+        /// Gets or sets secret used to verify GitHub webhook payloads
+        /// </summary>
+        public string GitHubWebhookSecret { get; set; }
+
+        /// <summary>
+        /// Gets or sets the access token to use for GitHub API requests. This
+        /// must be a personal access token with `repo` scope.
+        /// </summary>
+        public string GitHubAccessToken { get; set; }
+
         /// <summary>
         /// Gets or sets the amount of time a message should be invisible in the queue while being processed
         /// </summary>
@@ -64,11 +95,6 @@ public class PipelineWitnessSettings
         /// </summary>
         public TimeSpan BuildDefinitionLoopPeriod { get; set; } = TimeSpan.FromMinutes(5);
 
-        /// <summary>
-        /// Gets or sets the number of concurrent build complete queue workers to register
-        /// </summary>
-        public int BuildCompleteWorkerCount { get; set; } = 1;
-
         /// <summary>
         /// Gets or sets the artifact name used by the pipeline owners extraction build
         /// </summary>

tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Configuration/PostConfigureKeyVaultSettings.cs

@@ -0,0 +1,58 @@
+using System;
+using System.Linq;
+using System.Text.RegularExpressions;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+
+namespace Azure.Sdk.Tools.PipelineWitness.Configuration;
+
+public class PostConfigureKeyVaultSettings<T> : IPostConfigureOptions<T> where T : class
+{
+    private static readonly Regex secretRegex = new Regex(@"(?<vault>https://.*?\.vault\.azure\.net)/secrets/(?<secret>.*)", RegexOptions.Compiled, TimeSpan.FromSeconds(5));
+    private readonly ILogger logger;
+    private readonly ISecretClientProvider secretClientProvider;
+
+    public PostConfigureKeyVaultSettings(ILogger<PostConfigureKeyVaultSettings<T>> logger, ISecretClientProvider secretClientProvider)
+    {
+        this.logger = logger;
+        this.secretClientProvider = secretClientProvider;
+    }
+
+    public void PostConfigure(string name, T options)
+    {
+        var stringProperties = typeof(T)
+            .GetProperties()
+            .Where(x => x.PropertyType == typeof(string));
+
+        foreach (var property in stringProperties)
+        {
+            var value = (string)property.GetValue(options);
+
+            if (value != null)
+            {
+                var match = secretRegex.Match(value);
+
+                if (match.Success)
+                {
+                    var vaultUrl = match.Groups["vault"].Value;
+                    var secretName = match.Groups["secret"].Value;
+
+                    try
+                    {
+                        var secretClient = this.secretClientProvider.GetSecretClient(new Uri(vaultUrl));
+                        this.logger.LogInformation("Replacing setting property {PropertyName} with value from secret {SecretUrl}", property.Name, value);
+
+                        var response = secretClient.GetSecret(secretName);
+                        var secret = response.Value;
+
+                        property.SetValue(options, secret.Value);
+                    }
+                    catch (Exception exception)
+                    {
+                        this.logger.LogError(exception, "Unable to read secret {SecretName} from vault {VaultUrl}", secretName, vaultUrl);
+                    }
+                }
+            }
+        }
+    }
+}

tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Configuration/SecretClientProvider.cs

@@ -0,0 +1,21 @@
+using System;
+using Azure.Core;
+using Azure.Security.KeyVault.Secrets;
+
+namespace Azure.Sdk.Tools.PipelineWitness.Configuration
+{
+    public class SecretClientProvider : ISecretClientProvider
+    {
+        private readonly TokenCredential tokenCredential;
+
+        public SecretClientProvider(TokenCredential tokenCredential)
+        {
+            this.tokenCredential = tokenCredential;
+        }
+
+        public SecretClient GetSecretClient(Uri vaultUri)
+        {
+            return new SecretClient(vaultUri, this.tokenCredential);
+        }
+    }
+}

tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Controllers/GitHubEventsController.cs

@@ -0,0 +1,101 @@
+using System;
+using System.IO;
+using System.Linq;
+using System.Security.Cryptography;
+using System.Text;
+using System.Text.Json;
+using System.Threading.Tasks;
+using Azure.Sdk.Tools.PipelineWitness.Configuration;
+using Azure.Sdk.Tools.PipelineWitness.GitHubActions;
+using Azure.Storage.Queues;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+
+// For more information on enabling Web API for empty projects, visit https://go.microsoft.com/fwlink/?LinkID=397860
+
+namespace Azure.Sdk.Tools.PipelineWitness.Controllers
+{
+    [Route("api/githubevents")]
+    [ApiController]
+    public class GitHubEventsController : ControllerBase
+    {
+        private readonly QueueClient queueClient;
+        private readonly ILogger<GitHubEventsController> logger;
+        private readonly PipelineWitnessSettings settings;
+
+        public GitHubEventsController(ILogger<GitHubEventsController> logger, QueueServiceClient queueServiceClient, IOptions<PipelineWitnessSettings> options)
+        {
+            this.logger = logger;
+            this.settings = options.Value;
+            this.queueClient = queueServiceClient.GetQueueClient(this.settings.GitHubActionRunsQueueName);
+        }
+
+        // POST api/githubevents
+        [HttpPost]
+        public async Task<IActionResult> PostAsync()
+        {
+            var eventName = Request.Headers["X-GitHub-Event"].FirstOrDefault();
+            switch (eventName)
+            {
+                case "ping":
+                    return Ok();
+                case "workflow_run":
+                    return await ProcessWorkflowRunEventAsync();
+                default:
+                    this.logger.LogWarning("Received GitHub event {EventName} which is not supported", eventName);
+                    return BadRequest();
+            }
+        }
+
+        private static bool VerifySignature(string text, string key, string signature)
+        {
+            Encoding encoding = Encoding.UTF8;
+
+            byte[] textBytes = encoding.GetBytes(text);
+            byte[] keyBytes = encoding.GetBytes(key);
+
+            using HMACSHA256 hasher = new(keyBytes);
+            byte[] hashBytes = hasher.ComputeHash(textBytes);
+
+            var hash = BitConverter.ToString(hashBytes).Replace("-", "").ToLower();
+            var expectedSignature = $"sha256={hash}";
+            return signature == expectedSignature;
+        }
+
+        private async Task<IActionResult> ProcessWorkflowRunEventAsync()
+        {
+            using var reader = new StreamReader(Request.Body);
+            var body = await reader.ReadToEndAsync();
+            var signature = Request.Headers["X-Hub-Signature-256"].FirstOrDefault();
+
+            if (!VerifySignature(body, this.settings.GitHubWebhookSecret, signature))
+            {
+                this.logger.LogWarning("Received GitHub event {Event} with invalid signature", "workflow_run");
+                return Unauthorized();
+            }
+
+            var eventMessage = JsonDocument.Parse(body).RootElement;
+
+            string action = eventMessage.GetProperty("action").GetString();
+
+            this.logger.LogInformation("Received GitHub event {Event}.{Action}", "workflow_run", action);
+
+            if (action == "completed")
+            {
+                var queueMessage = new GitHubRunCompleteMessage
+                {
+                    Owner = eventMessage.GetProperty("repository").GetProperty("owner").GetProperty("login").GetString(),
+                    Repository = eventMessage.GetProperty("repository").GetProperty("name").GetString(),
+                    RunId = eventMessage.GetProperty("workflow_run").GetProperty("id").GetInt64(),
+                };
+
+                this.logger.LogInformation("Enqueuing GitHubRunCompleteMessage for {Owner}/{Repository} run {RunId}", queueMessage.Owner, queueMessage.Repository, queueMessage.RunId);
+
+                await this.queueClient.SendMessageAsync(JsonSerializer.Serialize(queueMessage));
+            }
+
+            return Ok();
+        }
+    }
+}