diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 78053babd13..f9b7f0fa344 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -39,7 +39,7 @@ /tools/mock-service-host/ @raych1 @tadelesh /tools/perf-automation/ @mikeharder @benbp /tools/pipeline-generator/ @weshaggard @benbp -/tools/pipeline-witness/ @praveenkuttappan @weshaggard +/tools/pipeline-witness/ @hallipr /tools/sdk-ai-bots/ @raych1 /tools/sdk-testgen/ @raych1 @tadelesh /tools/test-proxy/ @scbedd @mikeharder @benbp diff --git a/global.json b/global.json index 1bfb43897e4..184886fa93c 100644 --- a/global.json +++ b/global.json @@ -3,7 +3,7 @@ "Microsoft.Build.Traversal": "3.2.0" }, "sdk": { - "version": "7.0.102", + "version": "8.0.303", "rollForward": "feature" } } \ No newline at end of file diff --git a/tools/pipeline-witness/.gitignore b/tools/pipeline-witness/.gitignore new file mode 100644 index 00000000000..63666fbfc80 --- /dev/null +++ b/tools/pipeline-witness/.gitignore @@ -0,0 +1 @@ +appsettings.local.json \ No newline at end of file diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness.Tests/Azure.Sdk.Tools.PipelineWitness.Tests.csproj b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness.Tests/Azure.Sdk.Tools.PipelineWitness.Tests.csproj index f3f15b8e8c6..9ee9273c59f 100644 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness.Tests/Azure.Sdk.Tools.PipelineWitness.Tests.csproj +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness.Tests/Azure.Sdk.Tools.PipelineWitness.Tests.csproj @@ -1,7 +1,7 @@ - net6.0 + net8.0 false diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Azure.Sdk.Tools.PipelineWitness.csproj b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Azure.Sdk.Tools.PipelineWitness.csproj index 46d852a099e..53f41736ddd 100644 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Azure.Sdk.Tools.PipelineWitness.csproj +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Azure.Sdk.Tools.PipelineWitness.csproj @@ -1,11 +1,12 @@  - net6.0 + net8.0 bc5587e8-3503-4e1a-816c-1e219e4047f6 + @@ -15,6 +16,7 @@ + diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Configuration/ISecretClientProvider.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Configuration/ISecretClientProvider.cs new file mode 100644 index 00000000000..cb04a2967c4 --- /dev/null +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Configuration/ISecretClientProvider.cs @@ -0,0 +1,9 @@ +using System; +using Azure.Security.KeyVault.Secrets; + +namespace Azure.Sdk.Tools.PipelineWitness.Configuration; + +public interface ISecretClientProvider +{ + SecretClient GetSecretClient(Uri vaultUri); +} diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Configuration/PipelineWitnessSettings.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Configuration/PipelineWitnessSettings.cs index 3f77e6faa4e..479fac44f1d 100644 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Configuration/PipelineWitnessSettings.cs +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Configuration/PipelineWitnessSettings.cs @@ -29,6 +29,37 @@ public class PipelineWitnessSettings /// public string BuildCompleteQueueName { get; set; } + /// + /// Gets or sets the number of concurrent build complete queue workers to register + /// + public int BuildCompleteWorkerCount { get; set; } = 1; + + /// + /// Gets or sets whether the build definition worker is enabled + /// + public bool BuildDefinitionWorkerEnabled { get; set; } = true; + + /// + /// Gets or sets the name of the GitHub actions queue + /// + public string GitHubActionRunsQueueName { get; set; } + + /// + /// Gets or sets the name of the GitHub action queue workers to register + /// + public int GitHubActionRunsWorkerCount { get; set; } = 1; + + /// + /// Gets or sets secret used to verify GitHub webhook payloads + /// + public string GitHubWebhookSecret { get; set; } + + /// + /// Gets or sets the access token to use for GitHub API requests. This + /// must be a personal access token with `repo` scope. + /// + public string GitHubAccessToken { get; set; } + /// /// Gets or sets the amount of time a message should be invisible in the queue while being processed /// @@ -64,11 +95,6 @@ public class PipelineWitnessSettings /// public TimeSpan BuildDefinitionLoopPeriod { get; set; } = TimeSpan.FromMinutes(5); - /// - /// Gets or sets the number of concurrent build complete queue workers to register - /// - public int BuildCompleteWorkerCount { get; set; } = 1; - /// /// Gets or sets the artifact name used by the pipeline owners extraction build /// diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Configuration/PostConfigureKeyVaultSettings.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Configuration/PostConfigureKeyVaultSettings.cs new file mode 100644 index 00000000000..80e1fa488f5 --- /dev/null +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Configuration/PostConfigureKeyVaultSettings.cs @@ -0,0 +1,58 @@ +using System; +using System.Linq; +using System.Text.RegularExpressions; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace Azure.Sdk.Tools.PipelineWitness.Configuration; + +public class PostConfigureKeyVaultSettings : IPostConfigureOptions where T : class +{ + private static readonly Regex secretRegex = new Regex(@"(?https://.*?\.vault\.azure\.net)/secrets/(?.*)", RegexOptions.Compiled, TimeSpan.FromSeconds(5)); + private readonly ILogger logger; + private readonly ISecretClientProvider secretClientProvider; + + public PostConfigureKeyVaultSettings(ILogger> logger, ISecretClientProvider secretClientProvider) + { + this.logger = logger; + this.secretClientProvider = secretClientProvider; + } + + public void PostConfigure(string name, T options) + { + var stringProperties = typeof(T) + .GetProperties() + .Where(x => x.PropertyType == typeof(string)); + + foreach (var property in stringProperties) + { + var value = (string)property.GetValue(options); + + if (value != null) + { + var match = secretRegex.Match(value); + + if (match.Success) + { + var vaultUrl = match.Groups["vault"].Value; + var secretName = match.Groups["secret"].Value; + + try + { + var secretClient = this.secretClientProvider.GetSecretClient(new Uri(vaultUrl)); + this.logger.LogInformation("Replacing setting property {PropertyName} with value from secret {SecretUrl}", property.Name, value); + + var response = secretClient.GetSecret(secretName); + var secret = response.Value; + + property.SetValue(options, secret.Value); + } + catch (Exception exception) + { + this.logger.LogError(exception, "Unable to read secret {SecretName} from vault {VaultUrl}", secretName, vaultUrl); + } + } + } + } + } +} diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Configuration/SecretClientProvider.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Configuration/SecretClientProvider.cs new file mode 100644 index 00000000000..a00966dc3fe --- /dev/null +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Configuration/SecretClientProvider.cs @@ -0,0 +1,21 @@ +using System; +using Azure.Core; +using Azure.Security.KeyVault.Secrets; + +namespace Azure.Sdk.Tools.PipelineWitness.Configuration +{ + public class SecretClientProvider : ISecretClientProvider + { + private readonly TokenCredential tokenCredential; + + public SecretClientProvider(TokenCredential tokenCredential) + { + this.tokenCredential = tokenCredential; + } + + public SecretClient GetSecretClient(Uri vaultUri) + { + return new SecretClient(vaultUri, this.tokenCredential); + } + } +} diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Controllers/GitHubEventsController.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Controllers/GitHubEventsController.cs new file mode 100644 index 00000000000..b39e4482e6e --- /dev/null +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Controllers/GitHubEventsController.cs @@ -0,0 +1,101 @@ +using System; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using System.Threading.Tasks; +using Azure.Sdk.Tools.PipelineWitness.Configuration; +using Azure.Sdk.Tools.PipelineWitness.GitHubActions; +using Azure.Storage.Queues; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +// For more information on enabling Web API for empty projects, visit https://go.microsoft.com/fwlink/?LinkID=397860 + +namespace Azure.Sdk.Tools.PipelineWitness.Controllers +{ + [Route("api/githubevents")] + [ApiController] + public class GitHubEventsController : ControllerBase + { + private readonly QueueClient queueClient; + private readonly ILogger logger; + private readonly PipelineWitnessSettings settings; + + public GitHubEventsController(ILogger logger, QueueServiceClient queueServiceClient, IOptions options) + { + this.logger = logger; + this.settings = options.Value; + this.queueClient = queueServiceClient.GetQueueClient(this.settings.GitHubActionRunsQueueName); + } + + // POST api/githubevents + [HttpPost] + public async Task PostAsync() + { + var eventName = Request.Headers["X-GitHub-Event"].FirstOrDefault(); + switch (eventName) + { + case "ping": + return Ok(); + case "workflow_run": + return await ProcessWorkflowRunEventAsync(); + default: + this.logger.LogWarning("Received GitHub event {EventName} which is not supported", eventName); + return BadRequest(); + } + } + + private static bool VerifySignature(string text, string key, string signature) + { + Encoding encoding = Encoding.UTF8; + + byte[] textBytes = encoding.GetBytes(text); + byte[] keyBytes = encoding.GetBytes(key); + + using HMACSHA256 hasher = new(keyBytes); + byte[] hashBytes = hasher.ComputeHash(textBytes); + + var hash = BitConverter.ToString(hashBytes).Replace("-", "").ToLower(); + var expectedSignature = $"sha256={hash}"; + return signature == expectedSignature; + } + + private async Task ProcessWorkflowRunEventAsync() + { + using var reader = new StreamReader(Request.Body); + var body = await reader.ReadToEndAsync(); + var signature = Request.Headers["X-Hub-Signature-256"].FirstOrDefault(); + + if (!VerifySignature(body, this.settings.GitHubWebhookSecret, signature)) + { + this.logger.LogWarning("Received GitHub event {Event} with invalid signature", "workflow_run"); + return Unauthorized(); + } + + var eventMessage = JsonDocument.Parse(body).RootElement; + + string action = eventMessage.GetProperty("action").GetString(); + + this.logger.LogInformation("Received GitHub event {Event}.{Action}", "workflow_run", action); + + if (action == "completed") + { + var queueMessage = new GitHubRunCompleteMessage + { + Owner = eventMessage.GetProperty("repository").GetProperty("owner").GetProperty("login").GetString(), + Repository = eventMessage.GetProperty("repository").GetProperty("name").GetString(), + RunId = eventMessage.GetProperty("workflow_run").GetProperty("id").GetInt64(), + }; + + this.logger.LogInformation("Enqueuing GitHubRunCompleteMessage for {Owner}/{Repository} run {RunId}", queueMessage.Owner, queueMessage.Repository, queueMessage.RunId); + + await this.queueClient.SendMessageAsync(JsonSerializer.Serialize(queueMessage)); + } + + return Ok(); + } + } +} diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/GitHubActions/GitHubActionProcessor.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/GitHubActions/GitHubActionProcessor.cs new file mode 100644 index 00000000000..7c4a6ac989d --- /dev/null +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/GitHubActions/GitHubActionProcessor.cs @@ -0,0 +1,495 @@ +using System.Collections.Generic; +using System.IO.Compression; +using System.IO; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using System; +using System.Linq; +using System.Net; +using Octokit; +using Azure.Storage.Blobs; +using Microsoft.Extensions.Logging; +using Azure.Storage.Blobs.Models; +using Newtonsoft.Json; +using Newtonsoft.Json.Converters; +using Newtonsoft.Json.Serialization; +using System.Text; +using Microsoft.Azure.Pipelines.WebApi; + +namespace Azure.Sdk.Tools.PipelineWitness.GitHubActions +{ + public class GitHubActionProcessor + { + private const string RunsContainerName = "githubactionsruns"; + private const string JobsContainerName = "githubactionsjobs"; + private const string StepsContainerName = "githubactionssteps"; + private const string LogsContainerName = "githubactionslogs"; + private const string TimeFormat = @"yyyy-MM-dd\THH:mm:ss.fffffff\Z"; + + private static readonly ProductHeaderValue productHeaderValue1 = new("PipelineWitness", "1.0"); + private static readonly JsonSerializerSettings jsonSettings = new() + { + ContractResolver = new CamelCasePropertyNamesContractResolver(), + Converters = { new StringEnumConverter(new CamelCaseNamingStrategy()) }, + Formatting = Formatting.None, + }; + + private readonly ILogger logger; + private readonly GitHubClient client; + private readonly BlobContainerClient runsContainerClient; + private readonly BlobContainerClient jobsContainerClient; + private readonly BlobContainerClient stepsContainerClient; + private readonly BlobContainerClient logsContainerClient; + + public GitHubActionProcessor(ILogger logger, BlobServiceClient blobServiceClient, ICredentialStore credentials) + { + this.logger = logger; + this.logsContainerClient = blobServiceClient.GetBlobContainerClient(LogsContainerName); + this.runsContainerClient = blobServiceClient.GetBlobContainerClient(RunsContainerName); + this.jobsContainerClient = blobServiceClient.GetBlobContainerClient(JobsContainerName); + this.stepsContainerClient = blobServiceClient.GetBlobContainerClient(StepsContainerName); + this.client = new GitHubClient(productHeaderValue1, credentials); + } + + public async Task ProcessAsync(string owner, string repository, long runId) + { + WorkflowRun run = await GetWorkflowRunAsync(owner, repository, runId); + await ProcessWorkflowRunAsync(run); + + for (long attempt = 1; attempt < run.RunAttempt; attempt++) + { + WorkflowRun runAttempt = await this.client.Actions.Workflows.Runs.GetAttempt(owner, repository, runId, attempt); + await ProcessWorkflowRunAsync(runAttempt); + } + } + + private async Task ProcessWorkflowRunAsync(WorkflowRun run) + { + List jobs = await GetJobsAsync(run); + + await UploadRunBlobAsync(run); + await UploadJobsBlobAsync(run, jobs); + await UploadStepsBlobAsync(run, jobs); + await UploadLogsBlobAsync(run, jobs); + } + + private async Task UploadRunBlobAsync(WorkflowRun run) + { + string repository = run.Repository.FullName; + long runId = run.Id; + string runName = run.Name; + long attempt = run.RunAttempt; + + try + { + // even though runid/attempt is unique, we still add a date component to the path for easier browsing + // multiple attempts have the same runStartedAt, so the different attempt blobs will be in the same folder + string blobPath = $"{repository}/{run.RunStartedAt:yyyy/MM/dd}/{runId}-{attempt}.jsonl"; + BlobClient blobClient = this.runsContainerClient.GetBlobClient(blobPath); + + if (await blobClient.ExistsAsync()) + { + this.logger.LogInformation("Skipping existing workflow jobs for repository {Repository}, workflow {Workflow}, run {RunId}, attempt {Attempt}", repository, runName, runId, attempt); + return; + } + + this.logger.LogInformation("Processing workflow jobs for repository {Repository}, workflow {Workflow}, run {RunId}, attempt {Attempt}", repository, runName, runId, attempt); + + string content = JsonConvert.SerializeObject( + new + { + Repository = repository, + Workflow = runName, + run.WorkflowId, + RunId = run.Id, + run.RunNumber, + run.HeadBranch, + run.HeadSha, + run.RunAttempt, + run.Event, + Status = run.Status.StringValue, + Conclusion = run.Conclusion?.StringValue, + run.CheckSuiteId, + run.DisplayTitle, + run.Path, + RunStartedAt = run.RunStartedAt.ToString(TimeFormat), + CreatedAt = run.CreatedAt.ToString(TimeFormat), + UpdatedAt = run.UpdatedAt.ToString(TimeFormat), + run.NodeId, + run.CheckSuiteNodeId, + HeadRepository = run.HeadRepository?.FullName, + run.Url, + run.HtmlUrl, + EtlIngestDate = DateTime.UtcNow.ToString(TimeFormat), + }, jsonSettings); + + await blobClient.UploadAsync(new BinaryData(content)); + } + catch (RequestFailedException ex) when (ex.Status == (int)HttpStatusCode.Conflict) + { + this.logger.LogInformation("Ignoring existing blob exception for repository {Repository}, workflow {Workflow}, run {RunId}, attempt {Attempt}", repository, runName, runId, attempt); + } + catch (Exception ex) + { + this.logger.LogError(ex, "Error processing repository {Repository}, workflow {Workflow}, run {RunId}, attempt {Attempt}", repository, runName, runId, attempt); + throw; + } + } + + private async Task UploadJobsBlobAsync(WorkflowRun run, List jobs) + { + string repository = run.Repository.FullName; + long runId = run.Id; + string runName = run.Name; + long attempt = run.RunAttempt; + + try + { + string blobPath = $"{repository}/{run.RunStartedAt:yyyy/MM/dd}/{runId}-{attempt}.jsonl"; + BlobClient blobClient = this.jobsContainerClient.GetBlobClient(blobPath); + + if (await blobClient.ExistsAsync()) + { + this.logger.LogInformation("Skipping existing workflow jobs for repository {Repository}, workflow {Workflow}, run {RunId}, attempt {Attempt}", repository, runName, runId, attempt); + return; + } + + this.logger.LogInformation("Processing workflow jobs for repository {Repository}, workflow {Workflow}, run {RunId}, attempt {Attempt}", repository, runName, runId, attempt); + + StringBuilder builder = new(); + + foreach (var job in jobs) + { + builder.AppendLine(JsonConvert.SerializeObject( + new + { + Repository = repository, + Workflow = runName, + run.WorkflowId, + RunId = run.Id, + JobId = job.Id, + job.Name, + Status = job.Status.StringValue, + Conclusion = job.Conclusion?.StringValue, + CreatedAt = job.CreatedAt?.ToString(TimeFormat), + StartedAt = job.StartedAt.ToString(TimeFormat), + CompletedAt = job.CompletedAt?.ToString(TimeFormat), + job.NodeId, + job.HeadSha, + job.Labels, + job.RunnerId, + job.RunnerName, + job.RunnerGroupId, + job.RunnerGroupName, + job.HtmlUrl, + EtlIngestDate = DateTime.UtcNow.ToString(TimeFormat), + }, jsonSettings)); + } + + await blobClient.UploadAsync(new BinaryData(builder.ToString())); + } + catch (RequestFailedException ex) when (ex.Status == (int)HttpStatusCode.Conflict) + { + this.logger.LogInformation("Ignoring existing blob exception for repository {Repository}, workflow {Workflow}, run {RunId}, attempt {Attempt}", repository, runName, runId, attempt); + } + catch (Exception ex) + { + this.logger.LogError(ex, "Error processing repository {Repository}, workflow {Workflow}, run {RunId}, attempt {Attempt}", repository, runName, runId, attempt); + throw; + } + } + + private async Task UploadStepsBlobAsync(WorkflowRun run, List jobs) + { + string repository = run.Repository.FullName; + long runId = run.Id; + string runName = run.Name; + long attempt = run.RunAttempt; + + try + { + // logs with a given runId/attempt are immutable and retries add new attempts. + // even though runid/attempt is unique, we still add a date component to the path for easier browsing + // multiple attempts have the same runStartedAt, so the different attempt blobs will be in the same folder + string blobPath = $"{repository}/{run.RunStartedAt:yyyy/MM/dd}/{runId}-{attempt}.jsonl"; + BlobClient blobClient = this.stepsContainerClient.GetBlobClient(blobPath); + + if (await blobClient.ExistsAsync()) + { + this.logger.LogInformation("Skipping existing workflow steps for repository {Repository}, workflow {Workflow}, run {RunId}, attempt {Attempt}", repository, runName, runId, attempt); + return; + } + + this.logger.LogInformation("Processing workflow steps for repository {Repository}, workflow {Workflow}, run {RunId}, attempt {Attempt}", repository, runName, runId, attempt); + + StringBuilder builder = new(); + + foreach (var job in jobs) + { + foreach (var step in job.Steps) + { + builder.AppendLine(JsonConvert.SerializeObject( + new + { + Repository = repository, + Workflow = runName, + Job = job.Name, + run.WorkflowId, + RunId = run.Id, + JobId = job.Id, + StepNumber = step.Number, + step.Name, + Status = step.Status.StringValue, + Conclusion = step.Conclusion?.StringValue, + StartedAt = step.StartedAt?.ToString(TimeFormat), + CompletedAt = step.CompletedAt?.ToString(TimeFormat), + EtlIngestDate = DateTime.UtcNow.ToString(TimeFormat), + }, jsonSettings)); + } + } + + await blobClient.UploadAsync(new BinaryData(builder.ToString())); + } + catch (RequestFailedException ex) when (ex.Status == (int)HttpStatusCode.Conflict) + { + this.logger.LogInformation("Ignoring existing blob exception for repository {Repository}, workflow {Workflow}, run {RunId}, attempt {Attempt}", repository, runName, runId, attempt); + } + catch (Exception ex) + { + this.logger.LogError(ex, "Error processing repository {Repository}, workflow {Workflow}, run {RunId}, attempt {Attempt}", repository, runName, runId, attempt); + throw; + } + } + + private async Task UploadLogsBlobAsync(WorkflowRun run, List jobs) + { + string repository = run.Repository.FullName; + long runId = run.Id; + string runName = run.Name; + long attempt = run.RunAttempt; + + try + { + // logs with a given runId/attempt are immutable and retries add new attempts. + // even though runid/attempt is unique, we still add a date component to the path for easier browsing + // multiple attempts have the same runStartedAt, so the different attempt blobs will be in the same folder + string blobPath = $"{repository}/{run.RunStartedAt:yyyy/MM/dd}/{runId}-{attempt}.jsonl"; + BlobClient blobClient = this.logsContainerClient.GetBlobClient(blobPath); + + if (await blobClient.ExistsAsync()) + { + this.logger.LogInformation("Skipping existing log for repository {Repository}, workflow {Workflow}, run {RunId}, attempt {Attempt}", repository, runName, runId, attempt); + return; + } + + this.logger.LogInformation("Processing log for repository {Repository}, workflow {Workflow}, run {RunId}, attempt {Attempt}", repository, runName, runId, attempt); + + using ZipArchive archive = await GetLogsAsync(run); + + var logEntries = archive.Entries + .Select(x => new + { + Entry = x, + NameRegex = Regex.Match(x.FullName, @"^(?:(?.*)\/)?(?\d+)_(?[^\/]+)\.txt$"), + }) + .Where(x => x.NameRegex.Success) + .Select(x => new + { + ParentName = x.NameRegex.Groups["folder"].Value, + Index = x.NameRegex.Groups["index"].Value, + RecordName = x.NameRegex.Groups["name"].Value, + x.Entry + }) + .ToDictionary(x => string.IsNullOrEmpty(x.ParentName) ? x.RecordName : $"{x.ParentName}/{x.Index}", x => x.Entry); + + await using Stream blobStream = await blobClient.OpenWriteAsync(overwrite: true, new BlobOpenWriteOptions()); + await using StreamWriter blobWriter = new(blobStream); + + long characterCount = 0; + int lineCount = 0; + + foreach (var job in jobs) + { + // Retries may not run all jobs and skipped jobs will not have logs + // The jobs still appear in the API response, but their runnerName is empty + bool isRetrySkipped = string.IsNullOrEmpty(job.RunnerName) && attempt > 1; + + if (!logEntries.TryGetValue(job.Name, out ZipArchiveEntry jobEntry)) + { + if (!isRetrySkipped) + { + // All jobs in the first attempt or with runner names should have logs + this.logger.LogWarning("Missing log entry for job {JobName}", job.Name); + } + + continue; + } + + IList logLines = ReadLogLines(jobEntry, step: 0); + + IList stepLines = job.Steps + .Where(x => x.Conclusion != WorkflowJobConclusion.Skipped) + .OrderBy(x => x.Number) + .SelectMany(step => ReadLogLines(logEntries[$"{job.Name}/{step.Number}"], step.Number)) + .ToArray(); + + UpdateStepLines(logLines, stepLines); + + + foreach (LogLine logLine in logLines) + { + characterCount += logLine.Message.Length; + lineCount += 1; + + await blobWriter.WriteLineAsync(JsonConvert.SerializeObject(new + { + Repository = repository, + WorkflowName = runName, + run.WorkflowId, + RunId = run.Id, + JobId = job.Id, + StepNumber = logLine.Step, + LineNumber = logLine.Number, + Length = logLine.Message.Length, + Timestamp = logLine.Timestamp.ToString(TimeFormat), + Message = logLine.Message, + EtlIngestDate = DateTime.UtcNow.ToString(TimeFormat), + }, jsonSettings)); + } + } + + this.logger.LogInformation("Processed {CharacterCount} characters and {LineCount} lines for repository {Repository}, workflow {Workflow}, run {RunId}, attempt {Attempt}", characterCount, lineCount, repository, runName, runId, attempt); + } + catch (RequestFailedException ex) when (ex.Status == (int)HttpStatusCode.Conflict) + { + this.logger.LogInformation("Ignoring existing blob exception for repository {Repository}, workflow {Workflow}, run {RunId}, attempt {Attempt}", repository, runName, runId, attempt); + } + catch (Exception ex) + { + this.logger.LogError(ex, "Error processing repository {Repository}, workflow {Workflow}, run {RunId}, attempt {Attempt}", repository, runName, runId, attempt); + throw; + } + } + + private bool UpdateStepLines(IList jobLines, IList stepLines) + { + // For each line in the step, remove the corresponding line from the job + if (stepLines.Count == 0) + { + return true; + } + + // seek to the first line in the job that is after the first line in the step + for (int jobIndex = 0; jobIndex < jobLines.Count - stepLines.Count + 1; jobIndex++) + { + var isMatch = true; + + for (var stepIndex = 0; isMatch && stepIndex < stepLines.Count; stepIndex++) + { + var stepLine = stepLines[stepIndex]; + var jobLine = jobLines[jobIndex + stepIndex]; + + if (jobLine.Message != stepLine.Message) + { + isMatch = false; + } + } + + if (isMatch) + { + // Replace the step number and timestamp with the values from the step log + for (var stepIndex = 0; stepIndex < stepLines.Count; stepIndex++) + { + var stepLine = stepLines[stepIndex]; + var jobLine = jobLines[jobIndex + stepIndex]; + + jobLine.Step = stepLine.Step; + jobLine.Number = stepLine.Number; + jobLine.Timestamp = stepLine.Timestamp; + } + return true; + } + } + + return false; + } + + private IList ReadLogLines(ZipArchiveEntry entry, int step) + { + var result = new List(); + + using var logReader = new StreamReader(entry.Open()); + DateTimeOffset lastTimestamp = default; + + for (int lineNumber = 1; !logReader.EndOfStream; lineNumber++) + { + string line = logReader.ReadLine(); + Match logLine = Regex.Match(line, @"^(?\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\d(?:.\d+)?Z )?(?.*)$"); + string timeStampText = logLine.Groups["timestamp"].Value; + string message = StripAnsiiEsacpeSequences(logLine.Groups["message"].Value); + DateTimeOffset timestamp = !string.IsNullOrEmpty(timeStampText) ? DateTimeOffset.Parse(timeStampText) : lastTimestamp; + + result.Add(new LogLine + { + Step = step, + Number = lineNumber, + Timestamp = timestamp, + Message = message + }); + } + + return result; + } + + private string StripAnsiiEsacpeSequences(string input) + { + return Regex.Replace(input, @"\x1B\[[0-?]*[ -/]*[@-~]", ""); + } + + private async Task GetWorkflowRunAsync(string owner, string repository, long runId) + { + WorkflowRun workflowRun = await this.client.Actions.Workflows.Runs.Get(owner, repository, runId); + return workflowRun; + } + + private async Task> GetJobsAsync(WorkflowRun run) + { + List jobs = new(); + for (int pageNumber = 1; ; pageNumber++) + { + ApiOptions options = new() + { + PageSize = 100, + PageCount = 1, + StartPage = pageNumber + }; + + WorkflowJobsResponse jobsResponse = await this.client.Actions.Workflows.Jobs.List(run.Repository.Owner.Login, run.Repository.Name, run.Id, (int)run.RunAttempt, options); + + IReadOnlyList pageJobs = jobsResponse.Jobs; + if (pageJobs.Count == 0) + { + break; + } + + jobs.AddRange(pageJobs); + } + + return jobs; + } + + private async Task GetLogsAsync(WorkflowRun run) + { + var logBytes = await this.client.Actions.Workflows.Runs.GetAttemptLogs(run.Repository.Owner.Login, run.Repository.Name, run.Id, run.RunAttempt); + return new ZipArchive(new MemoryStream(logBytes), ZipArchiveMode.Read, false); + } + + private class LogLine + { + public int Step; + public int Number; + public DateTimeOffset Timestamp; + public string Message; + }; + } +} diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/GitHubActions/GitHubActionsRunQueueWorker.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/GitHubActions/GitHubActionsRunQueueWorker.cs new file mode 100644 index 00000000000..643d7a429e7 --- /dev/null +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/GitHubActions/GitHubActionsRunQueueWorker.cs @@ -0,0 +1,46 @@ +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Azure.Sdk.Tools.PipelineWitness.Configuration; +using Azure.Sdk.Tools.PipelineWitness.Services; +using Azure.Storage.Queues; +using Azure.Storage.Queues.Models; + +using Microsoft.ApplicationInsights; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace Azure.Sdk.Tools.PipelineWitness.GitHubActions +{ + internal class GitHubActionsRunQueueWorker : QueueWorkerBackgroundService + { + private readonly ILogger logger; + private readonly GitHubActionProcessor processor; + + public GitHubActionsRunQueueWorker( + ILogger logger, + GitHubActionProcessor processor, + QueueServiceClient queueServiceClient, + TelemetryClient telemetryClient, + IOptionsMonitor options) + : base( + logger, + telemetryClient, + queueServiceClient, + options.CurrentValue.GitHubActionRunsQueueName, + options) + { + this.logger = logger; + this.processor = processor; + } + + internal override async Task ProcessMessageAsync(QueueMessage message, CancellationToken cancellationToken) + { + this.logger.LogInformation("Processing build.complete event: {MessageText}", message.MessageText); + + var githubMessage = JsonSerializer.Deserialize(message.MessageText); + + await this.processor.ProcessAsync(githubMessage.Owner, githubMessage.Repository, githubMessage.RunId); + } + } +} diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/GitHubActions/GitHubCredentialStore.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/GitHubActions/GitHubCredentialStore.cs new file mode 100644 index 00000000000..79721015575 --- /dev/null +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/GitHubActions/GitHubCredentialStore.cs @@ -0,0 +1,84 @@ +using System; +using System.Diagnostics; +using System.IO; +using System.Runtime.InteropServices; +using System.Threading; +using System.Threading.Tasks; +using Azure.Sdk.Tools.PipelineWitness.Configuration; +using Microsoft.Extensions.Options; +using Octokit; + +namespace Azure.Sdk.Tools.PipelineWitness.GitHubActions; + +public class GitHubCredentialStore : ICredentialStore +{ + private readonly TimeSpan processTimeout = TimeSpan.FromSeconds(13); + private readonly PipelineWitnessSettings settings; + + public GitHubCredentialStore(IOptions options) + { + this.settings = options.Value; + } + + public async Task GetCredentials() + { + return string.IsNullOrEmpty(this.settings.GitHubAccessToken) + ? await GetCliCredentialsAsync() + : new Credentials(this.settings.GitHubAccessToken); + } + + private async Task GetCliCredentialsAsync() + { + Process process = new() + { + StartInfo = GetAzureCliProcessStartInfo(), + EnableRaisingEvents = true + }; + + using ProcessRunner processRunner = new(process, this.processTimeout, CancellationToken.None); + + string output = await processRunner.RunAsync().ConfigureAwait(false); + + return new Credentials(output, AuthenticationType.Bearer); + } + + private static ProcessStartInfo GetAzureCliProcessStartInfo() + { + string environmentPath = Environment.GetEnvironmentVariable("PATH"); + + string command = "gh auth token"; + + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + string programFiles = Environment.GetEnvironmentVariable("ProgramFiles"); + string programFilesx86 = Environment.GetEnvironmentVariable("ProgramFiles(x86)"); + string defaultPath = $"{programFilesx86}\\GitHub CLI;{programFiles}\\GitHub CLI"; + + return new ProcessStartInfo + { + FileName = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.System), "cmd.exe"), + Arguments = $"/d /c \"{command}\"", + UseShellExecute = false, + ErrorDialog = false, + CreateNoWindow = true, + WorkingDirectory = Environment.GetFolderPath(Environment.SpecialFolder.System), + Environment = { { "PATH", !string.IsNullOrEmpty(environmentPath) ? environmentPath : defaultPath } } + }; + } + else + { + string defaultPath = "/usr/bin:/usr/local/bin"; + + return new ProcessStartInfo + { + FileName = "/bin/sh", + Arguments = $"-c \"{command}\"", + UseShellExecute = false, + ErrorDialog = false, + CreateNoWindow = true, + WorkingDirectory = "/bin/", + Environment = { { "PATH", !string.IsNullOrEmpty(environmentPath) ? environmentPath : defaultPath } } + }; + } + } +} diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/GitHubActions/GitHubRunCompleteMessage.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/GitHubActions/GitHubRunCompleteMessage.cs new file mode 100644 index 00000000000..059fe0c3f78 --- /dev/null +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/GitHubActions/GitHubRunCompleteMessage.cs @@ -0,0 +1,8 @@ +namespace Azure.Sdk.Tools.PipelineWitness.GitHubActions; + +internal class GitHubRunCompleteMessage +{ + public string Owner { get; set; } + public string Repository { get; set; } + public long RunId { get; set; } +} diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/GitHubActions/ProcessRunner.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/GitHubActions/ProcessRunner.cs new file mode 100644 index 00000000000..9dbd435a767 --- /dev/null +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/GitHubActions/ProcessRunner.cs @@ -0,0 +1,161 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Threading; +using System.Threading.Tasks; + +namespace Azure.Sdk.Tools.PipelineWitness.GitHubActions; + +internal sealed class ProcessRunner : IDisposable +{ + private readonly Process process; + private readonly TimeSpan timeout; + private readonly TaskCompletionSource tcs; + private readonly TaskCompletionSource> outputTcs; + private readonly TaskCompletionSource> errorTcs; + private readonly ICollection outputData; + private readonly ICollection errorData; + + private readonly CancellationToken cancellationToken; + private readonly CancellationTokenSource timeoutCts; + private CancellationTokenRegistration ctRegistration; + public int ExitCode => this.process.ExitCode; + + public ProcessRunner(Process process, TimeSpan timeout, CancellationToken cancellationToken) + { + this.process = process; + this.timeout = timeout; + + this.outputData = new List(); + this.errorData = new List(); + this.outputTcs = new TaskCompletionSource>(TaskCreationOptions.RunContinuationsAsynchronously); + this.errorTcs = new TaskCompletionSource>(TaskCreationOptions.RunContinuationsAsynchronously); + this.tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + if (timeout.TotalMilliseconds >= 0) + { + this.timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + this.cancellationToken = this.timeoutCts.Token; + } + else + { + this.cancellationToken = cancellationToken; + } + } + + public Task RunAsync() + { + StartProcess(); + return this.tcs.Task; + } + + public string Run() + { + StartProcess(); + return this.tcs.Task.GetAwaiter().GetResult(); + } + + private void StartProcess() + { + if (TrySetCanceled() || this.tcs.Task.IsCompleted) + { + return; + } + + this.process.StartInfo.UseShellExecute = false; + this.process.StartInfo.RedirectStandardOutput = true; + this.process.StartInfo.RedirectStandardError = true; + + this.process.OutputDataReceived += (sender, args) => OnDataReceived(args, this.outputData, this.outputTcs); + this.process.ErrorDataReceived += (sender, args) => OnDataReceived(args, this.errorData, this.errorTcs); + this.process.Exited += (o, e) => _ = HandleExitAsync(); + + this.timeoutCts?.CancelAfter(this.timeout); + + if (!this.process.Start()) + { + TrySetException(new InvalidOperationException($"Failed to start process '{this.process.StartInfo.FileName}'")); + } + + this.process.BeginOutputReadLine(); + this.process.BeginErrorReadLine(); + this.ctRegistration = this.cancellationToken.Register(HandleCancel, false); + } + + private async ValueTask HandleExitAsync() + { + if (this.process.ExitCode == 0) + { + ICollection output = await this.outputTcs.Task.ConfigureAwait(false); + TrySetResult(string.Join(Environment.NewLine, output)); + } + else + { + ICollection error = await this.errorTcs.Task.ConfigureAwait(false); + TrySetException(new InvalidOperationException(string.Join(Environment.NewLine, error))); + } + } + + private void HandleCancel() + { + if (this.tcs.Task.IsCompleted) + { + return; + } + + if (!this.process.HasExited) + { + try + { + this.process.Kill(); + } + catch (Exception ex) + { + TrySetException(ex); + return; + } + } + + TrySetCanceled(); + } + + private static void OnDataReceived(DataReceivedEventArgs args, ICollection data, TaskCompletionSource> tcs) + { + if (args.Data != null) + { + data.Add(args.Data); + } + else + { + tcs.SetResult(data); + } + } + + private void TrySetResult(string result) + { + this.tcs.TrySetResult(result); + } + + private bool TrySetCanceled() + { + if (this.cancellationToken.IsCancellationRequested) + { + this.tcs.TrySetCanceled(this.cancellationToken); + } + + return this.cancellationToken.IsCancellationRequested; + } + + private void TrySetException(Exception exception) + { + this.tcs.TrySetException(exception); + } + + public void Dispose() + { + this.tcs.TrySetCanceled(); + this.process.Dispose(); + this.ctRegistration.Dispose(); + this.timeoutCts?.Dispose(); + } +} diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Startup.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Startup.cs index bcf0d4216fd..8c6523fc8d4 100644 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Startup.cs +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Startup.cs @@ -5,6 +5,7 @@ using Azure.Identity; using Azure.Sdk.Tools.PipelineWitness.ApplicationInsights; using Azure.Sdk.Tools.PipelineWitness.Configuration; +using Azure.Sdk.Tools.PipelineWitness.GitHubActions; using Azure.Sdk.Tools.PipelineWitness.Services; using Azure.Sdk.Tools.PipelineWitness.Services.WorkTokens; @@ -15,72 +16,83 @@ using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Options; using Microsoft.VisualStudio.Services.Client; using Microsoft.VisualStudio.Services.Common; using Microsoft.VisualStudio.Services.WebApi; +using Octokit; -namespace Azure.Sdk.Tools.PipelineWitness +namespace Azure.Sdk.Tools.PipelineWitness; + +public static class Startup { - public static class Startup + public static void Configure(WebApplicationBuilder builder) { - public static void Configure(WebApplicationBuilder builder) - { - PipelineWitnessSettings settings = new(); - IConfigurationSection settingsSection = builder.Configuration.GetSection("PipelineWitness"); - settingsSection.Bind(settings); + PipelineWitnessSettings settings = new(); + IConfigurationSection settingsSection = builder.Configuration.GetSection("PipelineWitness"); + settingsSection.Bind(settings); - builder.Services.AddApplicationInsightsTelemetry(builder.Configuration); - builder.Services.AddApplicationInsightsTelemetryProcessor(); - builder.Services.AddTransient(); + builder.Services.AddApplicationInsightsTelemetry(builder.Configuration); + builder.Services.AddApplicationInsightsTelemetryProcessor(); + builder.Services.AddTransient(); - builder.Services.AddSingleton(); + builder.Services.AddSingleton(); - builder.Services.AddAzureClients(azureBuilder => - { - azureBuilder.UseCredential(provider => provider.GetRequiredService()); - azureBuilder.AddCosmosServiceClient(new Uri(settings.CosmosAccountUri)); - azureBuilder.AddBlobServiceClient(new Uri(settings.BlobStorageAccountUri)); - azureBuilder.AddQueueServiceClient(new Uri(settings.QueueStorageAccountUri)) - .ConfigureOptions(o => o.MessageEncoding = Storage.Queues.QueueMessageEncoding.Base64); - }); + builder.Services.AddAzureClients(azureBuilder => + { + azureBuilder.UseCredential(provider => provider.GetRequiredService()); + azureBuilder.AddCosmosServiceClient(new Uri(settings.CosmosAccountUri)); + azureBuilder.AddBlobServiceClient(new Uri(settings.BlobStorageAccountUri)); + azureBuilder.AddQueueServiceClient(new Uri(settings.QueueStorageAccountUri)) + .ConfigureOptions(o => o.MessageEncoding = Storage.Queues.QueueMessageEncoding.Base64); + }); - builder.Services.AddSingleton(provider => new CosmosAsyncLockProvider(provider.GetRequiredService(), settings.CosmosDatabase, settings.CosmosAsyncLockContainer)); - builder.Services.AddTransient(CreateVssConnection); + builder.Services.AddSingleton(provider => new CosmosAsyncLockProvider(provider.GetRequiredService(), settings.CosmosDatabase, settings.CosmosAsyncLockContainer)); + builder.Services.AddTransient(CreateVssConnection); - builder.Services.AddLogging(); - builder.Services.AddTransient(); - builder.Services.AddTransient>(provider => provider.GetRequiredService); + builder.Services.AddLogging(); - builder.Services.Configure(settingsSection); + builder.Services.Configure(settingsSection); + builder.Services.AddSingleton(); + builder.Services.AddSingleton, PostConfigureKeyVaultSettings>(); - builder.Services.AddHostedService(settings.BuildCompleteWorkerCount); - builder.Services.AddHostedService(); - } + builder.Services.AddTransient(); + builder.Services.AddTransient>(provider => provider.GetRequiredService); + builder.Services.AddHostedService(settings.BuildCompleteWorkerCount); + + builder.Services.AddSingleton(); + builder.Services.AddTransient(); + builder.Services.AddHostedService(settings.GitHubActionRunsWorkerCount); - private static void AddHostedService(this IServiceCollection services, int instanceCount) where T : class, IHostedService + if (settings.BuildDefinitionWorkerEnabled) { - for (int i = 0; i < instanceCount; i++) - { - services.AddSingleton(); - } + builder.Services.AddHostedService(); } + } - private static void AddCosmosServiceClient(this TBuilder builder, Uri serviceUri) where TBuilder : IAzureClientFactoryBuilderWithCredential + private static void AddHostedService(this IServiceCollection services, int instanceCount) where T : class, IHostedService + { + for (int i = 0; i < instanceCount; i++) { - builder.RegisterClientFactory((CosmosClientOptions options, TokenCredential cred) => new CosmosClient(serviceUri.AbsoluteUri, cred, options)); + services.AddSingleton(); } + } - private static VssConnection CreateVssConnection(IServiceProvider provider) - { - TokenCredential azureCredential = provider.GetRequiredService(); - TokenRequestContext tokenRequestContext = new(VssAadSettings.DefaultScopes); - AccessToken token = azureCredential.GetToken(tokenRequestContext, CancellationToken.None); + private static void AddCosmosServiceClient(this TBuilder builder, Uri serviceUri) where TBuilder : IAzureClientFactoryBuilderWithCredential + { + builder.RegisterClientFactory((CosmosClientOptions options, TokenCredential cred) => new CosmosClient(serviceUri.AbsoluteUri, cred, options)); + } + + private static VssConnection CreateVssConnection(IServiceProvider provider) + { + TokenCredential azureCredential = provider.GetRequiredService(); + TokenRequestContext tokenRequestContext = new(VssAadSettings.DefaultScopes); + Azure.Core.AccessToken token = azureCredential.GetToken(tokenRequestContext, CancellationToken.None); - Uri organizationUrl = new("https://dev.azure.com/azure-sdk"); - VssAadCredential vssCredential = new(new VssAadToken("Bearer", token.Token)); - VssHttpRequestSettings settings = VssClientHttpRequestSettings.Default.Clone(); + Uri organizationUrl = new("https://dev.azure.com/azure-sdk"); + VssAadCredential vssCredential = new(new VssAadToken("Bearer", token.Token)); + VssHttpRequestSettings settings = VssClientHttpRequestSettings.Default.Clone(); - return new VssConnection(organizationUrl, vssCredential, settings); - } + return new VssConnection(organizationUrl, vssCredential, settings); } } diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/appsettings.development.json b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/appsettings.development.json index bc3a3f6b2e5..5eb9656d162 100644 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/appsettings.development.json +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/appsettings.development.json @@ -4,7 +4,7 @@ "FormatterName": "simple", "FormatterOptions": { "SingleLine": true, - "IncludeScopes": true, + "IncludeScopes": false, "TimestampFormat": "HH:mm:ss ", "UseUtcTimestamp": true, "JsonWriterOptions": { @@ -17,8 +17,11 @@ "QueueStorageAccountUri": "https://pipelinewitnesstest.queue.core.windows.net", "BlobStorageAccountUri": "https://pipelinelogstest.blob.core.windows.net", "CosmosAccountUri": "https://pipelinewitnesstest.documents.azure.com", + "GitHubWebhookSecret": "https://pipelinewitnesstest.vault.azure.net/secrets/github-webhook-validation-secret", + "GitHubAccessToken": null, "BuildDefinitionLoopPeriod": "00:01:00", + "BuildDefinitionWorkerEnabled": true, "BuildCompleteWorkerCount": 1, - "BuildLogBundlesWorkerCount": 1 + "GitHubActionRunsWorkerCount": 1 } } diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/appsettings.json b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/appsettings.json index ffa0ebc116b..ef559a750de 100644 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/appsettings.json +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/appsettings.json @@ -23,12 +23,16 @@ "CosmosAsyncLockContainer": "locks", "BuildCompleteQueueName": "azurepipelines-build-completed", "BuildCompleteWorkerCount": 10, + "BuildDefinitionLoopPeriod": "00:05:00", + "GitHubActionRunsQueueName": "github-actionrun-completed", + "GitHubActionRunsWorkerCount": 10, + "GitHubWebhookSecret": "https://pipelinewitnessprod.vault.azure.net/secrets/github-webhook-validation-secret", + "GitHubAccessToken": "https://pipelinewitnessprod.vault.azure.net/secrets/azuresdk-github-pat", "MessageLeasePeriod": "00:03:00", "MessageErrorSleepPeriod": "00:00:10", "MaxDequeueCount": 5, "Account": "azure-sdk", "Projects": [ "internal", "playground", "public" ], - "BuildDefinitionLoopPeriod": "00:05:00", "PipelineOwnersArtifactName": "pipelineOwners", "PipelineOwnersFilePath": "pipelineOwners/pipelineOwners.json", "PipelineOwnersDefinitionId": 5112 diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/appsettings.staging.json b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/appsettings.staging.json index c5495c655c0..36e27c43deb 100644 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/appsettings.staging.json +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/appsettings.staging.json @@ -2,6 +2,8 @@ "PipelineWitness": { "QueueStorageAccountUri": "https://pipelinewitnessstaging.queue.core.windows.net", "BlobStorageAccountUri": "https://pipelinelogsstaging.blob.core.windows.net", - "CosmosAccountUri": "https://pipelinewitnessstaging.documents.azure.com" + "CosmosAccountUri": "https://pipelinewitnessstaging.documents.azure.com", + "GitHubWebhookSecret": "https://pipelinewitnessstaging.vault.azure.net/secrets/github-webhook-validation-secret", + "GitHubAccessToken": "https://pipelinewitnessstaging.vault.azure.net/secrets/azuresdk-github-pat" } } diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/appsettings.test.json b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/appsettings.test.json index 428f706905c..7278dd05b6d 100644 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/appsettings.test.json +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/appsettings.test.json @@ -2,6 +2,8 @@ "PipelineWitness": { "QueueStorageAccountUri": "https://pipelinewitnesstest.queue.core.windows.net", "BlobStorageAccountUri": "https://pipelinelogstest.blob.core.windows.net", - "CosmosAccountUri": "https://pipelinewitnesstest.documents.azure.com" + "CosmosAccountUri": "https://pipelinewitnesstest.documents.azure.com", + "GitHubWebhookSecret": "https://pipelinewitnesstest.vault.azure.net/secrets/github-webhook-validation-secret", + "GitHubAccessToken": "https://pipelinewitnesstest.vault.azure.net/secrets/azuresdk-github-pat" } } diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/deploy.ps1 b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/deploy.ps1 new file mode 100644 index 00000000000..a5597110015 --- /dev/null +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/deploy.ps1 @@ -0,0 +1,47 @@ +<# +.SYNOPSIS + Builds and deploys the dotnet app. +#> +param( + [Parameter(Mandatory)] + [validateSet('staging', 'test')] + [string]$Target +) + +$repoRoot = Resolve-Path "$PSScriptRoot/../../.." +. "$repoRoot/eng/common/scripts/Helpers/CommandInvocation-Helpers.ps1" + +Push-Location $PSScriptRoot +try { + $subscriptionName = $Target -eq 'test' ? 'Azure SDK Developer Playground' : 'Azure SDK Engineering System' + $parametersFile = "../infrastructure/bicep/parameters.$Target.json" + + $parameters = (Get-Content -Path $parametersFile -Raw | ConvertFrom-Json).parameters + $resourceGroupName = $parameters.appResourceGroupName.value + $resourceName = $parameters.webAppName.value + + Write-Host "Deploying web app to:`n" + ` + " Subscription: $subscriptionName`n" + ` + " Resource Group: $resourceGroupName`n" + ` + " Resource: $resourceName`n" + + $artifactsPath = "$repoRoot/artifacts" + $publishPath = "$artifactsPath/app" + + Remove-Item $publishPath -Recurse -Force -ErrorAction SilentlyContinue + + Invoke-LoggedCommand "dotnet publish --configuration Release --output '$publishPath'" + + Compress-Archive -Path "$publishPath/*" -DestinationPath "$artifactsPath/pipeline-witness.zip" -Force + if($?) { + Write-Host "pipeline-witness.zip created" + } else { + Write-Error "Failed to create pipeline-witness.zip" + exit 1 + } + + Invoke-LoggedCommand "az webapp deploy --src-path '$artifactsPath/pipeline-witness.zip' --clean true --restart true --type zip --subscription '$subscriptionName' --resource-group '$resourceGroupName' --name '$resourceName'" +} +finally { + Pop-Location +} diff --git a/tools/pipeline-witness/docs/github-actions/rest-api.md b/tools/pipeline-witness/docs/github-actions/rest-api.md new file mode 100644 index 00000000000..12acccc55e9 --- /dev/null +++ b/tools/pipeline-witness/docs/github-actions/rest-api.md @@ -0,0 +1,22 @@ +# Actions REST API +https://docs.github.com/en/rest/actions?apiVersion=2022-11-28 + +## Hierarchy +``` +Workflow + WorkflowRuns + Attempts + Jobs + Steps + Jobs + Steps +``` + + +## Logs + +### Run +`/repos/{owner}/{repo}/actions/runs/{run_id}/logs` + +### Run Attempt +`/repos/{owner}/{repo}/actions/runs/{run_id}/attempts/{attempt_number}/logs` diff --git a/tools/pipeline-witness/docs/github-actions/rest-api/attempt-jobs.md b/tools/pipeline-witness/docs/github-actions/rest-api/attempt-jobs.md new file mode 100644 index 00000000000..b4016558bc3 --- /dev/null +++ b/tools/pipeline-witness/docs/github-actions/rest-api/attempt-jobs.md @@ -0,0 +1,118 @@ +# Run Attempt Jobs +`/repos/{owner}/{repo}/actions/runs/{run_id}/attempts/{attempt_number}/jobs` + +```json +{ + "total_count": 1, + "jobs": [ + { + "id": 399444496, + "run_id": 29679449, + "run_url": "https://api.github.com/repos/octo-org/octo-repo/actions/runs/29679449", + "node_id": "MDEyOldvcmtmbG93IEpvYjM5OTQ0NDQ5Ng==", + "head_sha": "f83a356604ae3c5d03e1b46ef4d1ca77d64a90b0", + "url": "https://api.github.com/repos/octo-org/octo-repo/actions/jobs/399444496", + "html_url": "https://github.com/octo-org/octo-repo/runs/399444496", + "status": "completed", + "conclusion": "success", + "started_at": "2020-01-20T17:42:40Z", + "completed_at": "2020-01-20T17:44:39Z", + "name": "build", + "steps": [ + { + "name": "Set up job", + "status": "completed", + "conclusion": "success", + "number": 1, + "started_at": "2020-01-20T09:42:40.000-08:00", + "completed_at": "2020-01-20T09:42:41.000-08:00" + }, + { + "name": "Run actions/checkout@v2", + "status": "completed", + "conclusion": "success", + "number": 2, + "started_at": "2020-01-20T09:42:41.000-08:00", + "completed_at": "2020-01-20T09:42:45.000-08:00" + }, + { + "name": "Set up Ruby", + "status": "completed", + "conclusion": "success", + "number": 3, + "started_at": "2020-01-20T09:42:45.000-08:00", + "completed_at": "2020-01-20T09:42:45.000-08:00" + }, + { + "name": "Run actions/cache@v3", + "status": "completed", + "conclusion": "success", + "number": 4, + "started_at": "2020-01-20T09:42:45.000-08:00", + "completed_at": "2020-01-20T09:42:48.000-08:00" + }, + { + "name": "Install Bundler", + "status": "completed", + "conclusion": "success", + "number": 5, + "started_at": "2020-01-20T09:42:48.000-08:00", + "completed_at": "2020-01-20T09:42:52.000-08:00" + }, + { + "name": "Install Gems", + "status": "completed", + "conclusion": "success", + "number": 6, + "started_at": "2020-01-20T09:42:52.000-08:00", + "completed_at": "2020-01-20T09:42:53.000-08:00" + }, + { + "name": "Run Tests", + "status": "completed", + "conclusion": "success", + "number": 7, + "started_at": "2020-01-20T09:42:53.000-08:00", + "completed_at": "2020-01-20T09:42:59.000-08:00" + }, + { + "name": "Deploy to Heroku", + "status": "completed", + "conclusion": "success", + "number": 8, + "started_at": "2020-01-20T09:42:59.000-08:00", + "completed_at": "2020-01-20T09:44:39.000-08:00" + }, + { + "name": "Post actions/cache@v3", + "status": "completed", + "conclusion": "success", + "number": 16, + "started_at": "2020-01-20T09:44:39.000-08:00", + "completed_at": "2020-01-20T09:44:39.000-08:00" + }, + { + "name": "Complete job", + "status": "completed", + "conclusion": "success", + "number": 17, + "started_at": "2020-01-20T09:44:39.000-08:00", + "completed_at": "2020-01-20T09:44:39.000-08:00" + } + ], + "check_run_url": "https://api.github.com/repos/octo-org/octo-repo/check-runs/399444496", + "labels": [ + "self-hosted", + "foo", + "bar" + ], + "runner_id": 1, + "runner_name": "my runner", + "runner_group_id": 2, + "runner_group_name": "my runner group", + "workflow_name": "CI", + "head_branch": "main" + } + ] +} +``` diff --git a/tools/pipeline-witness/docs/github-actions/rest-api/attempt.md b/tools/pipeline-witness/docs/github-actions/rest-api/attempt.md new file mode 100644 index 00000000000..0c6d5fe6617 --- /dev/null +++ b/tools/pipeline-witness/docs/github-actions/rest-api/attempt.md @@ -0,0 +1,241 @@ +# Run Attempt +`/repos/{owner}/{repo}/actions/runs/{run_id}/attempts/{attempt_number}` +```json +{ + "id": 30433642, + "name": "Build", + "node_id": "MDEyOldvcmtmbG93IFJ1bjI2OTI4OQ==", + "check_suite_id": 42, + "check_suite_node_id": "MDEwOkNoZWNrU3VpdGU0Mg==", + "head_branch": "main", + "head_sha": "acb5820ced9479c074f688cc328bf03f341a511d", + "path": ".github/workflows/build.yml@main", + "run_number": 562, + "event": "push", + "display_title": "Update README.md", + "status": "queued", + "conclusion": null, + "workflow_id": 159038, + "url": "https://api.github.com/repos/octo-org/octo-repo/actions/runs/30433642", + "html_url": "https://github.com/octo-org/octo-repo/actions/runs/30433642", + "pull_requests": [], + "created_at": "2020-01-22T19:33:08Z", + "updated_at": "2020-01-22T19:33:08Z", + "actor": { + "login": "octocat", + "id": 1, + "node_id": "MDQ6VXNlcjE=", + "avatar_url": "https://github.com/images/error/octocat_happy.gif", + "gravatar_id": "", + "url": "https://api.github.com/users/octocat", + "html_url": "https://github.com/octocat", + "followers_url": "https://api.github.com/users/octocat/followers", + "following_url": "https://api.github.com/users/octocat/following{/other_user}", + "gists_url": "https://api.github.com/users/octocat/gists{/gist_id}", + "starred_url": "https://api.github.com/users/octocat/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/octocat/subscriptions", + "organizations_url": "https://api.github.com/users/octocat/orgs", + "repos_url": "https://api.github.com/users/octocat/repos", + "events_url": "https://api.github.com/users/octocat/events{/privacy}", + "received_events_url": "https://api.github.com/users/octocat/received_events", + "type": "User", + "site_admin": false + }, + "run_attempt": 1, + "referenced_workflows": [ + { + "path": "octocat/Hello-World/.github/workflows/deploy.yml@main", + "sha": "86e8bc9ecf7d38b1ed2d2cfb8eb87ba9b35b01db", + "ref": "refs/heads/main" + }, + { + "path": "octo-org/octo-repo/.github/workflows/report.yml@v2", + "sha": "79e9790903e1c3373b1a3e3a941d57405478a232", + "ref": "refs/tags/v2" + }, + { + "path": "octo-org/octo-repo/.github/workflows/secure.yml@1595d4b6de6a9e9751fb270a41019ce507d4099e", + "sha": "1595d4b6de6a9e9751fb270a41019ce507d4099e" + } + ], + "run_started_at": "2020-01-22T19:33:08Z", + "triggering_actor": { + "login": "octocat", + "id": 1, + "node_id": "MDQ6VXNlcjE=", + "avatar_url": "https://github.com/images/error/octocat_happy.gif", + "gravatar_id": "", + "url": "https://api.github.com/users/octocat", + "html_url": "https://github.com/octocat", + "followers_url": "https://api.github.com/users/octocat/followers", + "following_url": "https://api.github.com/users/octocat/following{/other_user}", + "gists_url": "https://api.github.com/users/octocat/gists{/gist_id}", + "starred_url": "https://api.github.com/users/octocat/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/octocat/subscriptions", + "organizations_url": "https://api.github.com/users/octocat/orgs", + "repos_url": "https://api.github.com/users/octocat/repos", + "events_url": "https://api.github.com/users/octocat/events{/privacy}", + "received_events_url": "https://api.github.com/users/octocat/received_events", + "type": "User", + "site_admin": false + }, + "jobs_url": "https://api.github.com/repos/octo-org/octo-repo/actions/runs/30433642/jobs", + "logs_url": "https://api.github.com/repos/octo-org/octo-repo/actions/runs/30433642/logs", + "check_suite_url": "https://api.github.com/repos/octo-org/octo-repo/check-suites/414944374", + "artifacts_url": "https://api.github.com/repos/octo-org/octo-repo/actions/runs/30433642/artifacts", + "cancel_url": "https://api.github.com/repos/octo-org/octo-repo/actions/runs/30433642/cancel", + "rerun_url": "https://api.github.com/repos/octo-org/octo-repo/actions/runs/30433642/rerun", + "previous_attempt_url": "https://api.github.com/repos/octo-org/octo-repo/actions/runs/30433642/attempts/1", + "workflow_url": "https://api.github.com/repos/octo-org/octo-repo/actions/workflows/159038", + "head_commit": { + "id": "acb5820ced9479c074f688cc328bf03f341a511d", + "tree_id": "d23f6eedb1e1b9610bbc754ddb5197bfe7271223", + "message": "Create linter.yaml", + "timestamp": "2020-01-22T19:33:05Z", + "author": { + "name": "Octo Cat", + "email": "octocat@github.com" + }, + "committer": { + "name": "GitHub", + "email": "noreply@github.com" + } + }, + "repository": { + "id": 1296269, + "node_id": "MDEwOlJlcG9zaXRvcnkxMjk2MjY5", + "name": "Hello-World", + "full_name": "octocat/Hello-World", + "owner": { + "login": "octocat", + "id": 1, + "node_id": "MDQ6VXNlcjE=", + "avatar_url": "https://github.com/images/error/octocat_happy.gif", + "gravatar_id": "", + "url": "https://api.github.com/users/octocat", + "html_url": "https://github.com/octocat", + "followers_url": "https://api.github.com/users/octocat/followers", + "following_url": "https://api.github.com/users/octocat/following{/other_user}", + "gists_url": "https://api.github.com/users/octocat/gists{/gist_id}", + "starred_url": "https://api.github.com/users/octocat/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/octocat/subscriptions", + "organizations_url": "https://api.github.com/users/octocat/orgs", + "repos_url": "https://api.github.com/users/octocat/repos", + "events_url": "https://api.github.com/users/octocat/events{/privacy}", + "received_events_url": "https://api.github.com/users/octocat/received_events", + "type": "User", + "site_admin": false + }, + "private": false, + "html_url": "https://github.com/octocat/Hello-World", + "description": "This your first repo!", + "fork": false, + "url": "https://api.github.com/repos/octocat/Hello-World", + "archive_url": "https://api.github.com/repos/octocat/Hello-World/{archive_format}{/ref}", + "assignees_url": "https://api.github.com/repos/octocat/Hello-World/assignees{/user}", + "blobs_url": "https://api.github.com/repos/octocat/Hello-World/git/blobs{/sha}", + "branches_url": "https://api.github.com/repos/octocat/Hello-World/branches{/branch}", + "collaborators_url": "https://api.github.com/repos/octocat/Hello-World/collaborators{/collaborator}", + "comments_url": "https://api.github.com/repos/octocat/Hello-World/comments{/number}", + "commits_url": "https://api.github.com/repos/octocat/Hello-World/commits{/sha}", + "compare_url": "https://api.github.com/repos/octocat/Hello-World/compare/{base}...{head}", + "contents_url": "https://api.github.com/repos/octocat/Hello-World/contents/{+path}", + "contributors_url": "https://api.github.com/repos/octocat/Hello-World/contributors", + "deployments_url": "https://api.github.com/repos/octocat/Hello-World/deployments", + "downloads_url": "https://api.github.com/repos/octocat/Hello-World/downloads", + "events_url": "https://api.github.com/repos/octocat/Hello-World/events", + "forks_url": "https://api.github.com/repos/octocat/Hello-World/forks", + "git_commits_url": "https://api.github.com/repos/octocat/Hello-World/git/commits{/sha}", + "git_refs_url": "https://api.github.com/repos/octocat/Hello-World/git/refs{/sha}", + "git_tags_url": "https://api.github.com/repos/octocat/Hello-World/git/tags{/sha}", + "git_url": "git:github.com/octocat/Hello-World.git", + "issue_comment_url": "https://api.github.com/repos/octocat/Hello-World/issues/comments{/number}", + "issue_events_url": "https://api.github.com/repos/octocat/Hello-World/issues/events{/number}", + "issues_url": "https://api.github.com/repos/octocat/Hello-World/issues{/number}", + "keys_url": "https://api.github.com/repos/octocat/Hello-World/keys{/key_id}", + "labels_url": "https://api.github.com/repos/octocat/Hello-World/labels{/name}", + "languages_url": "https://api.github.com/repos/octocat/Hello-World/languages", + "merges_url": "https://api.github.com/repos/octocat/Hello-World/merges", + "milestones_url": "https://api.github.com/repos/octocat/Hello-World/milestones{/number}", + "notifications_url": "https://api.github.com/repos/octocat/Hello-World/notifications{?since,all,participating}", + "pulls_url": "https://api.github.com/repos/octocat/Hello-World/pulls{/number}", + "releases_url": "https://api.github.com/repos/octocat/Hello-World/releases{/id}", + "ssh_url": "git@github.com:octocat/Hello-World.git", + "stargazers_url": "https://api.github.com/repos/octocat/Hello-World/stargazers", + "statuses_url": "https://api.github.com/repos/octocat/Hello-World/statuses/{sha}", + "subscribers_url": "https://api.github.com/repos/octocat/Hello-World/subscribers", + "subscription_url": "https://api.github.com/repos/octocat/Hello-World/subscription", + "tags_url": "https://api.github.com/repos/octocat/Hello-World/tags", + "teams_url": "https://api.github.com/repos/octocat/Hello-World/teams", + "trees_url": "https://api.github.com/repos/octocat/Hello-World/git/trees{/sha}", + "hooks_url": "http://api.github.com/repos/octocat/Hello-World/hooks" + }, + "head_repository": { + "id": 217723378, + "node_id": "MDEwOlJlcG9zaXRvcnkyMTc3MjMzNzg=", + "name": "octo-repo", + "full_name": "octo-org/octo-repo", + "private": true, + "owner": { + "login": "octocat", + "id": 1, + "node_id": "MDQ6VXNlcjE=", + "avatar_url": "https://github.com/images/error/octocat_happy.gif", + "gravatar_id": "", + "url": "https://api.github.com/users/octocat", + "html_url": "https://github.com/octocat", + "followers_url": "https://api.github.com/users/octocat/followers", + "following_url": "https://api.github.com/users/octocat/following{/other_user}", + "gists_url": "https://api.github.com/users/octocat/gists{/gist_id}", + "starred_url": "https://api.github.com/users/octocat/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/octocat/subscriptions", + "organizations_url": "https://api.github.com/users/octocat/orgs", + "repos_url": "https://api.github.com/users/octocat/repos", + "events_url": "https://api.github.com/users/octocat/events{/privacy}", + "received_events_url": "https://api.github.com/users/octocat/received_events", + "type": "User", + "site_admin": false + }, + "html_url": "https://github.com/octo-org/octo-repo", + "description": null, + "fork": false, + "url": "https://api.github.com/repos/octo-org/octo-repo", + "forks_url": "https://api.github.com/repos/octo-org/octo-repo/forks", + "keys_url": "https://api.github.com/repos/octo-org/octo-repo/keys{/key_id}", + "collaborators_url": "https://api.github.com/repos/octo-org/octo-repo/collaborators{/collaborator}", + "teams_url": "https://api.github.com/repos/octo-org/octo-repo/teams", + "hooks_url": "https://api.github.com/repos/octo-org/octo-repo/hooks", + "issue_events_url": "https://api.github.com/repos/octo-org/octo-repo/issues/events{/number}", + "events_url": "https://api.github.com/repos/octo-org/octo-repo/events", + "assignees_url": "https://api.github.com/repos/octo-org/octo-repo/assignees{/user}", + "branches_url": "https://api.github.com/repos/octo-org/octo-repo/branches{/branch}", + "tags_url": "https://api.github.com/repos/octo-org/octo-repo/tags", + "blobs_url": "https://api.github.com/repos/octo-org/octo-repo/git/blobs{/sha}", + "git_tags_url": "https://api.github.com/repos/octo-org/octo-repo/git/tags{/sha}", + "git_refs_url": "https://api.github.com/repos/octo-org/octo-repo/git/refs{/sha}", + "trees_url": "https://api.github.com/repos/octo-org/octo-repo/git/trees{/sha}", + "statuses_url": "https://api.github.com/repos/octo-org/octo-repo/statuses/{sha}", + "languages_url": "https://api.github.com/repos/octo-org/octo-repo/languages", + "stargazers_url": "https://api.github.com/repos/octo-org/octo-repo/stargazers", + "contributors_url": "https://api.github.com/repos/octo-org/octo-repo/contributors", + "subscribers_url": "https://api.github.com/repos/octo-org/octo-repo/subscribers", + "subscription_url": "https://api.github.com/repos/octo-org/octo-repo/subscription", + "commits_url": "https://api.github.com/repos/octo-org/octo-repo/commits{/sha}", + "git_commits_url": "https://api.github.com/repos/octo-org/octo-repo/git/commits{/sha}", + "comments_url": "https://api.github.com/repos/octo-org/octo-repo/comments{/number}", + "issue_comment_url": "https://api.github.com/repos/octo-org/octo-repo/issues/comments{/number}", + "contents_url": "https://api.github.com/repos/octo-org/octo-repo/contents/{+path}", + "compare_url": "https://api.github.com/repos/octo-org/octo-repo/compare/{base}...{head}", + "merges_url": "https://api.github.com/repos/octo-org/octo-repo/merges", + "archive_url": "https://api.github.com/repos/octo-org/octo-repo/{archive_format}{/ref}", + "downloads_url": "https://api.github.com/repos/octo-org/octo-repo/downloads", + "issues_url": "https://api.github.com/repos/octo-org/octo-repo/issues{/number}", + "pulls_url": "https://api.github.com/repos/octo-org/octo-repo/pulls{/number}", + "milestones_url": "https://api.github.com/repos/octo-org/octo-repo/milestones{/number}", + "notifications_url": "https://api.github.com/repos/octo-org/octo-repo/notifications{?since,all,participating}", + "labels_url": "https://api.github.com/repos/octo-org/octo-repo/labels{/name}", + "releases_url": "https://api.github.com/repos/octo-org/octo-repo/releases{/id}", + "deployments_url": "https://api.github.com/repos/octo-org/octo-repo/deployments" + } +} +``` diff --git a/tools/pipeline-witness/docs/github-actions/rest-api/job.md b/tools/pipeline-witness/docs/github-actions/rest-api/job.md new file mode 100644 index 00000000000..319031a5f4d --- /dev/null +++ b/tools/pipeline-witness/docs/github-actions/rest-api/job.md @@ -0,0 +1,112 @@ +# Job +`/repos/{owner}/{repo}/actions/jobs/{job_id}` +```json +{ + "id": 399444496, + "run_id": 29679449, + "run_url": "https://api.github.com/repos/octo-org/octo-repo/actions/runs/29679449", + "node_id": "MDEyOldvcmtmbG93IEpvYjM5OTQ0NDQ5Ng==", + "head_sha": "f83a356604ae3c5d03e1b46ef4d1ca77d64a90b0", + "url": "https://api.github.com/repos/octo-org/octo-repo/actions/jobs/399444496", + "html_url": "https://github.com/octo-org/octo-repo/runs/399444496", + "status": "completed", + "conclusion": "success", + "started_at": "2020-01-20T17:42:40Z", + "completed_at": "2020-01-20T17:44:39Z", + "name": "build", + "steps": [ + { + "name": "Set up job", + "status": "completed", + "conclusion": "success", + "number": 1, + "started_at": "2020-01-20T09:42:40.000-08:00", + "completed_at": "2020-01-20T09:42:41.000-08:00" + }, + { + "name": "Run actions/checkout@v2", + "status": "completed", + "conclusion": "success", + "number": 2, + "started_at": "2020-01-20T09:42:41.000-08:00", + "completed_at": "2020-01-20T09:42:45.000-08:00" + }, + { + "name": "Set up Ruby", + "status": "completed", + "conclusion": "success", + "number": 3, + "started_at": "2020-01-20T09:42:45.000-08:00", + "completed_at": "2020-01-20T09:42:45.000-08:00" + }, + { + "name": "Run actions/cache@v3", + "status": "completed", + "conclusion": "success", + "number": 4, + "started_at": "2020-01-20T09:42:45.000-08:00", + "completed_at": "2020-01-20T09:42:48.000-08:00" + }, + { + "name": "Install Bundler", + "status": "completed", + "conclusion": "success", + "number": 5, + "started_at": "2020-01-20T09:42:48.000-08:00", + "completed_at": "2020-01-20T09:42:52.000-08:00" + }, + { + "name": "Install Gems", + "status": "completed", + "conclusion": "success", + "number": 6, + "started_at": "2020-01-20T09:42:52.000-08:00", + "completed_at": "2020-01-20T09:42:53.000-08:00" + }, + { + "name": "Run Tests", + "status": "completed", + "conclusion": "success", + "number": 7, + "started_at": "2020-01-20T09:42:53.000-08:00", + "completed_at": "2020-01-20T09:42:59.000-08:00" + }, + { + "name": "Deploy to Heroku", + "status": "completed", + "conclusion": "success", + "number": 8, + "started_at": "2020-01-20T09:42:59.000-08:00", + "completed_at": "2020-01-20T09:44:39.000-08:00" + }, + { + "name": "Post actions/cache@v3", + "status": "completed", + "conclusion": "success", + "number": 16, + "started_at": "2020-01-20T09:44:39.000-08:00", + "completed_at": "2020-01-20T09:44:39.000-08:00" + }, + { + "name": "Complete job", + "status": "completed", + "conclusion": "success", + "number": 17, + "started_at": "2020-01-20T09:44:39.000-08:00", + "completed_at": "2020-01-20T09:44:39.000-08:00" + } + ], + "check_run_url": "https://api.github.com/repos/octo-org/octo-repo/check-runs/399444496", + "labels": [ + "self-hosted", + "foo", + "bar" + ], + "runner_id": 1, + "runner_name": "my runner", + "runner_group_id": 2, + "runner_group_name": "my runner group", + "workflow_name": "CI", + "head_branch": "main" +} +``` diff --git a/tools/pipeline-witness/docs/github-actions/rest-api/run-jobs.md b/tools/pipeline-witness/docs/github-actions/rest-api/run-jobs.md new file mode 100644 index 00000000000..8f94a0068ed --- /dev/null +++ b/tools/pipeline-witness/docs/github-actions/rest-api/run-jobs.md @@ -0,0 +1,118 @@ +# Run Jobs +`/repos/{owner}/{repo}/actions/runs/{run_id}/jobs` + +```json +{ + "total_count": 1, + "jobs": [ + { + "id": 399444496, + "run_id": 29679449, + "run_url": "https://api.github.com/repos/octo-org/octo-repo/actions/runs/29679449", + "node_id": "MDEyOldvcmtmbG93IEpvYjM5OTQ0NDQ5Ng==", + "head_sha": "f83a356604ae3c5d03e1b46ef4d1ca77d64a90b0", + "url": "https://api.github.com/repos/octo-org/octo-repo/actions/jobs/399444496", + "html_url": "https://github.com/octo-org/octo-repo/runs/399444496", + "status": "completed", + "conclusion": "success", + "started_at": "2020-01-20T17:42:40Z", + "completed_at": "2020-01-20T17:44:39Z", + "name": "build", + "steps": [ + { + "name": "Set up job", + "status": "completed", + "conclusion": "success", + "number": 1, + "started_at": "2020-01-20T09:42:40.000-08:00", + "completed_at": "2020-01-20T09:42:41.000-08:00" + }, + { + "name": "Run actions/checkout@v2", + "status": "completed", + "conclusion": "success", + "number": 2, + "started_at": "2020-01-20T09:42:41.000-08:00", + "completed_at": "2020-01-20T09:42:45.000-08:00" + }, + { + "name": "Set up Ruby", + "status": "completed", + "conclusion": "success", + "number": 3, + "started_at": "2020-01-20T09:42:45.000-08:00", + "completed_at": "2020-01-20T09:42:45.000-08:00" + }, + { + "name": "Run actions/cache@v3", + "status": "completed", + "conclusion": "success", + "number": 4, + "started_at": "2020-01-20T09:42:45.000-08:00", + "completed_at": "2020-01-20T09:42:48.000-08:00" + }, + { + "name": "Install Bundler", + "status": "completed", + "conclusion": "success", + "number": 5, + "started_at": "2020-01-20T09:42:48.000-08:00", + "completed_at": "2020-01-20T09:42:52.000-08:00" + }, + { + "name": "Install Gems", + "status": "completed", + "conclusion": "success", + "number": 6, + "started_at": "2020-01-20T09:42:52.000-08:00", + "completed_at": "2020-01-20T09:42:53.000-08:00" + }, + { + "name": "Run Tests", + "status": "completed", + "conclusion": "success", + "number": 7, + "started_at": "2020-01-20T09:42:53.000-08:00", + "completed_at": "2020-01-20T09:42:59.000-08:00" + }, + { + "name": "Deploy to Heroku", + "status": "completed", + "conclusion": "success", + "number": 8, + "started_at": "2020-01-20T09:42:59.000-08:00", + "completed_at": "2020-01-20T09:44:39.000-08:00" + }, + { + "name": "Post actions/cache@v3", + "status": "completed", + "conclusion": "success", + "number": 16, + "started_at": "2020-01-20T09:44:39.000-08:00", + "completed_at": "2020-01-20T09:44:39.000-08:00" + }, + { + "name": "Complete job", + "status": "completed", + "conclusion": "success", + "number": 17, + "started_at": "2020-01-20T09:44:39.000-08:00", + "completed_at": "2020-01-20T09:44:39.000-08:00" + } + ], + "check_run_url": "https://api.github.com/repos/octo-org/octo-repo/check-runs/399444496", + "labels": [ + "self-hosted", + "foo", + "bar" + ], + "runner_id": 1, + "runner_name": "my runner", + "runner_group_id": 2, + "runner_group_name": "my runner group", + "workflow_name": "CI", + "head_branch": "main" + } + ] +} +``` diff --git a/tools/pipeline-witness/docs/github-actions/rest-api/run.md b/tools/pipeline-witness/docs/github-actions/rest-api/run.md new file mode 100644 index 00000000000..bee2db14736 --- /dev/null +++ b/tools/pipeline-witness/docs/github-actions/rest-api/run.md @@ -0,0 +1,241 @@ +# Workflow Run +`/repos/{owner}/{repo}/actions/runs/{run_id}` +```json +{ + "id": 30433642, + "name": "Build", + "node_id": "MDEyOldvcmtmbG93IFJ1bjI2OTI4OQ==", + "check_suite_id": 42, + "check_suite_node_id": "MDEwOkNoZWNrU3VpdGU0Mg==", + "head_branch": "main", + "head_sha": "acb5820ced9479c074f688cc328bf03f341a511d", + "path": ".github/workflows/build.yml@main", + "run_number": 562, + "event": "push", + "display_title": "Update README.md", + "status": "queued", + "conclusion": null, + "workflow_id": 159038, + "url": "https://api.github.com/repos/octo-org/octo-repo/actions/runs/30433642", + "html_url": "https://github.com/octo-org/octo-repo/actions/runs/30433642", + "pull_requests": [], + "created_at": "2020-01-22T19:33:08Z", + "updated_at": "2020-01-22T19:33:08Z", + "actor": { + "login": "octocat", + "id": 1, + "node_id": "MDQ6VXNlcjE=", + "avatar_url": "https://github.com/images/error/octocat_happy.gif", + "gravatar_id": "", + "url": "https://api.github.com/users/octocat", + "html_url": "https://github.com/octocat", + "followers_url": "https://api.github.com/users/octocat/followers", + "following_url": "https://api.github.com/users/octocat/following{/other_user}", + "gists_url": "https://api.github.com/users/octocat/gists{/gist_id}", + "starred_url": "https://api.github.com/users/octocat/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/octocat/subscriptions", + "organizations_url": "https://api.github.com/users/octocat/orgs", + "repos_url": "https://api.github.com/users/octocat/repos", + "events_url": "https://api.github.com/users/octocat/events{/privacy}", + "received_events_url": "https://api.github.com/users/octocat/received_events", + "type": "User", + "site_admin": false + }, + "run_attempt": 1, + "referenced_workflows": [ + { + "path": "octocat/Hello-World/.github/workflows/deploy.yml@main", + "sha": "86e8bc9ecf7d38b1ed2d2cfb8eb87ba9b35b01db", + "ref": "refs/heads/main" + }, + { + "path": "octo-org/octo-repo/.github/workflows/report.yml@v2", + "sha": "79e9790903e1c3373b1a3e3a941d57405478a232", + "ref": "refs/tags/v2" + }, + { + "path": "octo-org/octo-repo/.github/workflows/secure.yml@1595d4b6de6a9e9751fb270a41019ce507d4099e", + "sha": "1595d4b6de6a9e9751fb270a41019ce507d4099e" + } + ], + "run_started_at": "2020-01-22T19:33:08Z", + "triggering_actor": { + "login": "octocat", + "id": 1, + "node_id": "MDQ6VXNlcjE=", + "avatar_url": "https://github.com/images/error/octocat_happy.gif", + "gravatar_id": "", + "url": "https://api.github.com/users/octocat", + "html_url": "https://github.com/octocat", + "followers_url": "https://api.github.com/users/octocat/followers", + "following_url": "https://api.github.com/users/octocat/following{/other_user}", + "gists_url": "https://api.github.com/users/octocat/gists{/gist_id}", + "starred_url": "https://api.github.com/users/octocat/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/octocat/subscriptions", + "organizations_url": "https://api.github.com/users/octocat/orgs", + "repos_url": "https://api.github.com/users/octocat/repos", + "events_url": "https://api.github.com/users/octocat/events{/privacy}", + "received_events_url": "https://api.github.com/users/octocat/received_events", + "type": "User", + "site_admin": false + }, + "jobs_url": "https://api.github.com/repos/octo-org/octo-repo/actions/runs/30433642/jobs", + "logs_url": "https://api.github.com/repos/octo-org/octo-repo/actions/runs/30433642/logs", + "check_suite_url": "https://api.github.com/repos/octo-org/octo-repo/check-suites/414944374", + "artifacts_url": "https://api.github.com/repos/octo-org/octo-repo/actions/runs/30433642/artifacts", + "cancel_url": "https://api.github.com/repos/octo-org/octo-repo/actions/runs/30433642/cancel", + "rerun_url": "https://api.github.com/repos/octo-org/octo-repo/actions/runs/30433642/rerun", + "previous_attempt_url": "https://api.github.com/repos/octo-org/octo-repo/actions/runs/30433642/attempts/1", + "workflow_url": "https://api.github.com/repos/octo-org/octo-repo/actions/workflows/159038", + "head_commit": { + "id": "acb5820ced9479c074f688cc328bf03f341a511d", + "tree_id": "d23f6eedb1e1b9610bbc754ddb5197bfe7271223", + "message": "Create linter.yaml", + "timestamp": "2020-01-22T19:33:05Z", + "author": { + "name": "Octo Cat", + "email": "octocat@github.com" + }, + "committer": { + "name": "GitHub", + "email": "noreply@github.com" + } + }, + "repository": { + "id": 1296269, + "node_id": "MDEwOlJlcG9zaXRvcnkxMjk2MjY5", + "name": "Hello-World", + "full_name": "octocat/Hello-World", + "owner": { + "login": "octocat", + "id": 1, + "node_id": "MDQ6VXNlcjE=", + "avatar_url": "https://github.com/images/error/octocat_happy.gif", + "gravatar_id": "", + "url": "https://api.github.com/users/octocat", + "html_url": "https://github.com/octocat", + "followers_url": "https://api.github.com/users/octocat/followers", + "following_url": "https://api.github.com/users/octocat/following{/other_user}", + "gists_url": "https://api.github.com/users/octocat/gists{/gist_id}", + "starred_url": "https://api.github.com/users/octocat/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/octocat/subscriptions", + "organizations_url": "https://api.github.com/users/octocat/orgs", + "repos_url": "https://api.github.com/users/octocat/repos", + "events_url": "https://api.github.com/users/octocat/events{/privacy}", + "received_events_url": "https://api.github.com/users/octocat/received_events", + "type": "User", + "site_admin": false + }, + "private": false, + "html_url": "https://github.com/octocat/Hello-World", + "description": "This your first repo!", + "fork": false, + "url": "https://api.github.com/repos/octocat/Hello-World", + "archive_url": "https://api.github.com/repos/octocat/Hello-World/{archive_format}{/ref}", + "assignees_url": "https://api.github.com/repos/octocat/Hello-World/assignees{/user}", + "blobs_url": "https://api.github.com/repos/octocat/Hello-World/git/blobs{/sha}", + "branches_url": "https://api.github.com/repos/octocat/Hello-World/branches{/branch}", + "collaborators_url": "https://api.github.com/repos/octocat/Hello-World/collaborators{/collaborator}", + "comments_url": "https://api.github.com/repos/octocat/Hello-World/comments{/number}", + "commits_url": "https://api.github.com/repos/octocat/Hello-World/commits{/sha}", + "compare_url": "https://api.github.com/repos/octocat/Hello-World/compare/{base}...{head}", + "contents_url": "https://api.github.com/repos/octocat/Hello-World/contents/{+path}", + "contributors_url": "https://api.github.com/repos/octocat/Hello-World/contributors", + "deployments_url": "https://api.github.com/repos/octocat/Hello-World/deployments", + "downloads_url": "https://api.github.com/repos/octocat/Hello-World/downloads", + "events_url": "https://api.github.com/repos/octocat/Hello-World/events", + "forks_url": "https://api.github.com/repos/octocat/Hello-World/forks", + "git_commits_url": "https://api.github.com/repos/octocat/Hello-World/git/commits{/sha}", + "git_refs_url": "https://api.github.com/repos/octocat/Hello-World/git/refs{/sha}", + "git_tags_url": "https://api.github.com/repos/octocat/Hello-World/git/tags{/sha}", + "git_url": "git:github.com/octocat/Hello-World.git", + "issue_comment_url": "https://api.github.com/repos/octocat/Hello-World/issues/comments{/number}", + "issue_events_url": "https://api.github.com/repos/octocat/Hello-World/issues/events{/number}", + "issues_url": "https://api.github.com/repos/octocat/Hello-World/issues{/number}", + "keys_url": "https://api.github.com/repos/octocat/Hello-World/keys{/key_id}", + "labels_url": "https://api.github.com/repos/octocat/Hello-World/labels{/name}", + "languages_url": "https://api.github.com/repos/octocat/Hello-World/languages", + "merges_url": "https://api.github.com/repos/octocat/Hello-World/merges", + "milestones_url": "https://api.github.com/repos/octocat/Hello-World/milestones{/number}", + "notifications_url": "https://api.github.com/repos/octocat/Hello-World/notifications{?since,all,participating}", + "pulls_url": "https://api.github.com/repos/octocat/Hello-World/pulls{/number}", + "releases_url": "https://api.github.com/repos/octocat/Hello-World/releases{/id}", + "ssh_url": "git@github.com:octocat/Hello-World.git", + "stargazers_url": "https://api.github.com/repos/octocat/Hello-World/stargazers", + "statuses_url": "https://api.github.com/repos/octocat/Hello-World/statuses/{sha}", + "subscribers_url": "https://api.github.com/repos/octocat/Hello-World/subscribers", + "subscription_url": "https://api.github.com/repos/octocat/Hello-World/subscription", + "tags_url": "https://api.github.com/repos/octocat/Hello-World/tags", + "teams_url": "https://api.github.com/repos/octocat/Hello-World/teams", + "trees_url": "https://api.github.com/repos/octocat/Hello-World/git/trees{/sha}", + "hooks_url": "http://api.github.com/repos/octocat/Hello-World/hooks" + }, + "head_repository": { + "id": 217723378, + "node_id": "MDEwOlJlcG9zaXRvcnkyMTc3MjMzNzg=", + "name": "octo-repo", + "full_name": "octo-org/octo-repo", + "private": true, + "owner": { + "login": "octocat", + "id": 1, + "node_id": "MDQ6VXNlcjE=", + "avatar_url": "https://github.com/images/error/octocat_happy.gif", + "gravatar_id": "", + "url": "https://api.github.com/users/octocat", + "html_url": "https://github.com/octocat", + "followers_url": "https://api.github.com/users/octocat/followers", + "following_url": "https://api.github.com/users/octocat/following{/other_user}", + "gists_url": "https://api.github.com/users/octocat/gists{/gist_id}", + "starred_url": "https://api.github.com/users/octocat/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/octocat/subscriptions", + "organizations_url": "https://api.github.com/users/octocat/orgs", + "repos_url": "https://api.github.com/users/octocat/repos", + "events_url": "https://api.github.com/users/octocat/events{/privacy}", + "received_events_url": "https://api.github.com/users/octocat/received_events", + "type": "User", + "site_admin": false + }, + "html_url": "https://github.com/octo-org/octo-repo", + "description": null, + "fork": false, + "url": "https://api.github.com/repos/octo-org/octo-repo", + "forks_url": "https://api.github.com/repos/octo-org/octo-repo/forks", + "keys_url": "https://api.github.com/repos/octo-org/octo-repo/keys{/key_id}", + "collaborators_url": "https://api.github.com/repos/octo-org/octo-repo/collaborators{/collaborator}", + "teams_url": "https://api.github.com/repos/octo-org/octo-repo/teams", + "hooks_url": "https://api.github.com/repos/octo-org/octo-repo/hooks", + "issue_events_url": "https://api.github.com/repos/octo-org/octo-repo/issues/events{/number}", + "events_url": "https://api.github.com/repos/octo-org/octo-repo/events", + "assignees_url": "https://api.github.com/repos/octo-org/octo-repo/assignees{/user}", + "branches_url": "https://api.github.com/repos/octo-org/octo-repo/branches{/branch}", + "tags_url": "https://api.github.com/repos/octo-org/octo-repo/tags", + "blobs_url": "https://api.github.com/repos/octo-org/octo-repo/git/blobs{/sha}", + "git_tags_url": "https://api.github.com/repos/octo-org/octo-repo/git/tags{/sha}", + "git_refs_url": "https://api.github.com/repos/octo-org/octo-repo/git/refs{/sha}", + "trees_url": "https://api.github.com/repos/octo-org/octo-repo/git/trees{/sha}", + "statuses_url": "https://api.github.com/repos/octo-org/octo-repo/statuses/{sha}", + "languages_url": "https://api.github.com/repos/octo-org/octo-repo/languages", + "stargazers_url": "https://api.github.com/repos/octo-org/octo-repo/stargazers", + "contributors_url": "https://api.github.com/repos/octo-org/octo-repo/contributors", + "subscribers_url": "https://api.github.com/repos/octo-org/octo-repo/subscribers", + "subscription_url": "https://api.github.com/repos/octo-org/octo-repo/subscription", + "commits_url": "https://api.github.com/repos/octo-org/octo-repo/commits{/sha}", + "git_commits_url": "https://api.github.com/repos/octo-org/octo-repo/git/commits{/sha}", + "comments_url": "https://api.github.com/repos/octo-org/octo-repo/comments{/number}", + "issue_comment_url": "https://api.github.com/repos/octo-org/octo-repo/issues/comments{/number}", + "contents_url": "https://api.github.com/repos/octo-org/octo-repo/contents/{+path}", + "compare_url": "https://api.github.com/repos/octo-org/octo-repo/compare/{base}...{head}", + "merges_url": "https://api.github.com/repos/octo-org/octo-repo/merges", + "archive_url": "https://api.github.com/repos/octo-org/octo-repo/{archive_format}{/ref}", + "downloads_url": "https://api.github.com/repos/octo-org/octo-repo/downloads", + "issues_url": "https://api.github.com/repos/octo-org/octo-repo/issues{/number}", + "pulls_url": "https://api.github.com/repos/octo-org/octo-repo/pulls{/number}", + "milestones_url": "https://api.github.com/repos/octo-org/octo-repo/milestones{/number}", + "notifications_url": "https://api.github.com/repos/octo-org/octo-repo/notifications{?since,all,participating}", + "labels_url": "https://api.github.com/repos/octo-org/octo-repo/labels{/name}", + "releases_url": "https://api.github.com/repos/octo-org/octo-repo/releases{/id}", + "deployments_url": "https://api.github.com/repos/octo-org/octo-repo/deployments" + } +} +``` diff --git a/tools/pipeline-witness/docs/github-actions/rest-api/workflow.md b/tools/pipeline-witness/docs/github-actions/rest-api/workflow.md new file mode 100644 index 00000000000..fde2bccf38e --- /dev/null +++ b/tools/pipeline-witness/docs/github-actions/rest-api/workflow.md @@ -0,0 +1,16 @@ +# Workflow +`/repos/{owner}/{repo}/actions/workflows/{workflow_id}` +```json +{ + "id": 161335, + "node_id": "MDg6V29ya2Zsb3cxNjEzMzU=", + "name": "CI", + "path": ".github/workflows/blank.yaml", + "state": "active", + "created_at": "2020-01-08T23:48:37.000-08:00", + "updated_at": "2020-01-08T23:50:21.000-08:00", + "url": "https://api.github.com/repos/octo-org/octo-repo/actions/workflows/161335", + "html_url": "https://github.com/octo-org/octo-repo/blob/master/.github/workflows/161335", + "badge_url": "https://github.com/octo-org/octo-repo/workflows/CI/badge.svg" +} +``` diff --git a/tools/pipeline-witness/infrastructure/Assign-StoragePermissions.ps1 b/tools/pipeline-witness/infrastructure/Assign-DevPermissions.ps1 similarity index 84% rename from tools/pipeline-witness/infrastructure/Assign-StoragePermissions.ps1 rename to tools/pipeline-witness/infrastructure/Assign-DevPermissions.ps1 index 1afb3b6a5db..3f5bcbe41e3 100644 --- a/tools/pipeline-witness/infrastructure/Assign-StoragePermissions.ps1 +++ b/tools/pipeline-witness/infrastructure/Assign-DevPermissions.ps1 @@ -24,12 +24,14 @@ try { Write-Host "Reading parameters from $parametersFile" $parameters = (Get-Content -Path $parametersFile -Raw | ConvertFrom-Json).parameters $appResourceGroupName = $parameters.appResourceGroupName.value + $keyVaultName = $parameters.keyVaultName.value $appStorageAccountName = $parameters.appStorageAccountName.value $cosmosAccountName = $parameters.cosmosAccountName.value $logsResourceGroupName = $parameters.logsResourceGroupName.value $logsStorageAccountName = $parameters.logsStorageAccountName.value Write-Host "Adding Azure SDK Engineering System Team RBAC access to storage resources:`n" + ` + " Vault: $appResourceGroupName/$keyVaultName`n" + ` " Blob: $logsResourceGroupName/$logsStorageAccountName`n" + ` " Queue: $appResourceGroupName/$appStorageAccountName`n" + ` " Cosmos: $appResourceGroupName/$cosmosAccountName`n" @@ -46,6 +48,16 @@ try { exit 1 } + Write-Host "Granting 'Key Vault Administrator' access to $appStorageAccountName/$keyVaultName" + $scope = "/subscriptions/$subscriptionId/resourceGroups/$appStorageAccountName/providers/Microsoft.KeyVault/vaults/$keyVaultName" + $output = Invoke "az role assignment create --assignee '$azAdGroupId' --role 'Key Vault Administrator' --scope '$scope' --output none" + + if ($LASTEXITCODE -ne 0) { + Write-Output $output + Write-Error "Failed to grant access" + exit 1 + } + Write-Host "Granting 'Storage Blob Data Contributor' access to $logsResourceGroupName/$logsStorageAccountName" $scope = "/subscriptions/$subscriptionId/resourceGroups/$logsResourceGroupName/providers/Microsoft.Storage/storageAccounts/$logsStorageAccountName" $output = Invoke "az role assignment create --assignee '$azAdGroupId' --role 'Storage Blob Data Contributor' --scope '$scope' --output none" diff --git a/tools/pipeline-witness/infrastructure/bicep/appResourceGroup.bicep b/tools/pipeline-witness/infrastructure/bicep/appResourceGroup.bicep index 1ad73a302ac..12b88b1ac71 100644 --- a/tools/pipeline-witness/infrastructure/bicep/appResourceGroup.bicep +++ b/tools/pipeline-witness/infrastructure/bicep/appResourceGroup.bicep @@ -5,6 +5,7 @@ param appServicePlanName string param appStorageAccountName string param aspEnvironment string param cosmosAccountName string +param keyVaultName string param location string param vnetPrefix string param subnetPrefix string @@ -89,7 +90,8 @@ resource webApp 'Microsoft.Web/sites@2022-03-01' = { properties: { serverFarmId: appServicePlan.id siteConfig: { - linuxFxVersion: 'DOTNETCORE|6.0' + linuxFxVersion: 'DOTNETCORE|8.0' + alwaysOn: true } httpsOnly: true virtualNetworkSubnetId: subnet.id @@ -167,6 +169,25 @@ resource appStorageAccount 'Microsoft.Storage/storageAccounts@2021-09-01' = { resource buildCompletedQueue 'queues' = { name: 'azurepipelines-build-completed' } + + resource gitHubActionsQueue 'queues' = { + name: 'github-actionrun-completed' + } + } +} + +resource keyVault 'Microsoft.KeyVault/vaults@2023-07-01' = { + location: location + name: keyVaultName + properties: { + tenantId: subscription().tenantId + sku: { + family: 'A' + name: 'standard' + } + enableSoftDelete: true + softDeleteRetentionInDays: 90 + enableRbacAuthorization: true } } @@ -313,10 +334,29 @@ resource locksContainer 'Microsoft.DocumentDB/databaseAccounts/sqlDatabases/cont } } +// Assign Key Vault Secrets User role for the Web App on the Key Vault +resource secretsUserRoleDefinition 'Microsoft.Authorization/roleDefinitions@2018-01-01-preview' existing = { + scope: subscription() + // This is the Key Vault Reader role, which is the minimum role permission we can give. + // See https://learn.microsoft.com/en-us/azure/role-based-access-control/built-in-roles#key-vault + name: '4633458b-17de-408a-b874-0445c86b69e6' +} + +resource vaultRoleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(secretsUserRoleDefinition.id, webAppName, keyVault.id) + scope: keyVault + properties:{ + principalId: webApp.identity.principalId + roleDefinitionId: secretsUserRoleDefinition.id + description: 'Key Vault Secrets User for PipelineWitness' + } +} + + // Assign Storage Queue Data Contributor role for the Web App on the Queue Storage Account resource queueContributorRoleDefinition 'Microsoft.Authorization/roleDefinitions@2018-01-01-preview' existing = { scope: subscription() - // This is the Storage Blob Data Contributor role, which is the minimum role permission we can give. + // This is the Storage Queue Data Contributor role, which is the minimum role permission we can give. // See https://learn.microsoft.com/en-us/azure/role-based-access-control/built-in-roles#storage name: '974c5e8b-45b9-4653-ba55-5f855dd0fb88' } diff --git a/tools/pipeline-witness/infrastructure/bicep/bicepconfig.json b/tools/pipeline-witness/infrastructure/bicep/bicepconfig.json new file mode 100644 index 00000000000..32bd651f57e --- /dev/null +++ b/tools/pipeline-witness/infrastructure/bicep/bicepconfig.json @@ -0,0 +1,4 @@ +{ + "experimentalFeaturesEnabled": { + } + } \ No newline at end of file diff --git a/tools/pipeline-witness/infrastructure/bicep/logsResourceGroup.bicep b/tools/pipeline-witness/infrastructure/bicep/logsResourceGroup.bicep index e2c85460c5d..ac51b79e33e 100644 --- a/tools/pipeline-witness/infrastructure/bicep/logsResourceGroup.bicep +++ b/tools/pipeline-witness/infrastructure/bicep/logsResourceGroup.bicep @@ -1,46 +1,13 @@ param location string param logsStorageAccountName string +param devOpsEventHubNamespaceName string +param gitHubEventHubNamespaceName string param kustoClusterName string param kustoDatabaseName string param webAppName string param subnetId string param appIdentityPrincipalId string -var tables = [ - { - name: 'Build' - container: 'builds' - } - { - name: 'BuildDefinition' - container: 'builddefinitions' - } - { - name: 'BuildFailure' - container: 'buildfailures' - } - { - name: 'BuildLogLine' - container: 'buildloglines' - } - { - name: 'BuildTimelineRecord' - container: 'buildtimelinerecords' - } - { - name: 'PipelineOwner' - container: 'pipelineowners' - } - { - name: 'TestRun' - container: 'testruns' - } - { - name: 'TestRunResult' - container: 'testrunresults' - } -] - var kustoScript = loadTextContent('../artifacts/merged.kql') // Storage Account for output blobs @@ -104,19 +71,6 @@ resource logsStorageAccount 'Microsoft.Storage/storageAccounts@2021-09-01' = { } } -resource containers 'Microsoft.Storage/storageAccounts/blobServices/containers@2021-09-01' = [for table in tables: { - parent: logsStorageAccount::blobServices - name: table.container - properties: { - immutableStorageWithVersioning: { - enabled: false - } - defaultEncryptionScope: '$account-encryption-key' - denyEncryptionScopeOverride: false - publicAccess: 'None' - } -}] - // Event Grid resource eventGridTopic 'Microsoft.EventGrid/systemTopics@2022-06-15' = { name: logsStorageAccountName @@ -128,8 +82,28 @@ resource eventGridTopic 'Microsoft.EventGrid/systemTopics@2022-06-15' = { } // Event Hub -resource eventHubNamespace 'Microsoft.EventHub/namespaces@2022-01-01-preview' = { - name: logsStorageAccountName +resource devOpsEventHubNamespace 'Microsoft.EventHub/namespaces@2022-01-01-preview' = { + name: devOpsEventHubNamespaceName + location: location + sku: { + name: 'Standard' + tier: 'Standard' + capacity: 1 + } + properties: { + minimumTlsVersion: '1.0' + publicNetworkAccess: 'Enabled' + disableLocalAuth: false + zoneRedundant: false + isAutoInflateEnabled: false + maximumThroughputUnits: 0 + kafkaEnabled: true + } +} + +// Event Hub +resource gitHubEventHubNamespace 'Microsoft.EventHub/namespaces@2022-01-01-preview' = { + name: gitHubEventHubNamespaceName location: location sku: { name: 'Standard' @@ -209,60 +183,6 @@ resource kustoScriptInvocation 'Microsoft.Kusto/clusters/databases/scripts@2022- } } -resource eventHubs 'Microsoft.EventHub/namespaces/eventhubs@2022-01-01-preview' = [for (table, i) in tables: { - parent: eventHubNamespace - name: table.container - properties: { - messageRetentionInDays: 7 - partitionCount: 8 - status: 'Active' - } -}] - -resource eventGridSubscriptions 'Microsoft.EventGrid/systemTopics/eventSubscriptions@2022-06-15' = [for (table, i) in tables: { - parent: eventGridTopic - name: table.container - properties: { - destination: { - properties: { - resourceId: eventHubs[i].id - } - endpointType: 'EventHub' - } - filter: { - subjectBeginsWith: '/blobServices/default/containers/${table.container}' - includedEventTypes: [ - 'Microsoft.Storage.BlobCreated' - ] - } - eventDeliverySchema: 'EventGridSchema' - retryPolicy: { - maxDeliveryAttempts: 30 - eventTimeToLiveInMinutes: 1440 - } - } -}] - -resource kustoDataConnections 'Microsoft.Kusto/Clusters/Databases/DataConnections@2022-02-01' = [for (table, i) in tables: { - parent: kustoCluster::database - name: '${kustoDatabaseName}-${table.container}' - location: location - kind: 'EventGrid' - properties: { - ignoreFirstRecord: false - storageAccountResourceId: logsStorageAccount.id - eventHubResourceId: eventHubs[i].id - consumerGroup: '$Default' - tableName: table.name - mappingRuleName: '${table.name}_mapping' - dataFormat: 'JSON' - blobStorageEventType: 'Microsoft.Storage.BlobCreated' - databaseRouting: 'Single' - managedIdentityResourceId: kustoCluster.id - } - dependsOn: [ kustoScriptInvocation ] -}] - // Assign roles to the Kusto cluster and App Service resource blobContributorRoleDefinition 'Microsoft.Authorization/roleDefinitions@2018-01-01-preview' existing = { scope: subscription() @@ -300,12 +220,102 @@ resource eventHubsDataReceiverRoleDefinition 'Microsoft.Authorization/roleDefini name: 'a638d3c7-ab3a-418d-83e6-5f17a39d4fde' } -resource kustoEventHubsAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { - name: guid(eventHubsDataReceiverRoleDefinition.id, kustoClusterName, eventHubNamespace.id) - scope: eventHubNamespace +resource devOpsKustoEventHubsAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(eventHubsDataReceiverRoleDefinition.id, kustoClusterName, devOpsEventHubNamespace.id) + scope: devOpsEventHubNamespace properties:{ principalId: kustoCluster.identity.principalId roleDefinitionId: eventHubsDataReceiverRoleDefinition.id description: 'Blob Contributor for Kusto ingestion' } } + +resource gitHubKustoEventHubsAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(eventHubsDataReceiverRoleDefinition.id, kustoClusterName, gitHubEventHubNamespace.id) + scope: gitHubEventHubNamespace + properties:{ + principalId: kustoCluster.identity.principalId + roleDefinitionId: eventHubsDataReceiverRoleDefinition.id + description: 'Blob Contributor for Kusto ingestion' + } +} + +module devOpsTables 'tableResources.bicep' = { + name: 'devOpsTables' + scope: resourceGroup() + dependsOn:[ kustoScriptInvocation ] + params: { + location: location + logsStorageAccountName: logsStorageAccountName + eventHubNamespaceName: devOpsEventHubNamespace.name + eventGridTopicName: eventGridTopic.name + kustoClusterName: kustoCluster.name + kustoDatabaseName: kustoCluster::database.name + tables: [ + { + name: 'Build' + container: 'builds' + } + { + name: 'BuildDefinition' + container: 'builddefinitions' + } + { + name: 'BuildFailure' + container: 'buildfailures' + } + { + name: 'BuildLogLine' + container: 'buildloglines' + } + { + name: 'BuildTimelineRecord' + container: 'buildtimelinerecords' + } + { + name: 'PipelineOwner' + container: 'pipelineowners' + } + { + name: 'TestRun' + container: 'testruns' + } + { + name: 'TestRunResult' + container: 'testrunresults' + } + ] + } +} + +module gitHubTables 'tableResources.bicep' = { + name: 'gitHubTables' + scope: resourceGroup() + dependsOn:[ kustoScriptInvocation ] + params: { + location: location + logsStorageAccountName: logsStorageAccountName + eventHubNamespaceName: gitHubEventHubNamespace.name + eventGridTopicName: eventGridTopic.name + kustoClusterName: kustoCluster.name + kustoDatabaseName: kustoCluster::database.name + tables: [ + { + name: 'GitHubActionsRun' + container: 'githubactionsruns' + } + { + name: 'GitHubActionsJob' + container: 'githubactionsjobs' + } + { + name: 'GitHubActionsStep' + container: 'githubactionssteps' + } + { + name: 'GitHubActionsLogLine' + container: 'githubactionslogs' + } + ] + } +} diff --git a/tools/pipeline-witness/infrastructure/bicep/resourceGroups.bicep b/tools/pipeline-witness/infrastructure/bicep/main.bicep similarity index 86% rename from tools/pipeline-witness/infrastructure/bicep/resourceGroups.bicep rename to tools/pipeline-witness/infrastructure/bicep/main.bicep index 845f495c162..dac18925a1a 100644 --- a/tools/pipeline-witness/infrastructure/bicep/resourceGroups.bicep +++ b/tools/pipeline-witness/infrastructure/bicep/main.bicep @@ -12,6 +12,9 @@ param subnetPrefix string param cosmosAccountName string param appStorageAccountName string param aspEnvironment string +param keyVaultName string +param devOpsEventHubNamespaceName string +param gitHubEventHubNamespaceName string param logsResourceGroupName string param logsStorageAccountName string @@ -33,6 +36,7 @@ module pipelineWitness 'appResourceGroup.bicep' = { vnetPrefix: vnetPrefix subnetPrefix: subnetPrefix webAppName: webAppName + keyVaultName: keyVaultName cosmosAccountName: cosmosAccountName appStorageAccountName: appStorageAccountName aspEnvironment: aspEnvironment @@ -58,6 +62,8 @@ module pipelineLogs 'logsResourceGroup.bicep' = { kustoClusterName: kustoClusterName kustoDatabaseName: kustoDatabaseName webAppName: webAppName + devOpsEventHubNamespaceName: devOpsEventHubNamespaceName + gitHubEventHubNamespaceName: gitHubEventHubNamespaceName appIdentityPrincipalId: pipelineWitness.outputs.appIdentityPrincipalId subnetId: pipelineWitness.outputs.subnetId } diff --git a/tools/pipeline-witness/infrastructure/bicep/parameters.production.json b/tools/pipeline-witness/infrastructure/bicep/parameters.production.json index e20b6bbe45a..f6c5c3f1a10 100644 --- a/tools/pipeline-witness/infrastructure/bicep/parameters.production.json +++ b/tools/pipeline-witness/infrastructure/bicep/parameters.production.json @@ -14,6 +14,9 @@ "webAppName": { "value": "pipelinewitnessprod-app" }, + "keyVaultName": { + "value": "pipelinewitnessprod" + }, "appServicePlanName": { "value": "ASP-pipelinewitnessprod-b6fe" }, @@ -29,6 +32,12 @@ "logsStorageAccountName": { "value": "azsdkengsyspipelinelogs" }, + "devOpsEventHubNamespaceName": { + "value": "azsdkengsyspipelinelogs" + }, + "gitHubEventHubNamespaceName": { + "value": "azsdkengsysgithublogs" + }, "kustoClusterName": { "value": "azsdkengsys" }, diff --git a/tools/pipeline-witness/infrastructure/bicep/parameters.staging.json b/tools/pipeline-witness/infrastructure/bicep/parameters.staging.json index 32342d8c04a..907851c938c 100644 --- a/tools/pipeline-witness/infrastructure/bicep/parameters.staging.json +++ b/tools/pipeline-witness/infrastructure/bicep/parameters.staging.json @@ -14,6 +14,9 @@ "webAppName": { "value": "pipelinewitnessstaging-app" }, + "keyVaultName": { + "value": "pipelinewitnessstaging" + }, "appServicePlanName": { "value": "ASP-pipelinewitnessstaging-a2b5" }, @@ -29,6 +32,12 @@ "logsStorageAccountName": { "value": "pipelinelogsstaging" }, + "devOpsEventHubNamespaceName": { + "value": "pipelinelogsstaging" + }, + "gitHubEventHubNamespaceName": { + "value": "githublogsstaging" + }, "kustoClusterName": { "value": "azsdkengsys" }, diff --git a/tools/pipeline-witness/infrastructure/bicep/parameters.test.json b/tools/pipeline-witness/infrastructure/bicep/parameters.test.json index b7c356abe23..60f42045d1c 100644 --- a/tools/pipeline-witness/infrastructure/bicep/parameters.test.json +++ b/tools/pipeline-witness/infrastructure/bicep/parameters.test.json @@ -14,6 +14,9 @@ "webAppName": { "value": "pipelinewitnesstest-app" }, + "keyVaultName": { + "value": "pipelinewitnesstest" + }, "appServicePlanName": { "value": "ASP-pipelinewitnesstest-ab12" }, @@ -29,6 +32,12 @@ "logsStorageAccountName": { "value": "pipelinelogstest" }, + "devOpsEventHubNamespaceName": { + "value": "pipelinelogstest" + }, + "gitHubEventHubNamespaceName": { + "value": "githublogstest" + }, "kustoClusterName": { "value": "azsdkengsystest" }, diff --git a/tools/pipeline-witness/infrastructure/bicep/tableResources.bicep b/tools/pipeline-witness/infrastructure/bicep/tableResources.bicep new file mode 100644 index 00000000000..6228bde1e0a --- /dev/null +++ b/tools/pipeline-witness/infrastructure/bicep/tableResources.bicep @@ -0,0 +1,95 @@ +param location string +param logsStorageAccountName string +param eventHubNamespaceName string +param eventGridTopicName string +param kustoClusterName string +param kustoDatabaseName string +param tables array + +resource logsStorageAccount 'Microsoft.Storage/storageAccounts@2021-09-01' existing = { + name: logsStorageAccountName + resource blobServices 'blobServices' = { + name: 'default' + } +} + +resource eventHubNamespace 'Microsoft.EventHub/namespaces@2022-01-01-preview' existing = { + name: eventHubNamespaceName +} + +resource eventGridTopic 'Microsoft.EventGrid/systemTopics@2022-06-15' existing = { + name: eventGridTopicName +} + +resource kustoCluster 'Microsoft.Kusto/Clusters@2022-02-01' existing = { + name: kustoClusterName + resource database 'Databases' existing = { + name: kustoDatabaseName + } +} + +resource containers 'Microsoft.Storage/storageAccounts/blobServices/containers@2021-09-01' = [for table in tables: { + parent: logsStorageAccount::blobServices + name: table.container + properties: { + immutableStorageWithVersioning: { + enabled: false + } + defaultEncryptionScope: '$account-encryption-key' + denyEncryptionScopeOverride: false + publicAccess: 'None' + } +}] + +resource eventHubs 'Microsoft.EventHub/namespaces/eventhubs@2022-01-01-preview' = [for (table, i) in tables: { + parent: eventHubNamespace + name: table.container + properties: { + messageRetentionInDays: 7 + partitionCount: 8 + status: 'Active' + } +}] + +resource eventGridSubscriptions 'Microsoft.EventGrid/systemTopics/eventSubscriptions@2022-06-15' = [for (table, i) in tables: { + parent: eventGridTopic + name: table.container + properties: { + destination: { + properties: { + resourceId: eventHubs[i].id + } + endpointType: 'EventHub' + } + filter: { + subjectBeginsWith: '/blobServices/default/containers/${table.container}' + includedEventTypes: [ + 'Microsoft.Storage.BlobCreated' + ] + } + eventDeliverySchema: 'EventGridSchema' + retryPolicy: { + maxDeliveryAttempts: 30 + eventTimeToLiveInMinutes: 1440 + } + } +}] + +resource kustoDataConnections 'Microsoft.Kusto/Clusters/Databases/DataConnections@2022-02-01' = [for (table, i) in tables: { + parent: kustoCluster::database + name: '${kustoDatabaseName}-${table.container}' + location: location + kind: 'EventGrid' + properties: { + ignoreFirstRecord: false + storageAccountResourceId: logsStorageAccount.id + eventHubResourceId: eventHubs[i].id + consumerGroup: '$Default' + tableName: table.name + mappingRuleName: '${table.name}_mapping' + dataFormat: 'JSON' + blobStorageEventType: 'Microsoft.Storage.BlobCreated' + databaseRouting: 'Single' + managedIdentityResourceId: kustoCluster.id + } +}] diff --git a/tools/pipeline-witness/infrastructure/build.ps1 b/tools/pipeline-witness/infrastructure/build.ps1 index 47af9ea0ddb..3ce059adfe5 100644 --- a/tools/pipeline-witness/infrastructure/build.ps1 +++ b/tools/pipeline-witness/infrastructure/build.ps1 @@ -13,7 +13,7 @@ try { exit 1 } - az bicep build --file "./bicep/resourceGroup.bicep" --outdir "./artifacts" + az bicep build --file "./bicep/main.bicep" --outdir "./artifacts" if($?) { Write-Host "Built Bicep files" } else { diff --git a/tools/pipeline-witness/infrastructure/deploy.ps1 b/tools/pipeline-witness/infrastructure/deploy.ps1 index 5903e241b5f..31bfb16d65a 100644 --- a/tools/pipeline-witness/infrastructure/deploy.ps1 +++ b/tools/pipeline-witness/infrastructure/deploy.ps1 @@ -10,16 +10,14 @@ param( [switch]$removeRoleAssignments ) -function Invoke([string]$command) { - Write-Host "> $command" - Invoke-Expression $command -} +$repoRoot = Resolve-Path "$PSScriptRoot/../../.." +. "$repoRoot/eng/common/scripts/Helpers/CommandInvocation-Helpers.ps1" function RemoveStorageRoleAssignments($subscriptionId, $resourceGroup, $resourceName) { $scope = "/subscriptions/$subscriptionId/resourceGroups/$resourceGroup/providers/Microsoft.Storage/storageAccounts/$resourceName" Write-Host "Removing role assignments from $resourceGroup/$resourceName" - $existingAssignments = az role assignment list --scope $scope --output json | ConvertFrom-Json + $existingAssignments = Invoke-LoggedCommand "az role assignment list --scope $scope --output json" | ConvertFrom-Json if ($existingAssignments.Count -eq 0) { Write-Host " No role assignments found" @@ -28,13 +26,13 @@ function RemoveStorageRoleAssignments($subscriptionId, $resourceGroup, $resource foreach ($assignment in $existingAssignments) { Write-Host " Removing role assignment for '$($assignment.principalName)' in role '$($assignment.roleDefinitionName)'" - Invoke "az role assignment delete --assignee '$($assignment.principalId)' --role '$($assignment.roleDefinitionId)' --scope '$scope' --yes" + Invoke-LoggedCommand "az role assignment delete --assignee '$($assignment.principalId)' --role '$($assignment.roleDefinitionId)' --scope '$scope' --yes" } } function RemoveCosmosRoleAssignments($subscriptionId, $resourceGroup, $resourceName) { Write-Host "Removing cosmos role assignments from $resourceGroup/$resourceName" - $existingAssignments = az cosmosdb sql role assignment list --account-name $resourceName --resource-group $resourceGroup --output json | ConvertFrom-Json + $existingAssignments = Invoke-LoggedCommand "az cosmosdb sql role assignment list --account-name $resourceName --resource-group $resourceGroup --output json" | ConvertFrom-Json if ($existingAssignments.Count -eq 0) { Write-Host " No role assignments found" @@ -43,7 +41,7 @@ function RemoveCosmosRoleAssignments($subscriptionId, $resourceGroup, $resourceN foreach ($assignment in $existingAssignments) { Write-Host " Removing cosmos role assignment $($assignment.name)" - Invoke "az cosmosdb sql role assignment delete --account-name '$cosmosAccountName' --resource-group '$appResourceGroupName' --role-assignment-id '$($assignment.id)' --yes" + Invoke-LoggedCommand "az cosmosdb sql role assignment delete --account-name '$cosmosAccountName' --resource-group '$appResourceGroupName' --role-assignment-id '$($assignment.id)' --yes" } } @@ -61,8 +59,8 @@ try { $logsResourceGroupName = $parameters.logsResourceGroupName.value $logsStorageAccountName = $parameters.logsStorageAccountName.value - Invoke "az account set --subscription '$subscriptionName'" - $subscriptionId = az account show --query id -o tsv + Invoke-LoggedCommand "az account set --subscription '$subscriptionName'" + $subscriptionId = Invoke "az account show --query id -o tsv" ./Merge-KustoScripts.ps1 -OutputPath "./artifacts/merged.kql" if ($?) { @@ -85,7 +83,7 @@ try { RemoveCosmosRoleAssignments $subscriptionId $appResourceGroupName $cosmosAccountName } - Invoke "az deployment sub create --template-file './bicep/resourceGroups.bicep' --parameters '$parametersFile' --location '$location' --name '$deploymentName' --output none" + Invoke-LoggedCommand "az deployment sub create --template-file './bicep/main.bicep' --parameters '$parametersFile' --location '$location' --name '$deploymentName' --output none" if ($LASTEXITCODE -ne 0) { Write-Error "Failed to deploy resource groups" exit 1 @@ -93,4 +91,4 @@ try { } finally { Pop-Location -} \ No newline at end of file +} diff --git a/tools/pipeline-witness/infrastructure/kusto/tables/GitHubActionsJob.kql b/tools/pipeline-witness/infrastructure/kusto/tables/GitHubActionsJob.kql new file mode 100644 index 00000000000..8adf4041a37 --- /dev/null +++ b/tools/pipeline-witness/infrastructure/kusto/tables/GitHubActionsJob.kql @@ -0,0 +1,47 @@ +.create-merge table GitHubActionsJob ( + Repository: string, + Workflow: string, + WorkflowId: long, + RunId: long, + JobId: long, + Name: string, + Status: string, + Conclusion: string, + CreatedAt: datetime, + StartedAt: datetime, + CompletedAt: datetime, + NodeId: string, + HeadSha: string, + Labels: dynamic, + RunnerId: long, + RunnerName: string, + RunnerGroupId: long, + RunnerGroupName: string, + HtmlUrl: string, + CheckRunUrl: string, + EtlIngestDate: datetime +) with (folder='', docstring='') + +.create-or-alter table GitHubActionsJob ingestion json mapping 'GitHubActionsJob_mapping' ```[ + { "column": "Repository", "path": "$['repository']" }, + { "column": "Workflow", "path": "$['workflow']" }, + { "column": "WorkflowId", "path": "$['workflowId']" }, + { "column": "RunId", "path": "$['runId']" }, + { "column": "JobId", "path": "$['jobId']" }, + { "column": "Name", "path": "$['name']" }, + { "column": "Status", "path": "$['status']" }, + { "column": "Conclusion", "path": "$['conclusion']" }, + { "column": "CreatedAt", "path": "$['createdAt']" }, + { "column": "StartedAt", "path": "$['startedAt']" }, + { "column": "CompletedAt", "path": "$['completedAt']" }, + { "column": "NodeId", "path": "$['nodeId']" }, + { "column": "HeadSha", "path": "$['headSha']" }, + { "column": "Labels", "path": "$['labels']" }, + { "column": "RunnerId", "path": "$['runnerId']" }, + { "column": "RunnerName", "path": "$['runnerName']" }, + { "column": "RunnerGroupId", "path": "$['runnerGroupId']" }, + { "column": "RunnerGroupName", "path": "$['runnerGroupName']" }, + { "column": "HtmlUrl", "path": "$['htmlUrl']" }, + { "column": "CheckRunUrl", "path": "$['checkRunUrl']" }, + { "column": "EtlIngestDate", "path": "$['etlIngestDate']" } +]``` diff --git a/tools/pipeline-witness/infrastructure/kusto/tables/GitHubActionsLogLine.kql b/tools/pipeline-witness/infrastructure/kusto/tables/GitHubActionsLogLine.kql new file mode 100644 index 00000000000..1778ea33aea --- /dev/null +++ b/tools/pipeline-witness/infrastructure/kusto/tables/GitHubActionsLogLine.kql @@ -0,0 +1,27 @@ +.create-merge table GitHubActionsLogLine ( + Repository: string, + WorkflowName: string, + WorkflowId: long, + RunId: long, + JobId: long, + StepNumber: int, + LineNumber: int, + Length: int, + Timestamp: string, + Message: string, + EtlIngestDate: datetime +) with (folder='', docstring='') + +.create-or-alter table GitHubActionsLogLine ingestion json mapping 'GitHubActionsLogLine_mapping' ```[ + { "column": "Repository", "path": "$['repository']" }, + { "column": "WorkflowName", "path": "$['workflowName']" }, + { "column": "WorkflowId", "path": "$['workflowId']" }, + { "column": "RunId", "path": "$['runId']" }, + { "column": "JobId", "path": "$['jobId']" }, + { "column": "StepNumber", "path": "$['stepNumber']" }, + { "column": "LineNumber", "path": "$['lineNumber']" }, + { "column": "Length", "path": "$['length']" }, + { "column": "Timestamp", "path": "$['timestamp']" }, + { "column": "Message", "path": "$['message']" }, + { "column": "EtlIngestDate", "path": "$['etlIngestDate']" } +]``` diff --git a/tools/pipeline-witness/infrastructure/kusto/tables/GitHubActionsRun.kql b/tools/pipeline-witness/infrastructure/kusto/tables/GitHubActionsRun.kql new file mode 100644 index 00000000000..40c91f1f374 --- /dev/null +++ b/tools/pipeline-witness/infrastructure/kusto/tables/GitHubActionsRun.kql @@ -0,0 +1,51 @@ +.create-merge table GitHubActionsRun ( + Repository: string, + Workflow: string, + WorkflowId: long, + RunId: long, + RunNumber: long, + HeadBranch: string, + HeadSha: string, + RunAttempt: long, + Event: string, + Status: string, + Conclusion: string, + CheckSuiteId: long, + DisplayTitle: string, + Path: string, + RunStartedAt: datetime, + CreatedAt: datetime, + UpdatedAt: datetime, + NodeId: string, + CheckSuiteNodeId: string, + HeadRepository: string, + Url: string, + HtmlUrl: string, + EtlIngestDate: datetime +) with (folder='', docstring='') + +.create-or-alter table GitHubActionsRun ingestion json mapping 'GitHubActionsRun_mapping' ```[ + { "column": "Repository", "path": "$['repository']" }, + { "column": "Workflow", "path": "$['workflow']" }, + { "column": "WorkflowId", "path": "$['workflowId']" }, + { "column": "RunId", "path": "$['runId']" }, + { "column": "RunNumber", "path": "$['runNumber']" }, + { "column": "HeadBranch", "path": "$['headBranch']" }, + { "column": "HeadSha", "path": "$['headSha']" }, + { "column": "RunAttempt", "path": "$['runAttempt']" }, + { "column": "Event", "path": "$['event']" }, + { "column": "Status", "path": "$['status']" }, + { "column": "Conclusion", "path": "$['conclusion']" }, + { "column": "CheckSuiteId", "path": "$['checkSuiteId']" }, + { "column": "DisplayTitle", "path": "$['displayTitle']" }, + { "column": "Path", "path": "$['path']" }, + { "column": "RunStartedAt", "path": "$['runStartedAt']" }, + { "column": "CreatedAt", "path": "$['createdAt']" }, + { "column": "UpdatedAt", "path": "$['updatedAt']" }, + { "column": "NodeId", "path": "$['nodeId']" }, + { "column": "CheckSuiteNodeId", "path": "$['checkSuiteNodeId']" }, + { "column": "HeadRepository", "path": "$['headRepository']" }, + { "column": "Url", "path": "$['url']" }, + { "column": "HtmlUrl", "path": "$['htmlUrl']" }, + { "column": "EtlIngestDate", "path": "$['etlIngestDate']" } +]``` diff --git a/tools/pipeline-witness/infrastructure/kusto/tables/GitHubActionsStep.kql b/tools/pipeline-witness/infrastructure/kusto/tables/GitHubActionsStep.kql new file mode 100644 index 00000000000..b949a9c2465 --- /dev/null +++ b/tools/pipeline-witness/infrastructure/kusto/tables/GitHubActionsStep.kql @@ -0,0 +1,31 @@ +.create-merge table GitHubActionsStep ( + Repository: string, + Workflow: string, + Job: string, + WorkflowId: long, + RunId: long, + JobId: long, + StepNumber: int, + Name: string, + Status: string, + Conclusion: string, + StartedAt: datetime, + CompletedAt: datetime, + EtlIngestDate: datetime +) with (folder='', docstring='') + +.create-or-alter table GitHubActionsStep ingestion json mapping 'GitHubActionsStep_mapping' ```[ + { "column": "Repository", "path": "$['repository']" }, + { "column": "Workflow", "path": "$['workflow']" }, + { "column": "Job", "path": "$['job']" }, + { "column": "WorkflowId", "path": "$['workflowId']" }, + { "column": "RunId", "path": "$['runId']" }, + { "column": "JobId", "path": "$['jobId']" }, + { "column": "StepNumber", "path": "$['stepNumber']" }, + { "column": "Name", "path": "$['name']" }, + { "column": "Status", "path": "$['status']" }, + { "column": "Conclusion", "path": "$['conclusion']" }, + { "column": "StartedAt", "path": "$['startedAt']" }, + { "column": "CompletedAt", "path": "$['completedAt']" }, + { "column": "EtlIngestDate", "path": "$['etlIngestDate']" } +]```