diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness.Tests/BlobUploadProcessorIntegrationTests.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness.Tests/BlobUploadProcessorIntegrationTests.cs index 398aaab244c..2f0495d42f5 100644 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness.Tests/BlobUploadProcessorIntegrationTests.cs +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness.Tests/BlobUploadProcessorIntegrationTests.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; +using Azure.Sdk.Tools.PipelineWitness.AzurePipelines; using Azure.Sdk.Tools.PipelineWitness.Configuration; using Azure.Storage.Blobs; using Microsoft.Extensions.Logging.Abstractions; @@ -52,7 +53,7 @@ public async Task BasicBlobProcessInvokesSuccessfully() Assert.True(recentBuilds.Count > 0); int targetBuildId = recentBuilds.First().Id; - BlobUploadProcessor processor = new(logger: new NullLogger(), + AzurePipelinesProcessor processor = new(logger: new NullLogger(), blobServiceClient: blobServiceClient, vssConnection: this.visualStudioConnection, options: Options.Create(this.testSettings)); @@ -67,7 +68,7 @@ public async Task BasicBlobProcessInvokesSuccessfully() [InlineData(0, 10000, 0)] public void TestBatching(int startingNumber, int batchSize, int expectedBatchNumber) { - int numberOfBatches = BlobUploadProcessor.CalculateBatches(startingNumber, batchSize); + int numberOfBatches = AzurePipelinesProcessor.CalculateBatches(startingNumber, batchSize); Assert.Equal(expectedBatchNumber, numberOfBatches); } diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness.Tests/TestLogger.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness.Tests/TestLogger.cs index 9b5072fd67e..f583392c946 100644 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness.Tests/TestLogger.cs +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness.Tests/TestLogger.cs @@ -6,7 +6,7 @@ namespace Azure.Sdk.Tools.PipelineWitness.Tests { public class TestLogger : ILogger { - internal List Logs { get; } = new List(); + internal List Logs { get; } = []; public IDisposable BeginScope(TState state) { diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Azure.Sdk.Tools.PipelineWitness.csproj b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Azure.Sdk.Tools.PipelineWitness.csproj index 53f41736ddd..2d948a28181 100644 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Azure.Sdk.Tools.PipelineWitness.csproj +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Azure.Sdk.Tools.PipelineWitness.csproj @@ -3,6 +3,7 @@ net8.0 bc5587e8-3503-4e1a-816c-1e219e4047f6 + True diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/AzurePipelines/AzurePipelinesBuildDefinitionWorker.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/AzurePipelines/AzurePipelinesBuildDefinitionWorker.cs new file mode 100644 index 00000000000..40689337caa --- /dev/null +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/AzurePipelines/AzurePipelinesBuildDefinitionWorker.cs @@ -0,0 +1,48 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Azure.Sdk.Tools.PipelineWitness.Configuration; +using Azure.Sdk.Tools.PipelineWitness.Services; +using Azure.Sdk.Tools.PipelineWitness.Services.WorkTokens; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace Azure.Sdk.Tools.PipelineWitness.AzurePipelines +{ + public class AzurePipelinesBuildDefinitionWorker : PeriodicLockingBackgroundService + { + private readonly ILogger logger; + private readonly AzurePipelinesProcessor runProcessor; + private readonly IOptions options; + + public AzurePipelinesBuildDefinitionWorker( + ILogger logger, + AzurePipelinesProcessor runProcessor, + IAsyncLockProvider asyncLockProvider, + IOptions options) + : base( + logger, + asyncLockProvider, + options.Value.BuildDefinitionWorker) + { + this.logger = logger; + this.runProcessor = runProcessor; + this.options = options; + } + + protected override async Task ProcessAsync(CancellationToken cancellationToken) + { + var settings = this.options.Value; + foreach (string project in settings.Projects) + { + await this.runProcessor.UploadBuildDefinitionBlobsAsync(settings.Account, project); + } + } + + protected override Task ProcessExceptionAsync(Exception ex) + { + this.logger.LogError(ex, "Error processing build definitions"); + return Task.CompletedTask; + } + } +} diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/BlobUploadProcessor.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/AzurePipelines/AzurePipelinesProcessor.cs similarity index 94% rename from tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/BlobUploadProcessor.cs rename to tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/AzurePipelines/AzurePipelinesProcessor.cs index c14ae709fb0..3aa446edf2b 100644 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/BlobUploadProcessor.cs +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/AzurePipelines/AzurePipelinesProcessor.cs @@ -6,11 +6,11 @@ using System.Linq; using System.Net; using System.Text; -using System.Text.RegularExpressions; +using System.Threading; using System.Threading.Tasks; using Azure.Sdk.Tools.PipelineWitness.Configuration; -using Azure.Sdk.Tools.PipelineWitness.Services; +using Azure.Sdk.Tools.PipelineWitness.Utilities; using Azure.Storage.Blobs; using Azure.Storage.Blobs.Models; @@ -24,10 +24,10 @@ using Newtonsoft.Json.Converters; using Newtonsoft.Json.Serialization; -namespace Azure.Sdk.Tools.PipelineWitness +namespace Azure.Sdk.Tools.PipelineWitness.AzurePipelines { [SuppressMessage("Style", "IDE0037:Use inferred member name", Justification = "Explicit member names are added to json export objects for clarity")] - public class BlobUploadProcessor + public class AzurePipelinesProcessor { private const string BuildsContainerName = "builds"; private const string BuildLogLinesContainerName = "buildloglines"; @@ -39,6 +39,7 @@ public class BlobUploadProcessor private const int ApiBatchSize = 10000; private const string TimeFormat = @"yyyy-MM-dd\THH:mm:ss.fffffff\Z"; + private static readonly JsonSerializerSettings jsonSettings = new() { ContractResolver = new CamelCasePropertyNamesContractResolver(), @@ -46,7 +47,7 @@ public class BlobUploadProcessor Formatting = Formatting.None, }; - private readonly ILogger logger; + private readonly ILogger logger; private readonly TestResultsHttpClient testResultsClient; private readonly BuildHttpClient buildClient; private readonly BlobContainerClient buildLogLinesContainerClient; @@ -57,18 +58,15 @@ public class BlobUploadProcessor private readonly BlobContainerClient buildDefinitionsContainerClient; private readonly BlobContainerClient pipelineOwnersContainerClient; private readonly IOptions options; - private readonly Dictionary cachedDefinitionRevisions = new(); + private readonly Dictionary cachedDefinitionRevisions = []; - public BlobUploadProcessor( - ILogger logger, + public AzurePipelinesProcessor( + ILogger logger, BlobServiceClient blobServiceClient, VssConnection vssConnection, IOptions options) { - if (blobServiceClient == null) - { - throw new ArgumentNullException(nameof(blobServiceClient)); - } + ArgumentNullException.ThrowIfNull(blobServiceClient); this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); this.options = options ?? throw new ArgumentNullException(nameof(options)); @@ -81,10 +79,7 @@ public BlobUploadProcessor( this.buildDefinitionsContainerClient = blobServiceClient.GetBlobContainerClient(BuildDefinitionsContainerName); this.pipelineOwnersContainerClient = blobServiceClient.GetBlobContainerClient(PipelineOwnersContainerName); - if (vssConnection == null) - { - throw new ArgumentNullException(nameof(vssConnection)); - } + ArgumentNullException.ThrowIfNull(vssConnection); this.buildClient = vssConnection.GetClient(); this.testResultsClient = vssConnection.GetClient(); @@ -151,8 +146,6 @@ public async Task UploadBuildBlobsAsync(string account, Guid projectId, int buil return; } - await UploadBuildBlobAsync(account, build); - await UploadTestRunBlobsAsync(account, build); Timeline timeline = await this.buildClient.GetBuildTimelineAsync(projectId, buildId); @@ -185,13 +178,24 @@ public async Task UploadBuildBlobsAsync(string account, Guid projectId, int buil { await UploadPipelineOwnersBlobAsync(account, build, timeline); } + + // We upload the build blob last. This allows us to use the existence of the blob as a signal that build processing is complete. + await UploadBuildBlobAsync(account, build); + } + + public string GetBuildBlobName(Build build) + { + long changeTime = ((DateTimeOffset)build.LastChangedDate).ToUnixTimeSeconds(); + string blobName = $"{build.Project.Name}/{build.FinishTime:yyyy/MM/dd}/{build.Id}-{changeTime}.jsonl".ToLower(); + + return blobName; } private async Task UploadPipelineOwnersBlobAsync(string account, Build build, Timeline timeline) { try { - string blobPath = $"{build.Project.Name}/{build.FinishTime:yyyy/MM/dd}/{build.Id}-{timeline.ChangeId}.jsonl"; + string blobPath = $"{build.Project.Name}/{build.FinishTime:yyyy/MM/dd}/{build.Id}-{timeline.ChangeId}.jsonl".ToLower(); BlobClient blobClient = this.pipelineOwnersContainerClient.GetBlobClient(blobPath); if (await blobClient.ExistsAsync()) @@ -314,7 +318,7 @@ public async Task UploadBuildDefinitionBlobsAsync(string account, string project private async Task UploadBuildDefinitionBlobAsync(string account, BuildDefinition definition) { - string blobPath = $"{definition.Project.Name}/{definition.Id}-{definition.Revision}.jsonl"; + string blobPath = $"{definition.Project.Name}/{definition.Id}-{definition.Revision}.jsonl".ToLower(); try { @@ -390,7 +394,7 @@ private List GetBuildLogInfos(Build build, Timeline timeline, List { Dictionary logsById = logs.ToDictionary(l => l.Id); - List buildLogInfos = new(); + List buildLogInfos = []; foreach (BuildLog log in logs) { @@ -441,9 +445,7 @@ private async Task UploadBuildBlobAsync(string account, Build build) { try { - long changeTime = ((DateTimeOffset)build.LastChangedDate).ToUnixTimeSeconds(); - string blobPath = $"{build.Project.Name}/{build.FinishTime:yyyy/MM/dd}/{build.Id}-{changeTime}.jsonl"; - BlobClient blobClient = this.buildsContainerClient.GetBlobClient(blobPath); + BlobClient blobClient = this.buildsContainerClient.GetBlobClient(GetBuildBlobName(build)); if (await blobClient.ExistsAsync()) { @@ -493,7 +495,7 @@ private async Task UploadBuildBlobAsync(string account, Build build) SourceBranch = build.SourceBranch, SourceVersion = build.SourceVersion, Status = build.Status, - Tags = build.Tags?.Any() == true ? JsonConvert.SerializeObject(build.Tags, jsonSettings) : null, + Tags = build.Tags?.Count > 0 ? JsonConvert.SerializeObject(build.Tags, jsonSettings) : null, Url = $"https://dev.azure.com/{account}/{build.Project!.Name}/_build/results?buildId={build.Id}", ValidationResults = build.ValidationResults, EtlIngestDate = DateTime.UtcNow, @@ -522,7 +524,7 @@ private async Task UploadTimelineBlobAsync(string account, Build build, Timeline return; } - string blobPath = $"{build.Project.Name}/{build.FinishTime:yyyy/MM/dd}/{build.Id}-{timeline.ChangeId}.jsonl"; + string blobPath = $"{build.Project.Name}/{build.FinishTime:yyyy/MM/dd}/{build.Id}-{timeline.ChangeId}.jsonl".ToLower(); BlobClient blobClient = this.buildTimelineRecordsContainerClient.GetBlobClient(blobPath); if (await blobClient.ExistsAsync()) @@ -570,7 +572,7 @@ private async Task UploadTimelineBlobAsync(string account, Build build, Timeline TaskName = record.Task?.Name, TaskVersion = record.Task?.Version, Type = record.RecordType, - Issues = record.Issues?.Any() == true + Issues = record.Issues?.Count > 0 ? JsonConvert.SerializeObject(record.Issues, jsonSettings) : null, EtlIngestDate = DateTime.UtcNow, @@ -596,7 +598,7 @@ private async Task UploadLogLinesBlobAsync(string account, Build build, BuildLog { // we don't use FinishTime in the logs blob path to prevent duplicating logs when processing retries. // i.e. logs with a given buildid/logid are immutable and retries only add new logs. - string blobPath = $"{build.Project.Name}/{build.QueueTime:yyyy/MM/dd}/{build.Id}-{log.LogId}.jsonl"; + string blobPath = $"{build.Project.Name}/{build.QueueTime:yyyy/MM/dd}/{build.Id}-{log.LogId}.jsonl".ToLower(); BlobClient blobClient = this.buildLogLinesContainerClient.GetBlobClient(blobPath); if (await blobClient.ExistsAsync()) @@ -631,20 +633,10 @@ private async Task UploadLogLinesBlobAsync(string account, Build build, BuildLog lineNumber += 1; characterCount += line.Length; - // log lines usually follow the format: - // 2022-03-30T21:38:38.7007903Z Downloading task: AzureKeyVault (1.200.0) - // Sometimes, there's no leading timestamp, so we'll use the last timestamp we saw. - Match match = Regex.Match(line, @"^([^Z]{20,28}Z) (.*)$"); - - DateTimeOffset timestamp = match.Success - ? DateTime.ParseExact(match.Groups[1].Value, TimeFormat, null, - System.Globalization.DateTimeStyles.AssumeUniversal).ToUniversalTime() - : lastTimeStamp; + var (timestamp, message) = StringUtilities.ParseLogLine(line, lastTimeStamp); lastTimeStamp = timestamp; - string message = match.Success ? match.Groups[2].Value : line; - await blobWriter.WriteLineAsync(JsonConvert.SerializeObject(new { OrganizationName = account, @@ -682,7 +674,6 @@ private async Task UploadTestRunBlobsAsync(string account, Build build) try { string continuationToken = string.Empty; - int[] buildIds = new[] { build.Id }; DateTime minLastUpdatedDate = build.QueueTime!.Value.AddHours(-1); DateTime maxLastUpdatedDate = build.FinishTime!.Value.AddHours(1); @@ -705,7 +696,7 @@ private async Task UploadTestRunBlobsAsync(string account, Build build) rangeStart, rangeEnd, continuationToken: continuationToken, - buildIds: buildIds + buildIds: [ build.Id ] ); foreach (TestRun testRun in page) @@ -731,7 +722,7 @@ private async Task UploadTestRunBlobAsync(string account, Build build, TestRun t { try { - string blobPath = $"{build.Project.Name}/{testRun.CompletedDate:yyyy/MM/dd}/{testRun.Id}.jsonl"; + string blobPath = $"{build.Project.Name}/{testRun.CompletedDate:yyyy/MM/dd}/{testRun.Id}.jsonl".ToLower(); BlobClient blobClient = this.testRunsContainerClient.GetBlobClient(blobPath); if (await blobClient.ExistsAsync()) @@ -814,7 +805,7 @@ private async Task UploadTestRunResultBlobAsync(string account, Build build, Tes { try { - string blobPath = $"{build.Project.Name}/{testRun.CompletedDate:yyyy/MM/dd}/{testRun.Id}.jsonl"; + string blobPath = $"{build.Project.Name}/{testRun.CompletedDate:yyyy/MM/dd}/{testRun.Id}.jsonl".ToLower(); BlobClient blobClient = this.testResultsContainerClient.GetBlobClient(blobPath); if (await blobClient.ExistsAsync()) @@ -824,7 +815,7 @@ private async Task UploadTestRunResultBlobAsync(string account, Build build, Tes } StringBuilder builder = new(); - int batchCount = BlobUploadProcessor.CalculateBatches(testRun.TotalTests, batchSize: ApiBatchSize); + int batchCount = AzurePipelinesProcessor.CalculateBatches(testRun.TotalTests, batchSize: ApiBatchSize); for (int batchMultiplier = 0; batchMultiplier < batchCount; batchMultiplier++) { diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/AzurePipelines/BuildCompleteQueue.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/AzurePipelines/BuildCompleteQueue.cs new file mode 100644 index 00000000000..11b828fd7f2 --- /dev/null +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/AzurePipelines/BuildCompleteQueue.cs @@ -0,0 +1,27 @@ +using System.Text.Json; +using System.Threading.Tasks; +using Azure.Sdk.Tools.PipelineWitness.Configuration; +using Azure.Storage.Queues; +using Azure.Storage.Queues.Models; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace Azure.Sdk.Tools.PipelineWitness.AzurePipelines; + +public class BuildCompleteQueue +{ + private readonly ILogger logger; + private readonly QueueClient queueClient; + + public BuildCompleteQueue(ILogger logger, QueueServiceClient queueServiceClient, IOptions options) + { + this.logger = logger; + this.queueClient = queueServiceClient.GetQueueClient(options.Value.BuildCompleteQueueName); + } + + public async Task EnqueueMessageAsync(BuildCompleteQueueMessage message) + { + SendReceipt response = await this.queueClient.SendMessageAsync(JsonSerializer.Serialize(message)); + this.logger.LogDebug("Message added to queue with id {MessageId}", response.MessageId); + } +} diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/AzurePipelines/BuildCompleteQueueMessage.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/AzurePipelines/BuildCompleteQueueMessage.cs new file mode 100644 index 00000000000..9887dbabee0 --- /dev/null +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/AzurePipelines/BuildCompleteQueueMessage.cs @@ -0,0 +1,13 @@ +using System; + +namespace Azure.Sdk.Tools.PipelineWitness.AzurePipelines +{ + public class BuildCompleteQueueMessage + { + public string Account { get; set; } + + public Guid ProjectId { get; set; } + + public int BuildId { get; set; } + } +} diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/AzurePipelines/BuildCompleteQueueWorker.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/AzurePipelines/BuildCompleteQueueWorker.cs new file mode 100644 index 00000000000..dfb7b6a84f1 --- /dev/null +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/AzurePipelines/BuildCompleteQueueWorker.cs @@ -0,0 +1,68 @@ +using System; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Azure.Sdk.Tools.PipelineWitness.Configuration; +using Azure.Sdk.Tools.PipelineWitness.Controllers; +using Azure.Sdk.Tools.PipelineWitness.Services; +using Azure.Storage.Queues; +using Azure.Storage.Queues.Models; + +using Microsoft.ApplicationInsights; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace Azure.Sdk.Tools.PipelineWitness.AzurePipelines +{ + internal class BuildCompleteQueueWorker : QueueWorkerBackgroundService + { + private readonly ILogger logger; + private readonly AzurePipelinesProcessor runProcessor; + + public BuildCompleteQueueWorker( + ILogger logger, + AzurePipelinesProcessor runProcessor, + QueueServiceClient queueServiceClient, + TelemetryClient telemetryClient, + IOptionsMonitor options) + : base( + logger, + telemetryClient, + queueServiceClient, + options.CurrentValue.BuildCompleteQueueName, + options) + { + this.logger = logger; + this.runProcessor = runProcessor; + } + + internal override async Task ProcessMessageAsync(QueueMessage message, CancellationToken cancellationToken) + { + this.logger.LogInformation("Processing build.complete event: {MessageText}", message.MessageText); + + BuildCompleteQueueMessage queueMessage; + + if (message.MessageText.Contains("_apis/build/Builds")) + { + // Legacy message format. Parsing is now done in the DevopsEventsController. Use the controler to convert it. + if (!DevopsEventsController.TryConvertMessage(JsonDocument.Parse(message.MessageText), out queueMessage)) + { + this.logger.LogError("Failed to convert legacy message: {MessageText}", message.MessageText); + return; + } + } + else + { + queueMessage = JsonSerializer.Deserialize(message.MessageText); + } + + if (string.IsNullOrEmpty(queueMessage.Account) || queueMessage.ProjectId == Guid.Empty || queueMessage.BuildId == 0) + { + this.logger.LogError("Failed to deserialize message: {MessageText}", message.MessageText); + return; + } + + await this.runProcessor.UploadBuildBlobsAsync(queueMessage.Account, queueMessage.ProjectId, queueMessage.BuildId); + } + } +} diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Entities/BuildLogBundle.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/AzurePipelines/BuildLogBundle.cs similarity index 82% rename from tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Entities/BuildLogBundle.cs rename to tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/AzurePipelines/BuildLogBundle.cs index 7ad749c7508..ef63c855784 100644 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Entities/BuildLogBundle.cs +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/AzurePipelines/BuildLogBundle.cs @@ -1,7 +1,7 @@ using System; using System.Collections.Generic; -namespace Azure.Sdk.Tools.PipelineWitness +namespace Azure.Sdk.Tools.PipelineWitness.AzurePipelines { public class BuildLogBundle { @@ -25,6 +25,6 @@ public class BuildLogBundle public string DefinitionName { get; set; } - public List TimelineLogs { get; } = new List(); + public List TimelineLogs { get; } = []; } } diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Entities/BuildLogInfo.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/AzurePipelines/BuildLogInfo.cs similarity index 85% rename from tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Entities/BuildLogInfo.cs rename to tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/AzurePipelines/BuildLogInfo.cs index 1899c4634a6..f6f8763647c 100644 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Entities/BuildLogInfo.cs +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/AzurePipelines/BuildLogInfo.cs @@ -1,6 +1,6 @@ using System; -namespace Azure.Sdk.Tools.PipelineWitness +namespace Azure.Sdk.Tools.PipelineWitness.AzurePipelines { public class BuildLogInfo { diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/EnhancedBuildHttpClient.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/AzurePipelines/EnhancedBuildHttpClient.cs similarity index 97% rename from tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/EnhancedBuildHttpClient.cs rename to tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/AzurePipelines/EnhancedBuildHttpClient.cs index bbb91ac115e..ad309bdec81 100644 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/EnhancedBuildHttpClient.cs +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/AzurePipelines/EnhancedBuildHttpClient.cs @@ -3,11 +3,10 @@ using System.Net.Http; using System.Threading; using System.Threading.Tasks; - using Microsoft.TeamFoundation.Build.WebApi; using Microsoft.VisualStudio.Services.Common; -namespace Azure.Sdk.Tools.PipelineWitness.Services +namespace Azure.Sdk.Tools.PipelineWitness.AzurePipelines { public class EnhancedBuildHttpClient : BuildHttpClient diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Configuration/PeriodicProcessSettings.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Configuration/PeriodicProcessSettings.cs new file mode 100644 index 00000000000..73164d3c295 --- /dev/null +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Configuration/PeriodicProcessSettings.cs @@ -0,0 +1,32 @@ +using System; + +namespace Azure.Sdk.Tools.PipelineWitness.Configuration +{ + public class PeriodicProcessSettings + { + /// + /// Gets or sets whether the loop should be processed + /// + public bool Enabled { get; set; } + + /// + /// Gets or sets the amount of time between iterations of the build definition upload loop + /// + public TimeSpan LockLeasePeriod { get; set; } = TimeSpan.FromMinutes(5); + + /// + /// Gets or sets the amount of time between iterations of the loop + /// + public TimeSpan LoopPeriod { get; set; } + + /// + /// Gets or sets the amount of time between to wait between successful iterations + /// + public TimeSpan CooldownPeriod { get; set; } + + /// + /// Gets or sets the name of the distributed lock + /// + public string LockName { get; set; } + } +} diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Configuration/PipelineWitnessSettings.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Configuration/PipelineWitnessSettings.cs index 479fac44f1d..c67d59cc734 100644 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Configuration/PipelineWitnessSettings.cs +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Configuration/PipelineWitnessSettings.cs @@ -4,11 +4,6 @@ namespace Azure.Sdk.Tools.PipelineWitness.Configuration { public class PipelineWitnessSettings { - /// - /// Gets or sets the uri of the key vault to use - /// - public string KeyVaultUri { get; set; } - /// /// Gets or sets uri of the cosmos account to use /// @@ -34,11 +29,6 @@ public class PipelineWitnessSettings /// public int BuildCompleteWorkerCount { get; set; } = 1; - /// - /// Gets or sets whether the build definition worker is enabled - /// - public bool BuildDefinitionWorkerEnabled { get; set; } = true; - /// /// Gets or sets the name of the GitHub actions queue /// @@ -91,9 +81,9 @@ public class PipelineWitnessSettings public string Account { get; set; } /// - /// Gets or sets the amount of time between iterations of the build definition upload loop + /// Gets or sets the loops settins for the Build Definitions worker /// - public TimeSpan BuildDefinitionLoopPeriod { get; set; } = TimeSpan.FromMinutes(5); + public PeriodicProcessSettings BuildDefinitionWorker { get; set; } /// /// Gets or sets the artifact name used by the pipeline owners extraction build diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Configuration/PostConfigureKeyVaultSettings.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Configuration/PostConfigureKeyVaultSettings.cs index 80e1fa488f5..aa211ea9cfd 100644 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Configuration/PostConfigureKeyVaultSettings.cs +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Configuration/PostConfigureKeyVaultSettings.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Linq; using System.Text.RegularExpressions; using Microsoft.Extensions.Logging; @@ -6,16 +7,20 @@ namespace Azure.Sdk.Tools.PipelineWitness.Configuration; -public class PostConfigureKeyVaultSettings : IPostConfigureOptions where T : class +public partial class PostConfigureKeyVaultSettings : IPostConfigureOptions where T : class { - private static readonly Regex secretRegex = new Regex(@"(?https://.*?\.vault\.azure\.net)/secrets/(?.*)", RegexOptions.Compiled, TimeSpan.FromSeconds(5)); + [GeneratedRegex(@"^(?https://.*?\.vault\.azure\.net)/secrets/(?.*)$")] + private static partial Regex SecretUriRegex(); + private readonly ILogger logger; private readonly ISecretClientProvider secretClientProvider; + private readonly Dictionary valueCache; public PostConfigureKeyVaultSettings(ILogger> logger, ISecretClientProvider secretClientProvider) { this.logger = logger; this.secretClientProvider = secretClientProvider; + this.valueCache = []; } public void PostConfigure(string name, T options) @@ -26,11 +31,21 @@ public void PostConfigure(string name, T options) foreach (var property in stringProperties) { - var value = (string)property.GetValue(options); + var propertyValue = (string)property.GetValue(options); - if (value != null) + if (propertyValue != null) { - var match = secretRegex.Match(value); + if(this.valueCache.TryGetValue(propertyValue, out var cacheEntry)) + { + if (DateTimeOffset.UtcNow < cacheEntry.ExpirationTime) + { + this.logger.LogInformation("Replacing setting property {PropertyName} with value from cache", property.Name); + property.SetValue(options, cacheEntry.Value); + continue; + } + } + + var match = SecretUriRegex().Match(propertyValue); if (match.Success) { @@ -40,12 +55,14 @@ public void PostConfigure(string name, T options) try { var secretClient = this.secretClientProvider.GetSecretClient(new Uri(vaultUrl)); - this.logger.LogInformation("Replacing setting property {PropertyName} with value from secret {SecretUrl}", property.Name, value); + this.logger.LogInformation("Replacing setting property {PropertyName} with value from secret {SecretUrl}", property.Name, propertyValue); var response = secretClient.GetSecret(secretName); var secret = response.Value; property.SetValue(options, secret.Value); + + this.valueCache[propertyValue] = (ExpirationTime: DateTimeOffset.UtcNow.AddMinutes(5), secret.Value); } catch (Exception exception) { diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Configuration/SecretClientProvider.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Configuration/SecretClientProvider.cs index a00966dc3fe..cbf4d08fc12 100644 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Configuration/SecretClientProvider.cs +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Configuration/SecretClientProvider.cs @@ -1,4 +1,4 @@ -using System; +using System; using Azure.Core; using Azure.Security.KeyVault.Secrets; diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Controllers/DevopsEventsController.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Controllers/DevopsEventsController.cs index 1940b29c1fb..7bb5d165d06 100644 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Controllers/DevopsEventsController.cs +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Controllers/DevopsEventsController.cs @@ -1,8 +1,9 @@ +using System; using System.Text.Json; +using System.Text.RegularExpressions; using System.Threading.Tasks; +using Azure.Sdk.Tools.PipelineWitness.AzurePipelines; using Azure.Sdk.Tools.PipelineWitness.Configuration; -using Azure.Storage.Queues; -using Azure.Storage.Queues.Models; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Logging; @@ -14,14 +15,17 @@ namespace Azure.Sdk.Tools.PipelineWitness.Controllers { [Route("api/devopsevents")] [ApiController] - public class DevopsEventsController : ControllerBase + public partial class DevopsEventsController : ControllerBase { - private readonly QueueClient queueClient; + [GeneratedRegex(@"^https://dev.azure.com/(?[\w-]+)/(?[0-9a-fA-F-]+)/_apis/build/Builds/(?\d+)$")] + private static partial Regex BuildUriRegex(); + private readonly ILogger logger; + private readonly BuildCompleteQueue buildCompleteQueue; - public DevopsEventsController(ILogger logger, QueueServiceClient queueServiceClient, IOptions options) + public DevopsEventsController(ILogger logger, BuildCompleteQueue buildCompleteQueue, IOptions options) { - this.queueClient = queueServiceClient.GetQueueClient(options.Value.BuildCompleteQueueName); + this.buildCompleteQueue = buildCompleteQueue; this.logger = logger; } @@ -29,25 +33,58 @@ public DevopsEventsController(ILogger logger, QueueServi [HttpPost] public async Task PostAsync([FromBody] JsonDocument value) { - if (value == null) { - throw new BadHttpRequestException("Missing payload", 400); + this.logger.LogInformation("Message received in DevopsEventsController.PostAsync"); + + if (!TryConvertMessage(value, out BuildCompleteQueueMessage message) || message.Account != "azure-sdk") + { + string messageText = value.RootElement.GetRawText(); + this.logger.LogError("Message content invalid: {Content}", messageText); + throw new BadHttpRequestException("Invalid payload", 400); } - this.logger.LogInformation("Message received in DevopsEventsController.PostAsync"); - string message = value.RootElement.GetRawText(); + await this.buildCompleteQueue.EnqueueMessageAsync(message); + } - if (value.RootElement.TryGetProperty("resource", out var resource) - && resource.TryGetProperty("url", out var url) - && url.GetString().StartsWith("https://dev.azure.com/azure-sdk/")) + public static bool TryConvertMessage(JsonDocument value, out BuildCompleteQueueMessage message) + { + string buildUrl = value.RootElement.TryGetProperty("resource", out JsonElement resource) + && resource.TryGetProperty("url", out JsonElement resourceUrl) + && resourceUrl.ValueKind == JsonValueKind.String + ? resourceUrl.GetString() + : null; + + if (buildUrl == null) { - SendReceipt response = await this.queueClient.SendMessageAsync(message); - this.logger.LogInformation("Message added to queue with id {MessageId}", response.MessageId); + message = null; + return false; } - else + + Match match = BuildUriRegex().Match(buildUrl); + + if (!match.Success) { - this.logger.LogError("Message content invalid: {Content}", message); - throw new BadHttpRequestException("Invalid payload", 400); + message = null; + return false; } + + string account = match.Groups["account"].Value; + string projectIdString = match.Groups["project"].Value; + string buildIdString = match.Groups["build"].Value; + + if (string.IsNullOrEmpty(account) || !Guid.TryParse(projectIdString, out Guid projectId) || !int.TryParse(buildIdString, out int buildId)) + { + message = null; + return false; + } + + message = new BuildCompleteQueueMessage + { + Account = account, + ProjectId = projectId, + BuildId = buildId + }; + + return true; } } } diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Controllers/GitHubEventsController.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Controllers/GitHubEventsController.cs index b39e4482e6e..191aeeb80b2 100644 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Controllers/GitHubEventsController.cs +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Controllers/GitHubEventsController.cs @@ -83,7 +83,7 @@ private async Task ProcessWorkflowRunEventAsync() if (action == "completed") { - var queueMessage = new GitHubRunCompleteMessage + var queueMessage = new RunCompleteQueueMessage { Owner = eventMessage.GetProperty("repository").GetProperty("owner").GetProperty("login").GetString(), Repository = eventMessage.GetProperty("repository").GetProperty("name").GetString(), diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Entities/AzurePipelines/Failure.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Entities/AzurePipelines/Failure.cs deleted file mode 100644 index 95ce6e024ae..00000000000 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Entities/AzurePipelines/Failure.cs +++ /dev/null @@ -1,23 +0,0 @@ -using Microsoft.TeamFoundation.Build.WebApi; - -#nullable disable - -namespace Azure.Sdk.Tools.PipelineWitness.Entities.AzurePipelines -{ - public class Failure - { - public Failure() - { - } - - public Failure(TimelineRecord record, string classification) - { - this.Record = record; - this.Classification = classification; - } - - public TimelineRecord Record { get; set; } - public string Classification { get; set; } - } -} - diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/GitHubActions/GitHubActionProcessor.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/GitHubActions/GitHubActionProcessor.cs index 7c4a6ac989d..ecd69a8a50b 100644 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/GitHubActions/GitHubActionProcessor.cs +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/GitHubActions/GitHubActionProcessor.cs @@ -14,11 +14,12 @@ using Newtonsoft.Json.Converters; using Newtonsoft.Json.Serialization; using System.Text; -using Microsoft.Azure.Pipelines.WebApi; +using System.Threading; +using Azure.Sdk.Tools.PipelineWitness.Utilities; namespace Azure.Sdk.Tools.PipelineWitness.GitHubActions { - public class GitHubActionProcessor + public partial class GitHubActionProcessor { private const string RunsContainerName = "githubactionsruns"; private const string JobsContainerName = "githubactionsjobs"; @@ -26,7 +27,9 @@ public class GitHubActionProcessor private const string LogsContainerName = "githubactionslogs"; private const string TimeFormat = @"yyyy-MM-dd\THH:mm:ss.fffffff\Z"; - private static readonly ProductHeaderValue productHeaderValue1 = new("PipelineWitness", "1.0"); + [GeneratedRegex(@"^(?:(?.*)\/)?(?\d+)_(?[^\/]+)\.txt$")] + private static partial Regex LogFilePathRegex(); + private static readonly JsonSerializerSettings jsonSettings = new() { ContractResolver = new CamelCasePropertyNamesContractResolver(), @@ -48,7 +51,7 @@ public GitHubActionProcessor(ILogger logger, BlobServiceC this.runsContainerClient = blobServiceClient.GetBlobContainerClient(RunsContainerName); this.jobsContainerClient = blobServiceClient.GetBlobContainerClient(JobsContainerName); this.stepsContainerClient = blobServiceClient.GetBlobContainerClient(StepsContainerName); - this.client = new GitHubClient(productHeaderValue1, credentials); + this.client = new GitHubClient(new ProductHeaderValue("PipelineWitness", "1.0"), credentials); } public async Task ProcessAsync(string owner, string repository, long runId) @@ -63,14 +66,27 @@ public async Task ProcessAsync(string owner, string repository, long runId) } } + public string GetRunBlobName(WorkflowRun run) + { + string repository = run.Repository.FullName; + long runId = run.Id; + long attempt = run.RunAttempt; + DateTimeOffset runStartedAt = run.RunStartedAt; + + string blobName = $"{repository}/{runStartedAt:yyyy/MM/dd}/{runId}-{attempt}.jsonl".ToLower(); + return blobName; + } + private async Task ProcessWorkflowRunAsync(WorkflowRun run) { List jobs = await GetJobsAsync(run); - await UploadRunBlobAsync(run); await UploadJobsBlobAsync(run, jobs); await UploadStepsBlobAsync(run, jobs); await UploadLogsBlobAsync(run, jobs); + + // We upload the run blob last. This allows us to use the existence of the blob as a signal that run processing is complete. + await UploadRunBlobAsync(run); } private async Task UploadRunBlobAsync(WorkflowRun run) @@ -84,7 +100,7 @@ private async Task UploadRunBlobAsync(WorkflowRun run) { // even though runid/attempt is unique, we still add a date component to the path for easier browsing // multiple attempts have the same runStartedAt, so the different attempt blobs will be in the same folder - string blobPath = $"{repository}/{run.RunStartedAt:yyyy/MM/dd}/{runId}-{attempt}.jsonl"; + string blobPath = GetRunBlobName(run); BlobClient blobClient = this.runsContainerClient.GetBlobClient(blobPath); if (await blobClient.ExistsAsync()) @@ -145,7 +161,7 @@ private async Task UploadJobsBlobAsync(WorkflowRun run, List jobs) try { - string blobPath = $"{repository}/{run.RunStartedAt:yyyy/MM/dd}/{runId}-{attempt}.jsonl"; + string blobPath = $"{repository}/{run.RunStartedAt:yyyy/MM/dd}/{runId}-{attempt}.jsonl".ToLower(); BlobClient blobClient = this.jobsContainerClient.GetBlobClient(blobPath); if (await blobClient.ExistsAsync()) @@ -211,7 +227,7 @@ private async Task UploadStepsBlobAsync(WorkflowRun run, List jobs) // logs with a given runId/attempt are immutable and retries add new attempts. // even though runid/attempt is unique, we still add a date component to the path for easier browsing // multiple attempts have the same runStartedAt, so the different attempt blobs will be in the same folder - string blobPath = $"{repository}/{run.RunStartedAt:yyyy/MM/dd}/{runId}-{attempt}.jsonl"; + string blobPath = $"{repository}/{run.RunStartedAt:yyyy/MM/dd}/{runId}-{attempt}.jsonl".ToLower(); BlobClient blobClient = this.stepsContainerClient.GetBlobClient(blobPath); if (await blobClient.ExistsAsync()) @@ -273,7 +289,7 @@ private async Task UploadLogsBlobAsync(WorkflowRun run, List jobs) // logs with a given runId/attempt are immutable and retries add new attempts. // even though runid/attempt is unique, we still add a date component to the path for easier browsing // multiple attempts have the same runStartedAt, so the different attempt blobs will be in the same folder - string blobPath = $"{repository}/{run.RunStartedAt:yyyy/MM/dd}/{runId}-{attempt}.jsonl"; + string blobPath = $"{repository}/{run.RunStartedAt:yyyy/MM/dd}/{runId}-{attempt}.jsonl".ToLower(); BlobClient blobClient = this.logsContainerClient.GetBlobClient(blobPath); if (await blobClient.ExistsAsync()) @@ -290,7 +306,7 @@ private async Task UploadLogsBlobAsync(WorkflowRun run, List jobs) .Select(x => new { Entry = x, - NameRegex = Regex.Match(x.FullName, @"^(?:(?.*)\/)?(?\d+)_(?[^\/]+)\.txt$"), + NameRegex = LogFilePathRegex().Match(x.FullName), }) .Where(x => x.NameRegex.Success) .Select(x => new @@ -325,12 +341,12 @@ private async Task UploadLogsBlobAsync(WorkflowRun run, List jobs) continue; } - IList logLines = ReadLogLines(jobEntry, step: 0); + IList logLines = ReadLogLines(jobEntry, step: 0, job.StartedAt); IList stepLines = job.Steps .Where(x => x.Conclusion != WorkflowJobConclusion.Skipped) .OrderBy(x => x.Number) - .SelectMany(step => ReadLogLines(logEntries[$"{job.Name}/{step.Number}"], step.Number)) + .SelectMany(step => ReadLogLines(logEntries[$"{job.Name}/{step.Number}"], step.Number, step.StartedAt ?? job.StartedAt)) .ToArray(); UpdateStepLines(logLines, stepLines); @@ -350,9 +366,9 @@ await blobWriter.WriteLineAsync(JsonConvert.SerializeObject(new JobId = job.Id, StepNumber = logLine.Step, LineNumber = logLine.Number, - Length = logLine.Message.Length, + logLine.Message.Length, Timestamp = logLine.Timestamp.ToString(TimeFormat), - Message = logLine.Message, + logLine.Message, EtlIngestDate = DateTime.UtcNow.ToString(TimeFormat), }, jsonSettings)); } @@ -371,7 +387,7 @@ await blobWriter.WriteLineAsync(JsonConvert.SerializeObject(new } } - private bool UpdateStepLines(IList jobLines, IList stepLines) + private static bool UpdateStepLines(IList jobLines, IList stepLines) { // For each line in the step, remove the corresponding line from the job if (stepLines.Count == 0) @@ -414,20 +430,20 @@ private bool UpdateStepLines(IList jobLines, IList stepLines) return false; } - private IList ReadLogLines(ZipArchiveEntry entry, int step) + private static List ReadLogLines(ZipArchiveEntry entry, int step, DateTimeOffset logStartTime) { var result = new List(); using var logReader = new StreamReader(entry.Open()); - DateTimeOffset lastTimestamp = default; + DateTimeOffset lastTimestamp = logStartTime; for (int lineNumber = 1; !logReader.EndOfStream; lineNumber++) { string line = logReader.ReadLine(); - Match logLine = Regex.Match(line, @"^(?\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\d(?:.\d+)?Z )?(?.*)$"); - string timeStampText = logLine.Groups["timestamp"].Value; - string message = StripAnsiiEsacpeSequences(logLine.Groups["message"].Value); - DateTimeOffset timestamp = !string.IsNullOrEmpty(timeStampText) ? DateTimeOffset.Parse(timeStampText) : lastTimestamp; + + var (timestamp, message) = StringUtilities.ParseLogLine(line, lastTimestamp); + + lastTimestamp = timestamp; result.Add(new LogLine { @@ -441,11 +457,6 @@ private IList ReadLogLines(ZipArchiveEntry entry, int step) return result; } - private string StripAnsiiEsacpeSequences(string input) - { - return Regex.Replace(input, @"\x1B\[[0-?]*[ -/]*[@-~]", ""); - } - private async Task GetWorkflowRunAsync(string owner, string repository, long runId) { WorkflowRun workflowRun = await this.client.Actions.Workflows.Runs.Get(owner, repository, runId); @@ -454,7 +465,7 @@ private async Task GetWorkflowRunAsync(string owner, string reposit private async Task> GetJobsAsync(WorkflowRun run) { - List jobs = new(); + List jobs = []; for (int pageNumber = 1; ; pageNumber++) { ApiOptions options = new() diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/GitHubActions/GitHubCredentialStore.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/GitHubActions/GitHubCredentialStore.cs index 79721015575..3cd43655652 100644 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/GitHubActions/GitHubCredentialStore.cs +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/GitHubActions/GitHubCredentialStore.cs @@ -5,6 +5,7 @@ using System.Threading; using System.Threading.Tasks; using Azure.Sdk.Tools.PipelineWitness.Configuration; +using Azure.Sdk.Tools.PipelineWitness.Utilities; using Microsoft.Extensions.Options; using Octokit; diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/GitHubActions/RunCompleteQueue.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/GitHubActions/RunCompleteQueue.cs new file mode 100644 index 00000000000..016be1608b7 --- /dev/null +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/GitHubActions/RunCompleteQueue.cs @@ -0,0 +1,27 @@ +using System.Text.Json; +using System.Threading.Tasks; +using Azure.Sdk.Tools.PipelineWitness.Configuration; +using Azure.Storage.Queues; +using Azure.Storage.Queues.Models; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace Azure.Sdk.Tools.PipelineWitness.GitHubActions; + +public class RunCompleteQueue +{ + private readonly ILogger logger; + private readonly QueueClient queueClient; + + public RunCompleteQueue(ILogger logger, QueueServiceClient queueServiceClient, IOptions options) + { + this.logger = logger; + this.queueClient = queueServiceClient.GetQueueClient(options.Value.GitHubActionRunsQueueName); + } + + public async Task EnqueueMessageAsync(RunCompleteQueueMessage message) + { + SendReceipt response = await this.queueClient.SendMessageAsync(JsonSerializer.Serialize(message)); + this.logger.LogDebug("Message added to queue with id {MessageId}", response.MessageId); + } +} diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/GitHubActions/GitHubRunCompleteMessage.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/GitHubActions/RunCompleteQueueMessage.cs similarity index 81% rename from tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/GitHubActions/GitHubRunCompleteMessage.cs rename to tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/GitHubActions/RunCompleteQueueMessage.cs index 059fe0c3f78..a303e33d786 100644 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/GitHubActions/GitHubRunCompleteMessage.cs +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/GitHubActions/RunCompleteQueueMessage.cs @@ -1,6 +1,6 @@ namespace Azure.Sdk.Tools.PipelineWitness.GitHubActions; -internal class GitHubRunCompleteMessage +public class RunCompleteQueueMessage { public string Owner { get; set; } public string Repository { get; set; } diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/GitHubActions/GitHubActionsRunQueueWorker.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/GitHubActions/RunCompleteQueueWorker.cs similarity index 82% rename from tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/GitHubActions/GitHubActionsRunQueueWorker.cs rename to tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/GitHubActions/RunCompleteQueueWorker.cs index 643d7a429e7..c4b07773a4c 100644 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/GitHubActions/GitHubActionsRunQueueWorker.cs +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/GitHubActions/RunCompleteQueueWorker.cs @@ -12,13 +12,13 @@ namespace Azure.Sdk.Tools.PipelineWitness.GitHubActions { - internal class GitHubActionsRunQueueWorker : QueueWorkerBackgroundService + internal class RunCompleteQueueWorker : QueueWorkerBackgroundService { private readonly ILogger logger; private readonly GitHubActionProcessor processor; - public GitHubActionsRunQueueWorker( - ILogger logger, + public RunCompleteQueueWorker( + ILogger logger, GitHubActionProcessor processor, QueueServiceClient queueServiceClient, TelemetryClient telemetryClient, @@ -38,7 +38,7 @@ internal override async Task ProcessMessageAsync(QueueMessage message, Cancellat { this.logger.LogInformation("Processing build.complete event: {MessageText}", message.MessageText); - var githubMessage = JsonSerializer.Deserialize(message.MessageText); + var githubMessage = JsonSerializer.Deserialize(message.MessageText); await this.processor.ProcessAsync(githubMessage.Owner, githubMessage.Repository, githubMessage.RunId); } diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/AzurePipelinesBuildDefinitionWorker.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/AzurePipelinesBuildDefinitionWorker.cs deleted file mode 100644 index 127ff7671ef..00000000000 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/AzurePipelinesBuildDefinitionWorker.cs +++ /dev/null @@ -1,68 +0,0 @@ -using System; -using System.Diagnostics; -using System.Threading; -using System.Threading.Tasks; -using Azure.Sdk.Tools.PipelineWitness.Configuration; -using Azure.Sdk.Tools.PipelineWitness.Services.WorkTokens; -using Microsoft.Extensions.Hosting; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; - -namespace Azure.Sdk.Tools.PipelineWitness.Services -{ - public class AzurePipelinesBuildDefinitionWorker : BackgroundService - { - private readonly ILogger logger; - private readonly Func runProcessorFactory; - private readonly IOptions options; - private readonly IAsyncLockProvider asyncLockProvider; - - public AzurePipelinesBuildDefinitionWorker( - ILogger logger, - Func runProcessorFactory, - IAsyncLockProvider asyncLockProvider, - IOptions options) - { - this.logger = logger; - this.runProcessorFactory = runProcessorFactory; - this.options = options; - this.asyncLockProvider = asyncLockProvider; - } - - protected override async Task ExecuteAsync(CancellationToken stoppingToken) - { - TimeSpan processEvery = TimeSpan.FromMinutes(60); - - while (true) - { - Stopwatch stopWatch = Stopwatch.StartNew(); - PipelineWitnessSettings settings = this.options.Value; - - try - { - await using IAsyncLock asyncLock = await this.asyncLockProvider.GetLockAsync("UpdateBuildDefinitions", processEvery, stoppingToken); - - // if there's no asyncLock, this process has already completed in the last hour - if (asyncLock != null) - { - BlobUploadProcessor runProcessor = this.runProcessorFactory.Invoke(); - foreach (string project in settings.Projects) - { - await runProcessor.UploadBuildDefinitionBlobsAsync(settings.Account, project); - } - } - } - catch (Exception ex) - { - this.logger.LogError(ex, "Error processing build definitions"); - } - - TimeSpan duration = settings.BuildDefinitionLoopPeriod - stopWatch.Elapsed; - if (duration > TimeSpan.Zero) - { - await Task.Delay(duration, stoppingToken); - } - } - } - } -} diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/BuildCompleteQueueWorker.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/BuildCompleteQueueWorker.cs deleted file mode 100644 index 8aab90f643f..00000000000 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/BuildCompleteQueueWorker.cs +++ /dev/null @@ -1,83 +0,0 @@ -using System; -using System.Text.RegularExpressions; -using System.Threading; -using System.Threading.Tasks; - -using Azure.Sdk.Tools.PipelineWitness.Configuration; -using Azure.Storage.Queues; -using Azure.Storage.Queues.Models; - -using Microsoft.ApplicationInsights; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; - -using Newtonsoft.Json.Linq; - -namespace Azure.Sdk.Tools.PipelineWitness.Services -{ - internal class BuildCompleteQueueWorker : QueueWorkerBackgroundService - { - private readonly ILogger logger; - private readonly Func runProcessorFactory; - - public BuildCompleteQueueWorker( - ILogger logger, - Func runProcessorFactory, - QueueServiceClient queueServiceClient, - TelemetryClient telemetryClient, - IOptionsMonitor options) - : base( - logger, - telemetryClient, - queueServiceClient, - options.CurrentValue.BuildCompleteQueueName, - options) - { - this.logger = logger; - this.runProcessorFactory = runProcessorFactory; - } - - internal override async Task ProcessMessageAsync(QueueMessage message, CancellationToken cancellationToken) - { - this.logger.LogInformation("Processing build.complete event: {MessageText}", message.MessageText); - - JObject devopsEvent = JObject.Parse(message.MessageText); - - string buildUrl = devopsEvent["resource"]?.Value("url"); - - if (buildUrl == null) - { - this.logger.LogError("Message contained no build url. Message body: {MessageBody}", message.MessageText); - return; - } - - Match match = Regex.Match(buildUrl, @"^https://dev.azure.com/(?[\w-]+)/(?[0-9a-fA-F-]+)/_apis/build/Builds/(?\d+)$"); - - if (!match.Success) - { - this.logger.LogError("Message contained an invalid build url: {BuildUrl}", buildUrl); - return; - } - - string account = match.Groups["account"].Value; - string projectIdString = match.Groups["project"].Value; - string buildIdString = match.Groups["build"].Value; - - if (!Guid.TryParse(projectIdString, out Guid projectId)) - { - this.logger.LogError("Could not parse project id as a guid '{ProjectId}'", projectIdString); - return; - } - - if (!int.TryParse(buildIdString, out int buildId)) - { - this.logger.LogError("Could not parse build id as a guid '{BuildId}'", buildIdString); - return; - } - - BlobUploadProcessor runProcessor = this.runProcessorFactory.Invoke(); - - await runProcessor.UploadBuildBlobsAsync(account, projectId, buildId); - } - } -} diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/PeriodicLockingBackgroundService.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/PeriodicLockingBackgroundService.cs new file mode 100644 index 00000000000..04c762d6e59 --- /dev/null +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/PeriodicLockingBackgroundService.cs @@ -0,0 +1,121 @@ +using System; +using System.Diagnostics; +using System.Threading; +using System.Threading.Tasks; +using Azure.Sdk.Tools.PipelineWitness.Configuration; +using Azure.Sdk.Tools.PipelineWitness.Services.WorkTokens; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; + +namespace Azure.Sdk.Tools.PipelineWitness.Services +{ + public abstract class PeriodicLockingBackgroundService : BackgroundService + { + private readonly ILogger logger; + private readonly IAsyncLockProvider asyncLockProvider; + private readonly bool enabled; + private readonly string lockName; + private readonly TimeSpan loopDuration; + private readonly TimeSpan lockDuration; + private readonly TimeSpan cooldownDuration; + + public PeriodicLockingBackgroundService( + ILogger logger, + IAsyncLockProvider asyncLockProvider, + PeriodicProcessSettings settings) + { + this.logger = logger; + this.asyncLockProvider = asyncLockProvider; + this.enabled = settings.Enabled; + this.lockName = settings.LockName; + this.loopDuration = settings.LoopPeriod; + this.lockDuration = settings.LockLeasePeriod; + this.cooldownDuration = settings.CooldownPeriod; + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + while (this.enabled) + { + Stopwatch stopWatch = Stopwatch.StartNew(); + + try + { + // The lock duration prevents multiple instances from running at the same time + // We set the lock duration shorter than the cooldown period to allow for quicker retries should a worker fail + await using IAsyncLock asyncLock = await this.asyncLockProvider.GetLockAsync(this.lockName, this.lockDuration, stoppingToken); + + // if we couldn't aquire a lock, another instance is already processing or we're in the cooldown period + if (asyncLock != null) + { + var localCancellation = CancellationTokenSource.CreateLinkedTokenSource(stoppingToken); + + // concurrently process and keep the lock alive + var processTask = ProcessAsync(localCancellation.Token); + var renewLockTask = RenewLockAsync(asyncLock, this.lockDuration, localCancellation.Token); + await Task.WhenAny(processTask, renewLockTask); + + // whichever task completes first, cancel the other, then wait for both to complete + localCancellation.Cancel(); + await Task.WhenAll(processTask, renewLockTask); + + // awaiting processTask will have thrown an exception if it failed + + // if processTask completed successfully, attemt to extend the lock for the cooldown period + // the cooldown period prevents the process from running too frequently + await asyncLock.TryExtendAsync(this.cooldownDuration, stoppingToken); + asyncLock.ReleaseOnDispose = false; + } + else + { + this.logger.LogInformation("Lock {LockName} not acquired", this.lockName); + } + } + catch (Exception ex) + { + await ProcessExceptionAsync(ex); + } + + // Remove the time spent processing from the wait time to maintain the loop period + TimeSpan duration = this.loopDuration - stopWatch.Elapsed; + if (duration > TimeSpan.Zero) + { + await Task.Delay(duration, stoppingToken); + } + } + } + + protected abstract Task ProcessExceptionAsync(Exception ex); + + protected abstract Task ProcessAsync(CancellationToken cancellationToken); + + private async Task RenewLockAsync(IAsyncLock asyncLock, TimeSpan duration, CancellationToken cancellationToken) + { + // Renew the lock every half the duration until cancelled or the lock is lost + try + { + while (true) + { + var result = await asyncLock.TryExtendAsync(duration, cancellationToken); + + if (!result) + { + this.logger.LogWarning("Lock lost"); + break; + } + + this.logger.LogInformation("Lock renewed"); + await Task.Delay(duration / 2, cancellationToken); + } + } + catch (OperationCanceledException) + { + // Cancellation is expected, ignore cancellation exceptions + } + catch (Exception ex) + { + this.logger.LogError(ex, "Error renewing lock"); + } + } + } +} diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/WorkTokens/CosmosAsyncLock.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/WorkTokens/CosmosAsyncLock.cs index c0177481d7a..d376114a51e 100644 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/WorkTokens/CosmosAsyncLock.cs +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/WorkTokens/CosmosAsyncLock.cs @@ -10,16 +10,14 @@ class CosmosAsyncLock : IAsyncLock { private readonly string id; private readonly PartitionKey partitionKey; - private readonly TimeSpan duration; private readonly Container container; private string etag; - public CosmosAsyncLock(string id, string etag, TimeSpan duration, Container container) + public CosmosAsyncLock(string id, string etag, Container container) { this.id = id; this.partitionKey = new PartitionKey(id); this.etag = etag; - this.duration = duration; this.container = container; } @@ -39,12 +37,12 @@ public async ValueTask DisposeAsync() } } - public async Task TryRenewAsync(CancellationToken cancellationToken) + public async Task TryExtendAsync(TimeSpan duration, CancellationToken cancellationToken) { try { ItemResponse response = await this.container.ReplaceItemAsync( - new CosmosLockDocument(this.id, this.duration), + new CosmosLockDocument(this.id, duration), this.id, this.partitionKey, new ItemRequestOptions { IfMatchEtag = this.etag }, diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/WorkTokens/CosmosAsyncLockProvider.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/WorkTokens/CosmosAsyncLockProvider.cs index 2cd5299acfd..ef856847087 100644 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/WorkTokens/CosmosAsyncLockProvider.cs +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/WorkTokens/CosmosAsyncLockProvider.cs @@ -13,10 +13,7 @@ public class CosmosAsyncLockProvider : IAsyncLockProvider public CosmosAsyncLockProvider(CosmosClient cosmosClient, string databaseName, string containerName) { - if (cosmosClient == null) - { - throw new ArgumentNullException(nameof(cosmosClient)); - } + ArgumentNullException.ThrowIfNull(cosmosClient); this.container = cosmosClient.GetContainer(databaseName, containerName); } @@ -54,7 +51,7 @@ public async Task GetLockAsync(string id, TimeSpan duration, Cancell if (response.StatusCode == HttpStatusCode.OK) { - return new CosmosAsyncLock(id, response.ETag, duration, this.container); + return new CosmosAsyncLock(id, response.ETag, this.container); } } catch (CosmosException ex) when (ex.StatusCode == HttpStatusCode.Conflict) @@ -73,7 +70,7 @@ private async Task CreateLockAsync(string id, TimeSpan duration, Can new PartitionKey(id), cancellationToken: cancellationToken); - return new CosmosAsyncLock(id, response.ETag, duration, this.container); + return new CosmosAsyncLock(id, response.ETag, this.container); } catch (CosmosException ex) when (ex.StatusCode == HttpStatusCode.Conflict) { diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/WorkTokens/IAsyncLock.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/WorkTokens/IAsyncLock.cs index 74607d13ef8..f82771caea8 100644 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/WorkTokens/IAsyncLock.cs +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/WorkTokens/IAsyncLock.cs @@ -8,6 +8,6 @@ public interface IAsyncLock : IAsyncDisposable { bool ReleaseOnDispose { get; set; } - Task TryRenewAsync(CancellationToken cancellationToken); + Task TryExtendAsync(TimeSpan duration, CancellationToken cancellationToken); } } diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Startup.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Startup.cs index 8c6523fc8d4..a1dc3d65676 100644 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Startup.cs +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Startup.cs @@ -4,9 +4,9 @@ using Azure.Core.Extensions; using Azure.Identity; using Azure.Sdk.Tools.PipelineWitness.ApplicationInsights; +using Azure.Sdk.Tools.PipelineWitness.AzurePipelines; using Azure.Sdk.Tools.PipelineWitness.Configuration; using Azure.Sdk.Tools.PipelineWitness.GitHubActions; -using Azure.Sdk.Tools.PipelineWitness.Services; using Azure.Sdk.Tools.PipelineWitness.Services.WorkTokens; using Microsoft.ApplicationInsights.Extensibility; @@ -28,10 +28,16 @@ public static class Startup { public static void Configure(WebApplicationBuilder builder) { - PipelineWitnessSettings settings = new(); IConfigurationSection settingsSection = builder.Configuration.GetSection("PipelineWitness"); + PipelineWitnessSettings settings = new(); settingsSection.Bind(settings); + builder.Services.AddLogging(); + + builder.Services.Configure(settingsSection); + builder.Services.AddSingleton(); + builder.Services.AddSingleton, PostConfigureKeyVaultSettings>(); + builder.Services.AddApplicationInsightsTelemetry(builder.Configuration); builder.Services.AddApplicationInsightsTelemetryProcessor(); builder.Services.AddTransient(); @@ -50,24 +56,16 @@ public static void Configure(WebApplicationBuilder builder) builder.Services.AddSingleton(provider => new CosmosAsyncLockProvider(provider.GetRequiredService(), settings.CosmosDatabase, settings.CosmosAsyncLockContainer)); builder.Services.AddTransient(CreateVssConnection); - builder.Services.AddLogging(); - - builder.Services.Configure(settingsSection); - builder.Services.AddSingleton(); - builder.Services.AddSingleton, PostConfigureKeyVaultSettings>(); - - builder.Services.AddTransient(); - builder.Services.AddTransient>(provider => provider.GetRequiredService); + builder.Services.AddTransient(); + builder.Services.AddTransient(); builder.Services.AddHostedService(settings.BuildCompleteWorkerCount); builder.Services.AddSingleton(); builder.Services.AddTransient(); - builder.Services.AddHostedService(settings.GitHubActionRunsWorkerCount); + builder.Services.AddTransient(); + builder.Services.AddHostedService(settings.GitHubActionRunsWorkerCount); - if (settings.BuildDefinitionWorkerEnabled) - { - builder.Services.AddHostedService(); - } + builder.Services.AddHostedService(); } private static void AddHostedService(this IServiceCollection services, int instanceCount) where T : class, IHostedService diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/GitHubActions/ProcessRunner.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Utilities/ProcessRunner.cs similarity index 98% rename from tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/GitHubActions/ProcessRunner.cs rename to tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Utilities/ProcessRunner.cs index 9dbd435a767..53497e9bced 100644 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/GitHubActions/ProcessRunner.cs +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Utilities/ProcessRunner.cs @@ -1,10 +1,10 @@ -using System; +using System; using System.Collections.Generic; using System.Diagnostics; using System.Threading; using System.Threading.Tasks; -namespace Azure.Sdk.Tools.PipelineWitness.GitHubActions; +namespace Azure.Sdk.Tools.PipelineWitness.Utilities; internal sealed class ProcessRunner : IDisposable { diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Utilities/StringUtilities.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Utilities/StringUtilities.cs new file mode 100644 index 00000000000..a41255d54f7 --- /dev/null +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Utilities/StringUtilities.cs @@ -0,0 +1,41 @@ +using System; +using System.Text.RegularExpressions; + +namespace Azure.Sdk.Tools.PipelineWitness.Utilities; + +public partial class StringUtilities +{ + [GeneratedRegex(@"\x1B\[[0-?]*[ -/]*[@-~]")] + private static partial Regex AnsiiEscapeRegex(); + + [GeneratedRegex(@"^(?:(?\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\d(?:.\d+)?Z) )?(?.*)$")] + private static partial Regex TimestampedLogLineRegex(); + + public static string StripAnsiiEsacpeSequences(string input) + { + return AnsiiEscapeRegex().Replace(input, ""); + } + + public static (DateTimeOffset TimeStamp, string Message) ParseLogLine(string line, DateTimeOffset defaultTimestamp) + { + // log lines usually follow the format: + // 2022-03-30T21:38:38.7007903Z Downloading task: AzureKeyVault (1.200.0) + // If there's no leading timestamp, we return the entire line as Message. + Match match = TimestampedLogLineRegex().Match(line); + + if (!match.Success) + { + return (defaultTimestamp, StripAnsiiEsacpeSequences(line)); + } + + string timeStampText = match.Groups["timestamp"].Value; + + DateTimeOffset timestamp = !string.IsNullOrEmpty(timeStampText) + ? DateTimeOffset.Parse(timeStampText).ToUniversalTime() + : defaultTimestamp; + + string message = StripAnsiiEsacpeSequences(match.Groups["message"].Value); + + return (timestamp, message); + } +} diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/appsettings.development.json b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/appsettings.development.json index 5eb9656d162..b31726f0d9e 100644 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/appsettings.development.json +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/appsettings.development.json @@ -19,8 +19,10 @@ "CosmosAccountUri": "https://pipelinewitnesstest.documents.azure.com", "GitHubWebhookSecret": "https://pipelinewitnesstest.vault.azure.net/secrets/github-webhook-validation-secret", "GitHubAccessToken": null, - "BuildDefinitionLoopPeriod": "00:01:00", - "BuildDefinitionWorkerEnabled": true, + "BuildDefinitionWorker": { + "LoopPeriod": "00:01:00", + "Enabled": true + }, "BuildCompleteWorkerCount": 1, "GitHubActionRunsWorkerCount": 1 } diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/appsettings.json b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/appsettings.json index ef559a750de..ff8cc53a7f6 100644 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/appsettings.json +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/appsettings.json @@ -21,9 +21,17 @@ "CosmosAccountUri": "https://pipelinewitnessprod.documents.azure.com", "CosmosDatabase": "records", "CosmosAsyncLockContainer": "locks", + + "BuildDefinitionWorker": { + "Enabled": true, + "LoopPeriod": "00:05:00", + "CooldownPeriod": "7.00:00:00", + "LockName": "BuildDefinitionWorker" + }, + "BuildCompleteQueueName": "azurepipelines-build-completed", "BuildCompleteWorkerCount": 10, - "BuildDefinitionLoopPeriod": "00:05:00", + "GitHubActionRunsQueueName": "github-actionrun-completed", "GitHubActionRunsWorkerCount": 10, "GitHubWebhookSecret": "https://pipelinewitnessprod.vault.azure.net/secrets/github-webhook-validation-secret", diff --git a/tools/pipeline-witness/PipelineWitness.sln b/tools/pipeline-witness/PipelineWitness.sln index c93dd78ada5..40baa8d7d34 100644 --- a/tools/pipeline-witness/PipelineWitness.sln +++ b/tools/pipeline-witness/PipelineWitness.sln @@ -8,9 +8,6 @@ EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Sdk.Tools.PipelineWitness.Tests", "Azure.Sdk.Tools.PipelineWitness.Tests\Azure.Sdk.Tools.PipelineWitness.Tests.csproj", "{AE649A76-2DDA-4B45-A426-21425A0B988A}" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{FD213B4A-A8B5-400D-ABD3-1D5B1551CE3C}" -ProjectSection(SolutionItems) = preProject - .editorconfig = .editorconfig -EndProjectSection EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution diff --git a/tools/pipeline-witness/infrastructure/bicep/appResourceGroup.bicep b/tools/pipeline-witness/infrastructure/bicep/appResourceGroup.bicep index 12b88b1ac71..e8a319f8da9 100644 --- a/tools/pipeline-witness/infrastructure/bicep/appResourceGroup.bicep +++ b/tools/pipeline-witness/infrastructure/bicep/appResourceGroup.bicep @@ -9,10 +9,11 @@ param keyVaultName string param location string param vnetPrefix string param subnetPrefix string +param useVnet bool var cosmosContributorRoleId = '00000000-0000-0000-0000-000000000002' // Built-in Contributor role -resource networkSecurityGroup 'Microsoft.Network/networkSecurityGroups@2023-11-01' = { +resource networkSecurityGroup 'Microsoft.Network/networkSecurityGroups@2023-11-01' = if (useVnet) { name: networkSecurityGroupName location: 'westus2' properties: { @@ -20,7 +21,7 @@ resource networkSecurityGroup 'Microsoft.Network/networkSecurityGroups@2023-11-0 } } -resource vnet 'Microsoft.Network/virtualNetworks@2023-11-01' = { +resource vnet 'Microsoft.Network/virtualNetworks@2023-11-01' = if (useVnet) { name: vnetName location: 'westus2' properties: { @@ -34,7 +35,7 @@ resource vnet 'Microsoft.Network/virtualNetworks@2023-11-01' = { } } -resource subnet 'Microsoft.Network/virtualNetworks/subnets@2023-11-01' = { +resource subnet 'Microsoft.Network/virtualNetworks/subnets@2023-11-01' = if (useVnet) { parent: vnet name: 'default' properties: { @@ -94,7 +95,7 @@ resource webApp 'Microsoft.Web/sites@2022-03-01' = { alwaysOn: true } httpsOnly: true - virtualNetworkSubnetId: subnet.id + virtualNetworkSubnetId: useVnet ? subnet.id : null publicNetworkAccess: 'Enabled' } identity: { @@ -116,11 +117,13 @@ resource appStorageAccount 'Microsoft.Storage/storageAccounts@2021-09-01' = { minimumTlsVersion: 'TLS1_2' allowBlobPublicAccess: false allowSharedKeyAccess: false - networkAcls: { - bypass: 'AzureServices' - virtualNetworkRules: [{ id: subnet.id }] - defaultAction: 'Deny' - } + networkAcls: useVnet + ? { + bypass: 'AzureServices' + virtualNetworkRules: [{ id: subnet.id }] + defaultAction: 'Deny' + } + : null supportsHttpsTrafficOnly: true encryption: { services: { @@ -207,9 +210,7 @@ resource cosmosAccount 'Microsoft.DocumentDB/databaseAccounts@2024-02-15-preview enableAutomaticFailover: false enableMultipleWriteLocations: false isVirtualNetworkFilterEnabled: true - virtualNetworkRules: [{ - id: subnet.id - }] + virtualNetworkRules: useVnet ? [{ id: subnet.id }] : [] disableKeyBasedMetadataWriteAccess: false enableFreeTier: false enableAnalyticalStorage: false diff --git a/tools/pipeline-witness/infrastructure/bicep/logsResourceGroup.bicep b/tools/pipeline-witness/infrastructure/bicep/logsResourceGroup.bicep index ac51b79e33e..b4f4cd3462c 100644 --- a/tools/pipeline-witness/infrastructure/bicep/logsResourceGroup.bicep +++ b/tools/pipeline-witness/infrastructure/bicep/logsResourceGroup.bicep @@ -7,6 +7,7 @@ param kustoDatabaseName string param webAppName string param subnetId string param appIdentityPrincipalId string +param useVnet bool var kustoScript = loadTextContent('../artifacts/merged.kql') @@ -24,11 +25,13 @@ resource logsStorageAccount 'Microsoft.Storage/storageAccounts@2021-09-01' = { minimumTlsVersion: 'TLS1_2' allowBlobPublicAccess: false allowSharedKeyAccess: false - networkAcls: { - bypass: 'AzureServices' - virtualNetworkRules: [{ id: subnetId }] - defaultAction: 'Deny' - } + networkAcls: useVnet + ? { + bypass: 'AzureServices' + virtualNetworkRules: [{ id: subnetId }] + defaultAction: 'Deny' + } + : null supportsHttpsTrafficOnly: true encryption: { services: { @@ -164,7 +167,7 @@ resource kustoCluster 'Microsoft.Kusto/Clusters@2022-02-01' = { } } - resource managedEndpoint 'managedPrivateEndpoints' = { + resource managedEndpoint 'managedPrivateEndpoints' = if(useVnet) { name: logsStorageAccountName properties: { groupId: 'blob' @@ -240,6 +243,10 @@ resource gitHubKustoEventHubsAssignment 'Microsoft.Authorization/roleAssignments } } +// Data Explorer needs to a per-table cursor when importing data. Because the read cursor for Event Hubs is the +// consumer group and the basic tier for event hubs is limited to 1 consumer group per event hub and 10 event hubs per +// namespace, we need an event hub per table, so we split our tables across two namespaces. +// https://learn.microsoft.com/en-us/azure/event-hubs/event-hubs-quotas module devOpsTables 'tableResources.bicep' = { name: 'devOpsTables' scope: resourceGroup() diff --git a/tools/pipeline-witness/infrastructure/bicep/main.bicep b/tools/pipeline-witness/infrastructure/bicep/main.bicep index dac18925a1a..1df5b2d3a9f 100644 --- a/tools/pipeline-witness/infrastructure/bicep/main.bicep +++ b/tools/pipeline-witness/infrastructure/bicep/main.bicep @@ -9,6 +9,7 @@ param networkSecurityGroupName string param vnetName string param vnetPrefix string param subnetPrefix string +param useVnet bool param cosmosAccountName string param appStorageAccountName string param aspEnvironment string @@ -42,6 +43,7 @@ module pipelineWitness 'appResourceGroup.bicep' = { aspEnvironment: aspEnvironment networkSecurityGroupName: networkSecurityGroupName vnetName: vnetName + useVnet: useVnet } } @@ -66,5 +68,6 @@ module pipelineLogs 'logsResourceGroup.bicep' = { gitHubEventHubNamespaceName: gitHubEventHubNamespaceName appIdentityPrincipalId: pipelineWitness.outputs.appIdentityPrincipalId subnetId: pipelineWitness.outputs.subnetId + useVnet: useVnet } } diff --git a/tools/pipeline-witness/infrastructure/bicep/parameters.test.json b/tools/pipeline-witness/infrastructure/bicep/parameters.test.json index 60f42045d1c..6051c45069a 100644 --- a/tools/pipeline-witness/infrastructure/bicep/parameters.test.json +++ b/tools/pipeline-witness/infrastructure/bicep/parameters.test.json @@ -55,6 +55,9 @@ }, "subnetPrefix": { "value": "10.7.0.0/24" + }, + "useVnet": { + "value": false } } } \ No newline at end of file diff --git a/tools/pipeline-witness/monitored-repos.json b/tools/pipeline-witness/monitored-repos.json new file mode 100644 index 00000000000..65dbcad126c --- /dev/null +++ b/tools/pipeline-witness/monitored-repos.json @@ -0,0 +1,12 @@ +[ + "Azure/autorest.csharp", + "Azure/autorest.go", + "Azure/autorest.java", + "Azure/azure-sdk", + "Azure/azure-sdk-for-go", + "Azure/azure-sdk-for-js", + "Azure/azure-sdk-for-java", + "Azure/azure-sdk-for-net", + "Azure/azure-sdk-for-python", + "Azure/azure-sdk-tools" +] \ No newline at end of file