diff --git a/src/ApiService/ApiService/Functions/ValidateScriban.cs b/src/ApiService/ApiService/Functions/ValidateScriban.cs index d3f9b37f62..d32f70b523 100644 --- a/src/ApiService/ApiService/Functions/ValidateScriban.cs +++ b/src/ApiService/ApiService/Functions/ValidateScriban.cs @@ -103,6 +103,7 @@ public Async.Task Run([HttpTrigger(AuthorizationLevel.Anonymou null, null, null, + null, null ); diff --git a/src/ApiService/ApiService/OneFuzzTypes/Events.cs b/src/ApiService/ApiService/OneFuzzTypes/Events.cs index 7273bb9e7f..e5f94ee0a2 100644 --- a/src/ApiService/ApiService/OneFuzzTypes/Events.cs +++ b/src/ApiService/ApiService/OneFuzzTypes/Events.cs @@ -302,8 +302,13 @@ public record EventCrashReported( Container Container, [property: JsonPropertyName("filename")] String FileName, TaskConfig? TaskConfig -) : BaseEvent(); - +) : BaseEvent(), ITruncatable { + public BaseEvent Truncate(int maxLength) { + return this with { + Report = Report.Truncate(maxLength) + }; + } +} [EventType(EventType.RegressionReported)] public record EventRegressionReported( @@ -311,8 +316,13 @@ public record EventRegressionReported( Container Container, [property: JsonPropertyName("filename")] String FileName, TaskConfig? TaskConfig -) : BaseEvent(); - +) : BaseEvent(), ITruncatable { + public BaseEvent Truncate(int maxLength) { + return this with { + RegressionReport = RegressionReport.Truncate(maxLength) + }; + } +} [EventType(EventType.FileAdded)] public record EventFileAdded( diff --git a/src/ApiService/ApiService/OneFuzzTypes/Model.cs b/src/ApiService/ApiService/OneFuzzTypes/Model.cs index 83ed622734..0ce97b28d1 100644 --- a/src/ApiService/ApiService/OneFuzzTypes/Model.cs +++ b/src/ApiService/ApiService/OneFuzzTypes/Model.cs @@ -457,8 +457,31 @@ public record Report( string? MinimizedStackFunctionLinesSha256, string? ToolName, string? ToolVersion, - string? OnefuzzVersion -) : IReport; + string? OnefuzzVersion, + Uri? ReportUrl +) : IReport, ITruncatable { + public Report Truncate(int maxLength) { + return this with { + Executable = Executable[..maxLength], + CrashType = CrashType[..Math.Min(maxLength, CrashType.Length)], + CrashSite = CrashSite[..Math.Min(maxLength, CrashSite.Length)], + CallStack = TruncateUtils.TruncateList(CallStack, maxLength), + CallStackSha256 = CallStackSha256[..Math.Min(maxLength, CallStackSha256.Length)], + InputSha256 = InputSha256[..Math.Min(maxLength, InputSha256.Length)], + AsanLog = AsanLog?[..Math.Min(maxLength, AsanLog.Length)], + ScarinessDescription = ScarinessDescription?[..Math.Min(maxLength, ScarinessDescription.Length)], + MinimizedStack = MinimizedStack != null ? TruncateUtils.TruncateList(MinimizedStack, maxLength) : MinimizedStack, + MinimizedStackSha256 = MinimizedStackSha256?[..Math.Min(maxLength, MinimizedStackSha256.Length)], + MinimizedStackFunctionNames = MinimizedStackFunctionNames != null ? TruncateUtils.TruncateList(MinimizedStackFunctionNames, maxLength) : MinimizedStackFunctionNames, + MinimizedStackFunctionNamesSha256 = MinimizedStackFunctionNamesSha256?[..Math.Min(maxLength, MinimizedStackFunctionNamesSha256.Length)], + MinimizedStackFunctionLines = MinimizedStackFunctionLines != null ? TruncateUtils.TruncateList(MinimizedStackFunctionLines, maxLength) : MinimizedStackFunctionLines, + MinimizedStackFunctionLinesSha256 = MinimizedStackFunctionLinesSha256?[..Math.Min(maxLength, MinimizedStackFunctionLinesSha256.Length)], + ToolName = ToolName?[..Math.Min(maxLength, ToolName.Length)], + ToolVersion = ToolVersion?[..Math.Min(maxLength, ToolVersion.Length)], + OnefuzzVersion = OnefuzzVersion?[..Math.Min(maxLength, OnefuzzVersion.Length)], + }; + } +} public record NoReproReport( string InputSha, @@ -468,18 +491,40 @@ public record NoReproReport( Guid JobId, long Tries, string? Error -); +) : ITruncatable { + public NoReproReport Truncate(int maxLength) { + return this with { + Executable = Executable?[..maxLength], + Error = Error?[..maxLength] + }; + } +} public record CrashTestResult( Report? CrashReport, NoReproReport? NoReproReport -); +) : ITruncatable { + public CrashTestResult Truncate(int maxLength) { + return new CrashTestResult( + CrashReport?.Truncate(maxLength), + NoReproReport?.Truncate(maxLength) + ); + } +} public record RegressionReport( CrashTestResult CrashTestResult, - CrashTestResult? OriginalCrashTestResult -) : IReport; - + CrashTestResult? OriginalCrashTestResult, + Uri? ReportUrl +) : IReport, ITruncatable { + public RegressionReport Truncate(int maxLength) { + return new RegressionReport( + CrashTestResult.Truncate(maxLength), + OriginalCrashTestResult?.Truncate(maxLength), + ReportUrl + ); + } +} [JsonConverter(typeof(NotificationTemplateConverter))] #pragma warning disable CA1715 @@ -968,3 +1013,7 @@ public record TemplateRenderContext( string ReportFilename, string ReproCmd ); + +public interface ITruncatable { + public T Truncate(int maxLength); +} diff --git a/src/ApiService/ApiService/onefuzzlib/Reports.cs b/src/ApiService/ApiService/onefuzzlib/Reports.cs index da8847aa89..355939be60 100644 --- a/src/ApiService/ApiService/onefuzzlib/Reports.cs +++ b/src/ApiService/ApiService/onefuzzlib/Reports.cs @@ -44,10 +44,12 @@ public Reports(ILogTracer log, IContainers containers) { return null; } - return ParseReportOrRegression(blob.ToString(), filePath, expectReports); + var reportUrl = await _containers.GetFileUrl(container, fileName, StorageType.Corpus); + + return ParseReportOrRegression(blob.ToString(), filePath, reportUrl, expectReports); } - private IReport? ParseReportOrRegression(string content, string? filePath, bool expectReports = false) { + private IReport? ParseReportOrRegression(string content, string? filePath, Uri? reportUrl, bool expectReports = false) { var regressionReport = JsonSerializer.Deserialize(content, EntityConverter.GetJsonSerializerOptions()); if (regressionReport == null || regressionReport.CrashTestResult == null) { var report = JsonSerializer.Deserialize(content, EntityConverter.GetJsonSerializerOptions()); @@ -55,10 +57,14 @@ public Reports(ILogTracer log, IContainers containers) { _log.Error($"unable to parse report ({filePath:Tag:FilePath}) as a report or regression"); return null; } - return report; + return report != null ? report with { ReportUrl = reportUrl } : report; } - return regressionReport; + return regressionReport != null ? regressionReport with { ReportUrl = reportUrl } : regressionReport; } } -public interface IReport { }; +public interface IReport { + Uri? ReportUrl { + init; + } +}; diff --git a/src/ApiService/ApiService/onefuzzlib/Utils.cs b/src/ApiService/ApiService/onefuzzlib/Utils.cs index 3ecd7ce97d..b73aa721ef 100644 --- a/src/ApiService/ApiService/onefuzzlib/Utils.cs +++ b/src/ApiService/ApiService/onefuzzlib/Utils.cs @@ -39,3 +39,10 @@ public static async IAsyncEnumerable> Chunk(this IAsyncEn } } } + +public static class TruncateUtils { + public static List TruncateList(List data, int maxLength) { + int currentLength = 0; + return data.TakeWhile(curr => (currentLength += curr.Length) <= maxLength).ToList(); + } +} diff --git a/src/ApiService/ApiService/onefuzzlib/WebhookOperations.cs b/src/ApiService/ApiService/onefuzzlib/WebhookOperations.cs index 0c11e2d761..5d4599f7b7 100644 --- a/src/ApiService/ApiService/onefuzzlib/WebhookOperations.cs +++ b/src/ApiService/ApiService/onefuzzlib/WebhookOperations.cs @@ -34,6 +34,8 @@ async public Async.Task SendEvent(EventMessage eventMessage) { } async private Async.Task AddEvent(Webhook webhook, EventMessage eventMessage) { + (string, string)[] tags = { ("WebhookId", webhook.WebhookId.ToString()), ("EventId", eventMessage.EventId.ToString()) }; + var message = new WebhookMessageLog( EventId: eventMessage.EventId, EventType: eventMessage.EventType, @@ -46,8 +48,18 @@ async private Async.Task AddEvent(Webhook webhook, EventMessage eventMessage) { var r = await _context.WebhookMessageLogOperations.Replace(message); if (!r.IsOk) { - _logTracer.WithHttpStatus(r.ErrorV).Error($"Failed to replace webhook message log {webhook.WebhookId:Tag:WebhookId} - {eventMessage.EventId:Tag:EventId}"); + if (r.ErrorV.Reason.Contains("The entity is larger than the maximum allowed size") && eventMessage.Event is ITruncatable truncatableEvent) { + _logTracer.WithTags(tags).Warning($"The WebhookMessageLog was too long. Truncating event data and trying again."); + var truncatedEventMessage = message with { + Event = truncatableEvent.Truncate(1000) + }; + r = await _context.WebhookMessageLogOperations.Replace(truncatedEventMessage); + } + if (!r.IsOk) { + _logTracer.WithHttpStatus(r.ErrorV).WithTags(tags).Error($"Failed to replace webhook message log {webhook.WebhookId:Tag:WebhookId} - {eventMessage.EventId:Tag:EventId}"); + } } + await _context.WebhookMessageLogOperations.QueueWebhook(message); } @@ -57,7 +69,7 @@ public async Async.Task Send(WebhookMessageLog messageLog) { throw new Exception($"Invalid Webhook. Webhook with WebhookId: {messageLog.WebhookId} Not Found"); } - var (data, digest) = await BuildMessage(webhookId: webhook.WebhookId, eventId: messageLog.EventId, eventType: messageLog.EventType, webhookEvent: messageLog.Event, secretToken: webhook.SecretToken, messageFormat: webhook.MessageFormat); + var (data, digest) = await BuildMessage(webhookId: webhook.WebhookId, eventId: messageLog.EventId, eventType: messageLog.EventType, webhookEvent: messageLog.Event!, secretToken: webhook.SecretToken, messageFormat: webhook.MessageFormat); var headers = new Dictionary { { "User-Agent", $"onefuzz-webhook {_context.ServiceConfiguration.OneFuzzVersion}" } }; diff --git a/src/ApiService/ApiService/onefuzzlib/orm/Orm.cs b/src/ApiService/ApiService/onefuzzlib/orm/Orm.cs index b902c2f0e0..8b92017f42 100644 --- a/src/ApiService/ApiService/onefuzzlib/orm/Orm.cs +++ b/src/ApiService/ApiService/onefuzzlib/orm/Orm.cs @@ -79,15 +79,19 @@ public async IAsyncEnumerable QueryAsync(string? filter = null) { } public async Task> Replace(T entity) { - var tableClient = await GetTableClient(typeof(T).Name); - var tableEntity = _entityConverter.ToTableEntity(entity); - var response = await tableClient.UpsertEntityAsync(tableEntity, TableUpdateMode.Replace); - if (response.IsError) { - return ResultVoid<(HttpStatusCode, string)>.Error(((HttpStatusCode)response.Status, response.ReasonPhrase)); - } else { - // update ETag on success - entity.ETag = response.Headers.ETag; - return ResultVoid<(HttpStatusCode, string)>.Ok(); + try { + var tableClient = await GetTableClient(typeof(T).Name); + var tableEntity = _entityConverter.ToTableEntity(entity); + var response = await tableClient.UpsertEntityAsync(tableEntity, TableUpdateMode.Replace); + if (response.IsError) { + return ResultVoid<(HttpStatusCode, string)>.Error(((HttpStatusCode)response.Status, response.ReasonPhrase)); + } else { + // update ETag on success + entity.ETag = response.Headers.ETag; + return ResultVoid<(HttpStatusCode, string)>.Ok(); + } + } catch (RequestFailedException ex) { + return ResultVoid<(HttpStatusCode, string)>.Error(((HttpStatusCode)ex.Status, ex.Message)); } } diff --git a/src/ApiService/IntegrationTests/ReproVmssTests.cs b/src/ApiService/IntegrationTests/ReproVmssTests.cs index afed702c75..6046eb0e73 100644 --- a/src/ApiService/IntegrationTests/ReproVmssTests.cs +++ b/src/ApiService/IntegrationTests/ReproVmssTests.cs @@ -190,6 +190,7 @@ public async Async.Task CannotCreateVMForMissingReport() { null, null, null, + null, null ); diff --git a/src/ApiService/Tests/OrmModelsTest.cs b/src/ApiService/Tests/OrmModelsTest.cs index ec953b2319..c8c19edcd1 100644 --- a/src/ApiService/Tests/OrmModelsTest.cs +++ b/src/ApiService/Tests/OrmModelsTest.cs @@ -300,7 +300,7 @@ public static Gen WebhookMessageEventGrid() { } public static Gen Report() { - return Arb.Generate, Guid, int>>().Select( + return Arb.Generate, Guid, int, Uri?>>().Select( arg => new Report( InputUrl: arg.Item1, @@ -324,8 +324,8 @@ public static Gen Report() { MinimizedStackFunctionLinesSha256: arg.Item1, ToolName: arg.Item1, ToolVersion: arg.Item1, - OnefuzzVersion: arg.Item1 - + OnefuzzVersion: arg.Item1, + ReportUrl: arg.Item6 ) ); @@ -357,11 +357,12 @@ public static Gen CrashTestResult() { } public static Gen RegressionReport() { - return Arb.Generate>().Select( + return Arb.Generate>().Select( arg => new RegressionReport( arg.Item1, - arg.Item2 + arg.Item2, + arg.Item3 ) ); } diff --git a/src/ApiService/Tests/TemplateTests.cs b/src/ApiService/Tests/TemplateTests.cs index d1b7e16d6a..9da5d1ecc2 100644 --- a/src/ApiService/Tests/TemplateTests.cs +++ b/src/ApiService/Tests/TemplateTests.cs @@ -190,6 +190,7 @@ private static Report GetReport() { null, null, null, + null, null ); } diff --git a/src/ApiService/Tests/TruncationTests.cs b/src/ApiService/Tests/TruncationTests.cs new file mode 100644 index 0000000000..1ee4689209 --- /dev/null +++ b/src/ApiService/Tests/TruncationTests.cs @@ -0,0 +1,73 @@ +using System; +using System.Collections.Generic; +using FluentAssertions; +using Microsoft.OneFuzz.Service; +using Xunit; + +namespace Tests; + +public class TruncationTests { + [Fact] + public static void ReportIsTruncatable() { + var report = GenerateReport(); + + var truncatedReport = report.Truncate(5); + + truncatedReport.Executable.Should().Be("SOMES"); + truncatedReport.CallStack.Count.Should().Be(0); + } + + [Fact] + public static void TestListTruncation() { + var testList = new List { + "1", "2", "3", "456" + }; + + var truncatedList = TruncateUtils.TruncateList(testList, 3); + truncatedList.Count.Should().Be(3); + truncatedList.Should().BeEquivalentTo(new[] { "1", "2", "3" }); + } + + [Fact] + public static void TestNestedTruncation() { + var eventCrashReported = new EventCrashReported( + GenerateReport(), + Container.Parse("123"), + "abc", + null + ); + + var truncatedEvent = eventCrashReported.Truncate(3) as EventCrashReported; + truncatedEvent.Should().NotBeNull(); + truncatedEvent?.Report.Executable.Should().Be("SOM"); + truncatedEvent?.Report.CallStack.Count.Should().Be(0); + } + + private static Report GenerateReport() { + return new Report( + null, + null, + "SOMESUPRTLONGSTRINGSOMESUPRTLONGSTRINGSOMESUPRTLONGSTRINGSOMESUPRTLONGSTRING", + "abc", + "abc", + new List { "SOMESUPRTLONGSTRINGSOMESUPRTLONGSTRING" }, + "abc", + "abc", + null, + Guid.Empty, + Guid.Empty, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + new Uri("http://example.com") + ); + } +}