From a54016c8064139183990bda56b7199355b19a1ec Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Tue, 12 Sep 2023 13:38:28 +0200 Subject: [PATCH 01/49] Add embedded content transfering and build events binary equality --- .../BinaryLogger/BinaryLogRecordKind.cs | 2 +- .../BinaryLogReplayEventSource.cs | 66 ++++++++++----- .../Logging/BinaryLogger/BinaryLogger.cs | 32 +++++-- .../BinaryLogger/BuildEventArgsReader.cs | 60 +++++++++++--- .../BinaryLogger/BuildEventArgsWriter.cs | 73 ++++++++-------- .../Postprocessing/ArchiveFile.cs | 78 +++++++++++++++++ .../EmbeddedContentEventArgs.cs | 22 +++++ .../Postprocessing/EmbeddedContentKind.cs | 17 ++++ .../EmbeddedContentKindExtensions.cs | 22 +++++ .../Postprocessing/GreedyBufferedStream.cs | 83 +++++++++++++++++++ .../IBuildEventArgsReaderNotifications.cs | 2 +- .../IBuildEventStringsReader.cs | 0 .../Postprocessing/IEmbeddedContentSource.cs | 13 +++ .../StringReadEventArgs.cs | 0 src/Build/Microsoft.Build.csproj | 6 +- src/Framework/Traits.cs | 6 ++ 16 files changed, 405 insertions(+), 77 deletions(-) create mode 100644 src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFile.cs create mode 100644 src/Build/Logging/BinaryLogger/Postprocessing/EmbeddedContentEventArgs.cs create mode 100644 src/Build/Logging/BinaryLogger/Postprocessing/EmbeddedContentKind.cs create mode 100644 src/Build/Logging/BinaryLogger/Postprocessing/EmbeddedContentKindExtensions.cs create mode 100644 src/Build/Logging/BinaryLogger/Postprocessing/GreedyBufferedStream.cs rename src/Build/Logging/BinaryLogger/{ => Postprocessing}/IBuildEventArgsReaderNotifications.cs (89%) rename src/Build/Logging/BinaryLogger/{ => Postprocessing}/IBuildEventStringsReader.cs (100%) create mode 100644 src/Build/Logging/BinaryLogger/Postprocessing/IEmbeddedContentSource.cs rename src/Build/Logging/BinaryLogger/{ => Postprocessing}/StringReadEventArgs.cs (100%) diff --git a/src/Build/Logging/BinaryLogger/BinaryLogRecordKind.cs b/src/Build/Logging/BinaryLogger/BinaryLogRecordKind.cs index 9fe1638fd3a..28333110721 100644 --- a/src/Build/Logging/BinaryLogger/BinaryLogRecordKind.cs +++ b/src/Build/Logging/BinaryLogger/BinaryLogRecordKind.cs @@ -24,7 +24,7 @@ internal enum BinaryLogRecordKind ProjectEvaluationStarted, ProjectEvaluationFinished, ProjectImported, - ProjectImportArchive, + ProjectImportArchive = 17, TargetSkipped, PropertyReassignment, UninitializedPropertyRead, diff --git a/src/Build/Logging/BinaryLogger/BinaryLogReplayEventSource.cs b/src/Build/Logging/BinaryLogger/BinaryLogReplayEventSource.cs index a19a06c2d37..b21169ed135 100644 --- a/src/Build/Logging/BinaryLogger/BinaryLogReplayEventSource.cs +++ b/src/Build/Logging/BinaryLogger/BinaryLogReplayEventSource.cs @@ -26,11 +26,6 @@ static BinaryLogReplayEventSource() _ = ItemGroupLoggingHelper.ItemGroupIncludeLogMessagePrefix; } - /// - /// Raised once is created during replaying - /// - public event Action? NotificationsSourceCreated; - /// /// Read the provided binary log file and raise corresponding events for each BuildEventArgs /// @@ -68,6 +63,38 @@ public static BinaryReader OpenReader(string sourceFilePath) } } + /// + /// Creates a for the provided binary reader over binary log file. + /// Caller is responsible for disposing the returned reader. + /// + /// + /// Indicates whether the passed BinaryReader should be closed on disposing. + /// BuildEventArgsReader over the given binlog file binary reader. + public static BuildEventArgsReader OpenBuildEventsReader(BinaryReader binaryReader, bool closeInput) + { + int fileFormatVersion = binaryReader.ReadInt32(); + + // the log file is written using a newer version of file format + // that we don't know how to read + if (fileFormatVersion > BinaryLogger.FileFormatVersion) + { + var text = ResourceUtilities.FormatResourceStringStripCodeAndKeyword("UnsupportedLogFileFormat", fileFormatVersion, BinaryLogger.FileFormatVersion); + throw new NotSupportedException(text); + } + + return new BuildEventArgsReader(binaryReader, fileFormatVersion) { CloseInput = closeInput }; + } + + /// + /// Creates a for the provided binary log file. + /// Performs decompression and buffering in the optimal way. + /// Caller is responsible for disposing the returned reader. + /// + /// + /// BinaryReader of the given binlog file. + public static BuildEventArgsReader OpenBuildEventsReader(string sourceFilePath) + => OpenBuildEventsReader(OpenReader(sourceFilePath), true); + /// /// Read the provided binary log file and raise corresponding events for each BuildEventArgs /// @@ -75,30 +102,29 @@ public static BinaryReader OpenReader(string sourceFilePath) /// A indicating the replay should stop as soon as possible. public void Replay(string sourceFilePath, CancellationToken cancellationToken) { - using var binaryReader = OpenReader(sourceFilePath); - Replay(binaryReader, cancellationToken); + using var eventsReader = OpenBuildEventsReader(sourceFilePath); + Replay(eventsReader, cancellationToken); } /// /// Read the provided binary log file and raise corresponding events for each BuildEventArgs /// /// The binary log content binary reader - caller is responsible for disposing. + /// Indicates whether the passed BinaryReader should be closed on disposing. /// A indicating the replay should stop as soon as possible. - public void Replay(BinaryReader binaryReader, CancellationToken cancellationToken) + public void Replay(BinaryReader binaryReader, bool closeInput, CancellationToken cancellationToken) { - int fileFormatVersion = binaryReader.ReadInt32(); - - // the log file is written using a newer version of file format - // that we don't know how to read - if (fileFormatVersion > BinaryLogger.FileFormatVersion) - { - var text = ResourceUtilities.FormatResourceStringStripCodeAndKeyword("UnsupportedLogFileFormat", fileFormatVersion, BinaryLogger.FileFormatVersion); - throw new NotSupportedException(text); - } - - using var reader = new BuildEventArgsReader(binaryReader, fileFormatVersion); - NotificationsSourceCreated?.Invoke(reader); + using var reader = OpenBuildEventsReader(binaryReader, closeInput); + Replay(reader, cancellationToken); + } + /// + /// Read the provided binary log file and raise corresponding events for each BuildEventArgs + /// + /// The build events reader - caller is responsible for disposing. + /// A indicating the replay should stop as soon as possible. + public void Replay(BuildEventArgsReader reader, CancellationToken cancellationToken) + { while (!cancellationToken.IsCancellationRequested && reader.Read() is { } instance) { Dispatch(instance); diff --git a/src/Build/Logging/BinaryLogger/BinaryLogger.cs b/src/Build/Logging/BinaryLogger/BinaryLogger.cs index 6c4e32345fb..9c60914d4a0 100644 --- a/src/Build/Logging/BinaryLogger/BinaryLogger.cs +++ b/src/Build/Logging/BinaryLogger/BinaryLogger.cs @@ -63,6 +63,8 @@ public sealed class BinaryLogger : ILogger // - AssemblyLoadBuildEventArgs // version 17: // - Added extended data for types implementing IExtendedBuildEventArgs + // - Making ProjectStartedEventArgs, ProjectEvaluationFinishedEventArgs, AssemblyLoadBuildEventArgs equal + // between de/serialization roundtrips. internal const int FileFormatVersion = 17; private Stream stream; @@ -117,7 +119,12 @@ public enum ProjectImportsCollectionMode /// /// Initializes the logger by subscribing to events of the specified event source. /// - public void Initialize(IEventSource eventSource) + public void Initialize(IEventSource eventSource) => Initialize(eventSource, null); + + /// + /// Initializes the logger by subscribing to events of the specified event source and embedded content source. + /// + public void Initialize(IEventSource eventSource, IEmbeddedContentSource embeddedFilesSource) { _initialTargetOutputLogging = Environment.GetEnvironmentVariable("MSBUILDTARGETOUTPUTLOGGING"); _initialLogImports = Traits.Instance.EscapeHatches.LogProjectImports; @@ -132,6 +139,13 @@ public void Initialize(IEventSource eventSource) ProcessParameters(); + if (embeddedFilesSource != null) + { + CollectProjectImports = ProjectImportsCollectionMode.None; + embeddedFilesSource.EmbeddedContentRead += args => + eventArgsWriter.WriteBlob(args.ContentKind.ToBinaryLogRecordKind(), args.ContentStream, args.Length); + } + try { string logDirectory = null; @@ -180,7 +194,9 @@ public void Initialize(IEventSource eventSource) // wrapping the GZipStream in a buffered stream significantly improves performance // and the max throughput is reached with a 32K buffer. See details here: // https://github.com/dotnet/runtime/issues/39233#issuecomment-745598847 - stream = new BufferedStream(stream, bufferSize: 32768); + stream = Traits.Instance.DeterministicBinlogStreamBuffering ? + new GreedyBufferedStream(stream, bufferSize: 32768) : + new BufferedStream(stream, bufferSize: 32768); binaryWriter = new BinaryWriter(stream); eventArgsWriter = new BuildEventArgsWriter(binaryWriter); @@ -189,9 +205,15 @@ public void Initialize(IEventSource eventSource) eventArgsWriter.EmbedFile += EventArgsWriter_EmbedFile; } - binaryWriter.Write(FileFormatVersion); - - LogInitialInfo(); + if (embeddedFilesSource == null) + { + binaryWriter.Write(FileFormatVersion); + LogInitialInfo(); + } + else + { + binaryWriter.Write(embeddedFilesSource.FileFormatVersion); + } eventSource.AnyEventRaised += EventSource_AnyEventRaised; diff --git a/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs b/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs index 6c5ccd717d6..747a078a50c 100644 --- a/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs +++ b/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs @@ -63,9 +63,19 @@ public BuildEventArgsReader(BinaryReader binaryReader, int fileFormatVersion) this.fileFormatVersion = fileFormatVersion; } + /// + /// Directs whether the passed should be closed when this instance is disposed. + /// Defaults to "false". + /// + public bool CloseInput { private get; set; } = false; + public void Dispose() { stringStorage.Dispose(); + if (CloseInput) + { + binaryReader.Dispose(); + } } /// @@ -81,11 +91,14 @@ public void Dispose() /// public event Action? StringEncountered; + public int FileFormatVersion => fileFormatVersion; + /// - /// Raised when the log reader encounters a binary blob embedded in the stream. - /// The arguments include the blob kind and the byte buffer with the contents. + /// Raised when the log reader encounters a project import archive (embedded content) in the stream. + /// The subscriber must read the exactly given length of binary data from the stream - otherwise exception is raised. + /// If no subscriber is attached, the data is skipped. /// - internal event Action? OnBlobRead; + public event Action? EmbeddedContentRead; /// /// Reads the next log record from the . @@ -113,7 +126,7 @@ public void Dispose() } else if (recordKind == BinaryLogRecordKind.ProjectImportArchive) { - ReadBlob(recordKind); + ReadEmbeddedContent(recordKind); } recordNumber += 1; @@ -212,11 +225,30 @@ private static bool IsAuxiliaryRecord(BinaryLogRecordKind recordKind) || recordKind == BinaryLogRecordKind.ProjectImportArchive; } - private void ReadBlob(BinaryLogRecordKind kind) + private void ReadEmbeddedContent(BinaryLogRecordKind recordKind) { int length = ReadInt32(); - byte[] bytes = binaryReader.ReadBytes(length); - OnBlobRead?.Invoke(kind, bytes); + if (EmbeddedContentRead != null) + { + long preEventPosition = binaryReader.BaseStream.CanSeek ? binaryReader.BaseStream.Position : 0; + EmbeddedContentRead(new EmbeddedContentEventArgs(recordKind.ToEmbeddedContentKind(), binaryReader.BaseStream, length)); + long postEventPosition = binaryReader.BaseStream.CanSeek ? binaryReader.BaseStream.Position : length; + if (postEventPosition - preEventPosition != length) + { + throw new InvalidDataException($"The {nameof(EmbeddedContentRead)} event handler must read exactly {length} bytes from the stream."); + } + } + else + { + if (binaryReader.BaseStream.CanSeek) + { + binaryReader.BaseStream.Seek(length, SeekOrigin.Current); + } + else + { + binaryReader.ReadBytes(length); + } + } } private void ReadNameValueList() @@ -419,11 +451,12 @@ private BuildEventArgs ReadProjectEvaluationFinishedEventArgs() if (fileFormatVersion >= 12) { - IEnumerable? globalProperties = null; - if (ReadBoolean()) + if (fileFormatVersion < 17) { - globalProperties = ReadStringDictionary(); + // Throw away, but need to advance past it + ReadBoolean(); } + IEnumerable? globalProperties = ReadStringDictionary(); var propertyList = ReadPropertyList(); var itemList = ReadProjectItems(); @@ -474,10 +507,12 @@ private BuildEventArgs ReadProjectStartedEventArgs() if (fileFormatVersion > 6) { - if (ReadBoolean()) + if (fileFormatVersion < 17) { - globalProperties = ReadStringDictionary(); + // Throw away, but need to advance past it + ReadBoolean(); } + globalProperties = ReadStringDictionary(); } var propertyList = ReadPropertyList(); @@ -950,6 +985,7 @@ private AssemblyLoadBuildEventArgs ReadAssemblyLoadEventArgs() mvid, appDomainName); SetCommonFields(e, fields); + e.ProjectFile = fields.ProjectFile; return e; } diff --git a/src/Build/Logging/BinaryLogger/BuildEventArgsWriter.cs b/src/Build/Logging/BinaryLogger/BuildEventArgsWriter.cs index 0a21182e83c..d962530bcc1 100644 --- a/src/Build/Logging/BinaryLogger/BuildEventArgsWriter.cs +++ b/src/Build/Logging/BinaryLogger/BuildEventArgsWriter.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; +using System.Buffers; using System.Collections; using System.Collections.Generic; using System.Globalization; @@ -229,20 +230,15 @@ private void WriteCore(BuildEventArgs e) } } - public void WriteBlob(BinaryLogRecordKind kind, byte[] bytes) - { - // write the blob directly to the underlying writer, - // bypassing the memory stream - using var redirection = RedirectWritesToOriginalWriter(); - - Write(kind); - Write(bytes.Length); - Write(bytes); - } + public void WriteBlob(BinaryLogRecordKind kind, Stream stream, int length) + => WriteBlobImpl(kind, stream, length); public void WriteBlob(BinaryLogRecordKind kind, Stream stream) + => WriteBlobImpl(kind, stream, null); + + private void WriteBlobImpl(BinaryLogRecordKind kind, Stream stream, int? length) { - if (stream.Length > int.MaxValue) + if (stream.CanSeek && stream.Length > int.MaxValue) { throw new ArgumentOutOfRangeException(nameof(stream)); } @@ -252,8 +248,8 @@ public void WriteBlob(BinaryLogRecordKind kind, Stream stream) using var redirection = RedirectWritesToOriginalWriter(); Write(kind); - Write((int)stream.Length); - Write(stream); + Write(length ?? (int)stream.Length); + Write(stream, length); } /// @@ -317,15 +313,7 @@ private void Write(ProjectEvaluationFinishedEventArgs e) WriteBuildEventArgsFields(e, writeMessage: false); WriteDeduplicatedString(e.ProjectFile); - if (e.GlobalProperties == null) - { - Write(false); - } - else - { - Write(true); - WriteProperties(e.GlobalProperties); - } + WriteProperties(e.GlobalProperties); WriteProperties(e.Properties); @@ -366,15 +354,7 @@ private void Write(ProjectStartedEventArgs e) WriteDeduplicatedString(e.TargetNames); WriteDeduplicatedString(e.ToolsVersion); - if (e.GlobalProperties == null) - { - Write(false); - } - else - { - Write(true); - Write(e.GlobalProperties); - } + Write(e.GlobalProperties); WriteProperties(e.Properties); @@ -1124,7 +1104,7 @@ private void Write(BinaryLogRecordKind kind) Write((int)kind); } - private void Write(int value) + internal void Write(int value) { BinaryWriterExtensions.Write7BitEncodedInt(binaryWriter, value); } @@ -1139,9 +1119,34 @@ private void Write(byte[] bytes) binaryWriter.Write(bytes); } - private void Write(Stream stream) + private void Write(Stream stream, int? length) { - stream.CopyTo(binaryWriter.BaseStream); + if (length == null) + { + stream.CopyTo(binaryWriter.BaseStream); + return; + } + + // borrowed from runtime from Stream.cs + const int defaultCopyBufferSize = 81920; + int bufferSize = Math.Min(defaultCopyBufferSize, length.Value); + + byte[] buffer = ArrayPool.Shared.Rent(bufferSize); + try + { + int bytesRead; + while ( + length > 0 && + (bytesRead = stream.Read(buffer, 0, Math.Min(buffer.Length, length.Value))) != 0) + { + binaryWriter.BaseStream.Write(buffer, 0, bytesRead); + length -= bytesRead; + } + } + finally + { + ArrayPool.Shared.Return(buffer); + } } private void Write(byte b) diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFile.cs b/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFile.cs new file mode 100644 index 00000000000..59868eb43dd --- /dev/null +++ b/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFile.cs @@ -0,0 +1,78 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; +using System.IO.Compression; + +namespace Microsoft.Build.Logging +{ + public class ArchiveFile : IDisposable + { + public ArchiveFile(string fullPath, StreamReader contentReader) + { + FullPath = fullPath; + _contentReader = contentReader; + } + + public static ArchiveFile From(ZipArchiveEntry entry) + { + return new ArchiveFile(CalculateArchivePath(entry.FullName), new StreamReader(entry.Open())); + } + + public string FullPath { get; } + + public StreamReader GetContentReader() + { + if (_stringAcquired) + { + throw new InvalidOperationException("Content already acquired as string via GetContent."); + } + + _streamAcquired = true; + return _contentReader; + } + + public string GetContent() + { + if (_streamAcquired) + { + throw new InvalidOperationException("Content already acquired as StreamReader via GetContnetReader."); + } + + if (!_stringAcquired) + { + _stringAcquired = true; + _content = _contentReader.ReadToEnd(); + } + + return _content!; + } + + private bool _streamAcquired; + private bool _stringAcquired; + private readonly StreamReader _contentReader; + private string? _content; + + public static string CalculateArchivePath(string filePath) + { + string archivePath = filePath; + + if (filePath.Contains(":") || (!filePath.StartsWith("\\") && !filePath.StartsWith("/"))) + { + archivePath = archivePath.Replace(":", ""); + archivePath = archivePath.Replace("/", "\\"); + archivePath = archivePath.Replace("\\\\", "\\"); + } + else + { + archivePath = archivePath.Replace("\\", "/"); + archivePath = archivePath.Replace("//", "/"); + } + + return archivePath; + } + + public void Dispose() => _contentReader.Dispose(); + } +} diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/EmbeddedContentEventArgs.cs b/src/Build/Logging/BinaryLogger/Postprocessing/EmbeddedContentEventArgs.cs new file mode 100644 index 00000000000..b5d6e985783 --- /dev/null +++ b/src/Build/Logging/BinaryLogger/Postprocessing/EmbeddedContentEventArgs.cs @@ -0,0 +1,22 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; + +namespace Microsoft.Build.Logging +{ + public sealed class EmbeddedContentEventArgs : EventArgs + { + public EmbeddedContentEventArgs(EmbeddedContentKind contentKind, Stream contentStream, int length) + { + ContentKind = contentKind; + ContentStream = contentStream; + Length = length; + } + + public EmbeddedContentKind ContentKind { get; } + public Stream ContentStream { get; } + public int Length { get; } + } +} diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/EmbeddedContentKind.cs b/src/Build/Logging/BinaryLogger/Postprocessing/EmbeddedContentKind.cs new file mode 100644 index 00000000000..7f694c91363 --- /dev/null +++ b/src/Build/Logging/BinaryLogger/Postprocessing/EmbeddedContentKind.cs @@ -0,0 +1,17 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Microsoft.Build.Logging +{ + public enum EmbeddedContentKind + { + Unknown = -1, + ProjectImportArchive = 17, + } +} diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/EmbeddedContentKindExtensions.cs b/src/Build/Logging/BinaryLogger/Postprocessing/EmbeddedContentKindExtensions.cs new file mode 100644 index 00000000000..73e9251cd77 --- /dev/null +++ b/src/Build/Logging/BinaryLogger/Postprocessing/EmbeddedContentKindExtensions.cs @@ -0,0 +1,22 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.Build.Logging +{ + internal static class EmbeddedContentKindExtensions + { + internal static EmbeddedContentKind ToEmbeddedContentKind(this BinaryLogRecordKind kind) + { + return kind == BinaryLogRecordKind.ProjectImportArchive + ? EmbeddedContentKind.ProjectImportArchive + : EmbeddedContentKind.Unknown; + } + + internal static BinaryLogRecordKind ToBinaryLogRecordKind(this EmbeddedContentKind kind) + { + return kind == EmbeddedContentKind.ProjectImportArchive + ? BinaryLogRecordKind.ProjectImportArchive + : (BinaryLogRecordKind)EmbeddedContentKind.Unknown; + } + } +} diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/GreedyBufferedStream.cs b/src/Build/Logging/BinaryLogger/Postprocessing/GreedyBufferedStream.cs new file mode 100644 index 00000000000..e334eac4b8f --- /dev/null +++ b/src/Build/Logging/BinaryLogger/Postprocessing/GreedyBufferedStream.cs @@ -0,0 +1,83 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; + +namespace Microsoft.Build.Logging +{ + /// + /// This is write-only, append-only stream that always buffers the wrapped stream + /// into the chunks of the same size (except the possible shorter last chunk). + /// So unlike the it never writes to the wrapped stream + /// until it has full chunk or is closing. + /// + /// This is not supposed to bring performance benefits, but it allows to avoid nondeterministic + /// GZipStream output for the identical input. + /// + internal class GreedyBufferedStream : Stream + { + private readonly Stream _stream; + private readonly byte[] _buffer; + private int _position; + + public GreedyBufferedStream(Stream stream, int bufferSize) + { + _stream = stream; + _buffer = new byte[bufferSize]; + } + + public override void Flush() + { + _stream.Write(_buffer, 0, _position); + _position = 0; + } + + public override int Read(byte[] buffer, int offset, int count) => throw UnsupportedException; + + public override long Seek(long offset, SeekOrigin origin) => throw UnsupportedException; + + public override void SetLength(long value) => throw UnsupportedException; + + public override void Write(byte[] buffer, int offset, int count) + { + // Appends input to the buffer until it is full - then flushes it to the wrapped stream. + // Repeat above until all input is processed. + + int srcOffset = offset; + do + { + int currentCount = Math.Min(count, _buffer.Length - _position); + Buffer.BlockCopy(buffer, srcOffset, _buffer, _position, currentCount); + _position += currentCount; + count -= currentCount; + srcOffset += currentCount; + + if (_position == _buffer.Length) + { + Flush(); + } + } while (count > 0); + } + + public override bool CanRead => false; + public override bool CanSeek => false; + public override bool CanWrite => _stream.CanWrite; + public override long Length => _stream.Length + _position; + + public override long Position + { + get => _stream.Position + _position; + set => throw UnsupportedException; + } + + public override void Close() + { + Flush(); + _stream.Close(); + base.Close(); + } + + private Exception UnsupportedException => new NotSupportedException("GreedyBufferedStream is write-only, append-only"); + } +} diff --git a/src/Build/Logging/BinaryLogger/IBuildEventArgsReaderNotifications.cs b/src/Build/Logging/BinaryLogger/Postprocessing/IBuildEventArgsReaderNotifications.cs similarity index 89% rename from src/Build/Logging/BinaryLogger/IBuildEventArgsReaderNotifications.cs rename to src/Build/Logging/BinaryLogger/Postprocessing/IBuildEventArgsReaderNotifications.cs index 415bd7c71fd..41ec404904e 100644 --- a/src/Build/Logging/BinaryLogger/IBuildEventArgsReaderNotifications.cs +++ b/src/Build/Logging/BinaryLogger/Postprocessing/IBuildEventArgsReaderNotifications.cs @@ -6,7 +6,7 @@ namespace Microsoft.Build.Logging /// /// An interface for notifications from BuildEventArgsReader /// - public interface IBuildEventArgsReaderNotifications : IBuildEventStringsReader + public interface IBuildEventArgsReaderNotifications : IBuildEventStringsReader, IEmbeddedContentSource { /* For future use */ } diff --git a/src/Build/Logging/BinaryLogger/IBuildEventStringsReader.cs b/src/Build/Logging/BinaryLogger/Postprocessing/IBuildEventStringsReader.cs similarity index 100% rename from src/Build/Logging/BinaryLogger/IBuildEventStringsReader.cs rename to src/Build/Logging/BinaryLogger/Postprocessing/IBuildEventStringsReader.cs diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/IEmbeddedContentSource.cs b/src/Build/Logging/BinaryLogger/Postprocessing/IEmbeddedContentSource.cs new file mode 100644 index 00000000000..e0080148b1a --- /dev/null +++ b/src/Build/Logging/BinaryLogger/Postprocessing/IEmbeddedContentSource.cs @@ -0,0 +1,13 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; + +namespace Microsoft.Build.Logging +{ + public interface IEmbeddedContentSource + { + int FileFormatVersion { get; } + event Action EmbeddedContentRead; + } +} diff --git a/src/Build/Logging/BinaryLogger/StringReadEventArgs.cs b/src/Build/Logging/BinaryLogger/Postprocessing/StringReadEventArgs.cs similarity index 100% rename from src/Build/Logging/BinaryLogger/StringReadEventArgs.cs rename to src/Build/Logging/BinaryLogger/Postprocessing/StringReadEventArgs.cs diff --git a/src/Build/Microsoft.Build.csproj b/src/Build/Microsoft.Build.csproj index a42e76cc270..81439958264 100644 --- a/src/Build/Microsoft.Build.csproj +++ b/src/Build/Microsoft.Build.csproj @@ -160,10 +160,6 @@ - - - - @@ -527,6 +523,8 @@ + + diff --git a/src/Framework/Traits.cs b/src/Framework/Traits.cs index 04b2fc90237..3c81043f963 100644 --- a/src/Framework/Traits.cs +++ b/src/Framework/Traits.cs @@ -120,6 +120,12 @@ public Traits() /// public readonly int LogPropertyTracking = ParseIntFromEnvironmentVariableOrDefault("MsBuildLogPropertyTracking", 0); // Default to logging nothing via the property tracker. + /// + /// Turn on greedy buffering stream decorator for binlog writer. + /// This will ensure that 2 identical binlog contents will result into identical binlog files (as writing different chunks to GZipStream can lead to different result). + /// + public readonly bool DeterministicBinlogStreamBuffering = !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("MSBUILDDETERMNISTICBINLOG")); + /// /// When evaluating items, this is the minimum number of items on the running list to use a dictionary-based remove optimization. /// From 1ed8999fe524389589f54f848f6f4715ab91318a Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Fri, 15 Sep 2023 15:53:09 +0200 Subject: [PATCH 02/49] Added support for embedded files; fixing, cleaning --- .../BinaryLogReplayEventSource.cs | 21 +- .../Logging/BinaryLogger/BinaryLogger.cs | 82 +++---- .../BinaryLogger/BuildEventArgsReader.cs | 75 ++++++- .../BinaryLogger/BuildEventArgsWriter.cs | 13 +- .../Postprocessing/ArchiveFile.cs | 51 ++--- .../Postprocessing/ArchiveFileEventArgs.cs | 47 ++++ .../ArchiveFileEventArgsExtensions.cs | 23 ++ .../Postprocessing/CleanupScope.cs | 15 ++ .../EmbeddedContentEventArgs.cs | 8 +- .../IBuildEventArgsReaderNotifications.cs | 3 +- .../IBuildEventStringsReader.cs | 6 - .../Postprocessing/IBuildFileReader.cs | 34 +++ .../Postprocessing/IEmbeddedContentSource.cs | 15 +- .../BinaryLogger/Postprocessing/SubStream.cs | 56 +++++ .../BinaryLogger/ProjectImportsCollector.cs | 211 ++++++++++++------ 15 files changed, 506 insertions(+), 154 deletions(-) create mode 100644 src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFileEventArgs.cs create mode 100644 src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFileEventArgsExtensions.cs create mode 100644 src/Build/Logging/BinaryLogger/Postprocessing/CleanupScope.cs create mode 100644 src/Build/Logging/BinaryLogger/Postprocessing/IBuildFileReader.cs create mode 100644 src/Build/Logging/BinaryLogger/Postprocessing/SubStream.cs diff --git a/src/Build/Logging/BinaryLogger/BinaryLogReplayEventSource.cs b/src/Build/Logging/BinaryLogger/BinaryLogReplayEventSource.cs index b21169ed135..b6128c390d2 100644 --- a/src/Build/Logging/BinaryLogger/BinaryLogReplayEventSource.cs +++ b/src/Build/Logging/BinaryLogger/BinaryLogReplayEventSource.cs @@ -16,7 +16,7 @@ namespace Microsoft.Build.Logging /// by implementing IEventSource and raising corresponding events. /// /// The class is public so that we can call it from MSBuild.exe when replaying a log file. - public sealed class BinaryLogReplayEventSource : EventArgsDispatcher + public sealed class BinaryLogReplayEventSource : EventArgsDispatcher, IEmbeddedContentSource { /// Touches the static constructor /// to ensure it initializes @@ -125,10 +125,29 @@ public void Replay(BinaryReader binaryReader, bool closeInput, CancellationToken /// A indicating the replay should stop as soon as possible. public void Replay(BuildEventArgsReader reader, CancellationToken cancellationToken) { + _fileFormatVersionRead?.Invoke(reader.FileFormatVersion); + reader.EmbeddedContentRead += _embeddedContentRead; + while (!cancellationToken.IsCancellationRequested && reader.Read() is { } instance) { Dispatch(instance); } } + + private Action? _fileFormatVersionRead; + event Action ILogVersionInfo.FileFormatVersionRead + { + add => _fileFormatVersionRead += value; + remove => _fileFormatVersionRead -= value; + } + private Action? _embeddedContentRead; + /// + event Action? IEmbeddedContentSource.EmbeddedContentRead + { + // Explicitly implemented event has to declare explicit add/remove accessors + // https://stackoverflow.com/a/2268472/2308106 + add => _embeddedContentRead += value; + remove => _embeddedContentRead -= value; + } } } diff --git a/src/Build/Logging/BinaryLogger/BinaryLogger.cs b/src/Build/Logging/BinaryLogger/BinaryLogger.cs index 9c60914d4a0..dd8dae49445 100644 --- a/src/Build/Logging/BinaryLogger/BinaryLogger.cs +++ b/src/Build/Logging/BinaryLogger/BinaryLogger.cs @@ -94,7 +94,12 @@ public enum ProjectImportsCollectionMode /// /// Create an external .ProjectImports.zip archive for the project files. /// - ZipFile + ZipFile, + + /// + /// Don't collect any files from build events, but instead replay them from the given event source (if that one supports it). + /// + Replay, } /// @@ -116,15 +121,10 @@ public enum ProjectImportsCollectionMode /// public string Parameters { get; set; } - /// - /// Initializes the logger by subscribing to events of the specified event source. - /// - public void Initialize(IEventSource eventSource) => Initialize(eventSource, null); - /// /// Initializes the logger by subscribing to events of the specified event source and embedded content source. /// - public void Initialize(IEventSource eventSource, IEmbeddedContentSource embeddedFilesSource) + public void Initialize(IEventSource eventSource) { _initialTargetOutputLogging = Environment.GetEnvironmentVariable("MSBUILDTARGETOUTPUTLOGGING"); _initialLogImports = Traits.Instance.EscapeHatches.LogProjectImports; @@ -137,14 +137,9 @@ public void Initialize(IEventSource eventSource, IEmbeddedContentSource embedded Traits.Instance.EscapeHatches.LogProjectImports = true; bool logPropertiesAndItemsAfterEvaluation = Traits.Instance.EscapeHatches.LogPropertiesAndItemsAfterEvaluation ?? true; - ProcessParameters(); - - if (embeddedFilesSource != null) - { - CollectProjectImports = ProjectImportsCollectionMode.None; - embeddedFilesSource.EmbeddedContentRead += args => - eventArgsWriter.WriteBlob(args.ContentKind.ToBinaryLogRecordKind(), args.ContentStream, args.Length); - } + bool replayInitialInfo; + ILogVersionInfo versionInfo = null; + ProcessParameters(out replayInitialInfo); try { @@ -166,7 +161,7 @@ public void Initialize(IEventSource eventSource, IEmbeddedContentSource embedded stream = new FileStream(FilePath, FileMode.Create); - if (CollectProjectImports != ProjectImportsCollectionMode.None) + if (CollectProjectImports != ProjectImportsCollectionMode.None && CollectProjectImports != ProjectImportsCollectionMode.Replay) { projectImportsCollector = new ProjectImportsCollector(FilePath, CollectProjectImports == ProjectImportsCollectionMode.ZipFile); } @@ -180,6 +175,20 @@ public void Initialize(IEventSource eventSource, IEmbeddedContentSource embedded { eventSource4.IncludeEvaluationPropertiesAndItems(); } + + if (eventSource is IEmbeddedContentSource embeddedFilesSource) + { + if (CollectProjectImports == ProjectImportsCollectionMode.Replay) + { + embeddedFilesSource.EmbeddedContentRead += args => + eventArgsWriter.WriteBlob(args.ContentKind.ToBinaryLogRecordKind(), args.ContentStream, args.Length); + } + + if (replayInitialInfo) + { + versionInfo = embeddedFilesSource; + } + } } catch (Exception e) { @@ -205,14 +214,14 @@ public void Initialize(IEventSource eventSource, IEmbeddedContentSource embedded eventArgsWriter.EmbedFile += EventArgsWriter_EmbedFile; } - if (embeddedFilesSource == null) + if (versionInfo == null) { binaryWriter.Write(FileFormatVersion); LogInitialInfo(); } else { - binaryWriter.Write(embeddedFilesSource.FileFormatVersion); + versionInfo.FileFormatVersionRead += version => binaryWriter.Write(version); } eventSource.AnyEventRaised += EventSource_AnyEventRaised; @@ -252,32 +261,18 @@ public void Shutdown() Traits.Instance.EscapeHatches.LogProjectImports = _initialLogImports; + if (projectImportsCollector != null) { projectImportsCollector.Close(); if (CollectProjectImports == ProjectImportsCollectionMode.Embed) { - var archiveFilePath = projectImportsCollector.ArchiveFilePath; + projectImportsCollector.ProcessResult( + streamToEmbed => eventArgsWriter.WriteBlob(BinaryLogRecordKind.ProjectImportArchive, streamToEmbed), + LogMessage); - // It is possible that the archive couldn't be created for some reason. - // Only embed it if it actually exists. - if (FileSystems.Default.FileExists(archiveFilePath)) - { - using (FileStream fileStream = File.OpenRead(archiveFilePath)) - { - if (fileStream.Length > int.MaxValue) - { - LogMessage("Imported files archive exceeded 2GB limit and it's not embedded."); - } - else - { - eventArgsWriter.WriteBlob(BinaryLogRecordKind.ProjectImportArchive, fileStream); - } - } - - File.Delete(archiveFilePath); - } + projectImportsCollector.DeleteArchive(); } projectImportsCollector = null; @@ -326,7 +321,7 @@ private void CollectImports(BuildEventArgs e) { projectImportsCollector.AddFile(projectArgs.ProjectFile); } - else if (e is MetaprojectGeneratedEventArgs metaprojectArgs) + else if (e is MetaprojectGeneratedEventArgs { metaprojectXml: { } } metaprojectArgs) { projectImportsCollector.AddFileFromMemory(metaprojectArgs.ProjectFile, metaprojectArgs.metaprojectXml); } @@ -341,13 +336,14 @@ private void CollectImports(BuildEventArgs e) /// /// /// - private void ProcessParameters() + private void ProcessParameters(out bool replayInitialInfo) { if (Parameters == null) { throw new LoggerException(ResourceUtilities.FormatResourceStringStripCodeAndKeyword("InvalidBinaryLoggerParameters", "")); } + replayInitialInfo = false; var parameters = Parameters.Split(MSBuildConstants.SemicolonChar, StringSplitOptions.RemoveEmptyEntries); foreach (var parameter in parameters) { @@ -363,6 +359,14 @@ private void ProcessParameters() { CollectProjectImports = ProjectImportsCollectionMode.ZipFile; } + else if (string.Equals(parameter, "ProjectImports=Replay", StringComparison.OrdinalIgnoreCase)) + { + CollectProjectImports = ProjectImportsCollectionMode.Replay; + } + else if (string.Equals(parameter, "ReplayInitialInfo", StringComparison.OrdinalIgnoreCase)) + { + replayInitialInfo = true; + } else if (parameter.EndsWith(".binlog", StringComparison.OrdinalIgnoreCase)) { FilePath = parameter; diff --git a/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs b/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs index 747a078a50c..6c53eeaa5d7 100644 --- a/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs +++ b/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs @@ -5,6 +5,8 @@ using System.Collections; using System.Collections.Generic; using System.IO; +using System.IO.Compression; +using System.Linq; using System.Reflection; using System.Text; using Microsoft.Build.BackEnd; @@ -98,7 +100,10 @@ public void Dispose() /// The subscriber must read the exactly given length of binary data from the stream - otherwise exception is raised. /// If no subscriber is attached, the data is skipped. /// - public event Action? EmbeddedContentRead; + internal event Action? EmbeddedContentRead; + + /// + public event Action? ArchiveFileEncountered; /// /// Reads the next log record from the . @@ -228,16 +233,72 @@ private static bool IsAuxiliaryRecord(BinaryLogRecordKind recordKind) private void ReadEmbeddedContent(BinaryLogRecordKind recordKind) { int length = ReadInt32(); - if (EmbeddedContentRead != null) + + if (ArchiveFileEncountered != null) { - long preEventPosition = binaryReader.BaseStream.CanSeek ? binaryReader.BaseStream.Position : 0; - EmbeddedContentRead(new EmbeddedContentEventArgs(recordKind.ToEmbeddedContentKind(), binaryReader.BaseStream, length)); - long postEventPosition = binaryReader.BaseStream.CanSeek ? binaryReader.BaseStream.Position : length; - if (postEventPosition - preEventPosition != length) + // We could create ZipArchive over the target stream, and write to that directly, + // however, binlog format needs to know stream size upfront - which is unknown, + // so we would need to write the size after - and that would require the target stream to be seekable (which it's not) + ProjectImportsCollector? projectImportsCollector = null; + + if (EmbeddedContentRead != null) + { + projectImportsCollector = + new ProjectImportsCollector(Path.GetRandomFileName(), false, runOnBackground: false); + } + + Stream embeddedStream = new SubStream(binaryReader.BaseStream, length); + + // We are intentionally not grace handling corrupt embedded stream + + using var zipArchive = new ZipArchive(embeddedStream, ZipArchiveMode.Read); + + foreach (var entry in zipArchive.Entries/*.OrderBy(e => e.LastWriteTime)*/) + { + var file = ArchiveFile.From(entry); + ArchiveFileEventArgs archiveFileEventArgs = new(file); + // ArchiveFileEventArgs is not IDisposable as we do not want to clutter exposed API + using var cleanupScope = new CleanupScope(archiveFileEventArgs.Dispose); + ArchiveFileEncountered(archiveFileEventArgs); + + if (projectImportsCollector != null) + { + var resultFile = archiveFileEventArgs.ObtainArchiveFile(); + + if (resultFile.CanUseReader) + { + projectImportsCollector.AddFileFromMemory( + resultFile.FullPath, + resultFile.GetContentReader().BaseStream, + makePathAbsolute: false, + entryCreationStamp: entry.LastWriteTime); + } + else + { + projectImportsCollector.AddFileFromMemory( + resultFile.FullPath, + resultFile.GetContent(), + encoding: resultFile.Encoding, + makePathAbsolute: false, + entryCreationStamp: entry.LastWriteTime); + } + } + } + + if (EmbeddedContentRead != null) { - throw new InvalidDataException($"The {nameof(EmbeddedContentRead)} event handler must read exactly {length} bytes from the stream."); + projectImportsCollector!.ProcessResult( + streamToEmbed => EmbeddedContentRead(new EmbeddedContentEventArgs(EmbeddedContentKind.ProjectImportArchive, streamToEmbed)), + error => throw new InvalidDataException(error)); + projectImportsCollector.DeleteArchive(); } } + else if (EmbeddedContentRead != null) + { + EmbeddedContentRead(new EmbeddedContentEventArgs( + recordKind.ToEmbeddedContentKind(), + new SubStream(binaryReader.BaseStream, length))); + } else { if (binaryReader.BaseStream.CanSeek) diff --git a/src/Build/Logging/BinaryLogger/BuildEventArgsWriter.cs b/src/Build/Logging/BinaryLogger/BuildEventArgsWriter.cs index d962530bcc1..99ffe22d96d 100644 --- a/src/Build/Logging/BinaryLogger/BuildEventArgsWriter.cs +++ b/src/Build/Logging/BinaryLogger/BuildEventArgsWriter.cs @@ -230,13 +230,7 @@ private void WriteCore(BuildEventArgs e) } } - public void WriteBlob(BinaryLogRecordKind kind, Stream stream, int length) - => WriteBlobImpl(kind, stream, length); - - public void WriteBlob(BinaryLogRecordKind kind, Stream stream) - => WriteBlobImpl(kind, stream, null); - - private void WriteBlobImpl(BinaryLogRecordKind kind, Stream stream, int? length) + public void WriteBlob(BinaryLogRecordKind kind, Stream stream, int? length = null) { if (stream.CanSeek && stream.Length > int.MaxValue) { @@ -1121,9 +1115,10 @@ private void Write(byte[] bytes) private void Write(Stream stream, int? length) { + Stream destinationStream = binaryWriter.BaseStream; if (length == null) { - stream.CopyTo(binaryWriter.BaseStream); + stream.CopyTo(destinationStream); return; } @@ -1139,7 +1134,7 @@ private void Write(Stream stream, int? length) length > 0 && (bytesRead = stream.Read(buffer, 0, Math.Min(buffer.Length, length.Value))) != 0) { - binaryWriter.BaseStream.Write(buffer, 0, bytesRead); + destinationStream.Write(buffer, 0, bytesRead); length -= bytesRead; } } diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFile.cs b/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFile.cs index 59868eb43dd..5f3f75ab761 100644 --- a/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFile.cs +++ b/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFile.cs @@ -4,29 +4,46 @@ using System; using System.IO; using System.IO.Compression; +using System.Text; namespace Microsoft.Build.Logging { - public class ArchiveFile : IDisposable + public class ArchiveFile { - public ArchiveFile(string fullPath, StreamReader contentReader) + public ArchiveFile(string fullPath, Stream contentStream) { FullPath = fullPath; - _contentReader = contentReader; + // We need to specify encoding without preamble - as then StreamReader will + // automatically adjust the encoding to match the preamble (if present). + _contentReader = new StreamReader(contentStream, new System.Text.UTF8Encoding(false)); + } + + public ArchiveFile(string fullPath, string content, Encoding? contentEncoding = null) + { + FullPath = fullPath; + _content = content; + _stringAcquired = true; + _contentReader = StreamReader.Null; + _stringEncoding = contentEncoding ?? Encoding.UTF8; } public static ArchiveFile From(ZipArchiveEntry entry) { - return new ArchiveFile(CalculateArchivePath(entry.FullName), new StreamReader(entry.Open())); + return new ArchiveFile(entry.FullName, entry.Open()); } public string FullPath { get; } + public Encoding Encoding => _stringEncoding ?? _contentReader.CurrentEncoding; + + public bool CanUseReader => !_stringAcquired; + public bool CanUseString => !_streamAcquired; + public StreamReader GetContentReader() { if (_stringAcquired) { - throw new InvalidOperationException("Content already acquired as string via GetContent."); + throw new InvalidOperationException("Content already acquired as string via GetContent or initialized as string only."); } _streamAcquired = true; @@ -53,26 +70,10 @@ public string GetContent() private bool _stringAcquired; private readonly StreamReader _contentReader; private string? _content; + private readonly Encoding? _stringEncoding; - public static string CalculateArchivePath(string filePath) - { - string archivePath = filePath; - - if (filePath.Contains(":") || (!filePath.StartsWith("\\") && !filePath.StartsWith("/"))) - { - archivePath = archivePath.Replace(":", ""); - archivePath = archivePath.Replace("/", "\\"); - archivePath = archivePath.Replace("\\\\", "\\"); - } - else - { - archivePath = archivePath.Replace("\\", "/"); - archivePath = archivePath.Replace("//", "/"); - } - - return archivePath; - } - - public void Dispose() => _contentReader.Dispose(); + // Intentionally not exposing this publicly (e.g. as IDisposable implementation) + // as we don't want to user to be bothered with ownership and disposing concerns. + internal void Dispose() => _contentReader.Dispose(); } } diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFileEventArgs.cs b/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFileEventArgs.cs new file mode 100644 index 00000000000..7045f261d86 --- /dev/null +++ b/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFileEventArgs.cs @@ -0,0 +1,47 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; + +namespace Microsoft.Build.Logging; + +public class ArchiveFileEventArgs : EventArgs +{ + private ArchiveFile _archiveFile; + private bool _resultSet; + private Action _disposeAction; + + public ArchiveFileEventArgs(ArchiveFile archiveFile) => + (_archiveFile, _resultSet, _disposeAction) = (archiveFile, true, archiveFile.Dispose); + + public ArchiveFile ObtainArchiveFile() + { + if (!_resultSet) + { + throw new InvalidOperationException( + "ArchiveFile was obtained, but the final edited version was not set."); + } + + _resultSet = false; + return _archiveFile; + } + + public void SetResult(string resultPath, Stream resultStream) + { + _archiveFile = new ArchiveFile(resultPath, resultStream); + _disposeAction += _archiveFile.Dispose; + _resultSet = true; + } + + public void SetResult(string resultPath, string resultContent) + { + _archiveFile = new ArchiveFile(resultPath, resultContent, _archiveFile.Encoding); + _disposeAction += _archiveFile.Dispose; + _resultSet = true; + } + + // Intentionally not exposing this publicly (e.g. as IDisposable implementation) + // as we don't want to user to be bothered with ownership and disposing concerns. + internal void Dispose() => _disposeAction(); +} diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFileEventArgsExtensions.cs b/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFileEventArgsExtensions.cs new file mode 100644 index 00000000000..818cffaa91a --- /dev/null +++ b/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFileEventArgsExtensions.cs @@ -0,0 +1,23 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; + +namespace Microsoft.Build.Logging; + +public static class ArchiveFileEventArgsExtensions +{ + public static Action ToArchiveFileHandler(this Action stringHandler) + { + return args => + { + var archiveFile = args.ObtainArchiveFile(); + var pathArgs = new StringReadEventArgs(archiveFile.FullPath); + stringHandler(pathArgs); + var contentArgs = new StringReadEventArgs(archiveFile.GetContent()); + stringHandler(contentArgs); + + args.SetResult(pathArgs.StringToBeUsed, contentArgs.StringToBeUsed); + }; + } +} diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/CleanupScope.cs b/src/Build/Logging/BinaryLogger/Postprocessing/CleanupScope.cs new file mode 100644 index 00000000000..bef26ff4d13 --- /dev/null +++ b/src/Build/Logging/BinaryLogger/Postprocessing/CleanupScope.cs @@ -0,0 +1,15 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; + +namespace Microsoft.Build.Logging; + +internal class CleanupScope : IDisposable +{ + private readonly Action _disposeAction; + + public CleanupScope(Action disposeAction) => _disposeAction = disposeAction; + + public void Dispose() => _disposeAction(); +} diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/EmbeddedContentEventArgs.cs b/src/Build/Logging/BinaryLogger/Postprocessing/EmbeddedContentEventArgs.cs index b5d6e985783..ce372c888d3 100644 --- a/src/Build/Logging/BinaryLogger/Postprocessing/EmbeddedContentEventArgs.cs +++ b/src/Build/Logging/BinaryLogger/Postprocessing/EmbeddedContentEventArgs.cs @@ -15,8 +15,14 @@ public EmbeddedContentEventArgs(EmbeddedContentKind contentKind, Stream contentS Length = length; } + public EmbeddedContentEventArgs(EmbeddedContentKind contentKind, Stream contentStream) + { + ContentKind = contentKind; + ContentStream = contentStream; + } + public EmbeddedContentKind ContentKind { get; } public Stream ContentStream { get; } - public int Length { get; } + public int? Length { get; } } } diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/IBuildEventArgsReaderNotifications.cs b/src/Build/Logging/BinaryLogger/Postprocessing/IBuildEventArgsReaderNotifications.cs index 41ec404904e..5f49119570f 100644 --- a/src/Build/Logging/BinaryLogger/Postprocessing/IBuildEventArgsReaderNotifications.cs +++ b/src/Build/Logging/BinaryLogger/Postprocessing/IBuildEventArgsReaderNotifications.cs @@ -6,7 +6,8 @@ namespace Microsoft.Build.Logging /// /// An interface for notifications from BuildEventArgsReader /// - public interface IBuildEventArgsReaderNotifications : IBuildEventStringsReader, IEmbeddedContentSource + // todo: IEmbeddedContentSource should not be here ideally + public interface IBuildEventArgsReaderNotifications : IBuildEventStringsReader, IBuildFileReader { /* For future use */ } diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/IBuildEventStringsReader.cs b/src/Build/Logging/BinaryLogger/Postprocessing/IBuildEventStringsReader.cs index e9e7651ee78..bf3d54f8ff8 100644 --- a/src/Build/Logging/BinaryLogger/Postprocessing/IBuildEventStringsReader.cs +++ b/src/Build/Logging/BinaryLogger/Postprocessing/IBuildEventStringsReader.cs @@ -16,11 +16,5 @@ public interface IBuildEventStringsReader /// The passed event arg can be reused and should not be stored. /// public event Action? StringReadDone; - - /// - /// An event that allows the caller to be notified when a string is encountered in the binary log. - /// BinaryReader passed in ctor is at the beginning of the string at this point. - /// - public event Action? StringEncountered; } } diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/IBuildFileReader.cs b/src/Build/Logging/BinaryLogger/Postprocessing/IBuildFileReader.cs new file mode 100644 index 00000000000..9910ee1a06a --- /dev/null +++ b/src/Build/Logging/BinaryLogger/Postprocessing/IBuildFileReader.cs @@ -0,0 +1,34 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; + +namespace Microsoft.Build.Logging; + +public interface IBuildFileReader +{ + /// + /// An event that allows the caller to be notified when an embedded file is encountered in the binary log. + /// Subscribing to this event obligates the subscriber to read the file content and set the resulting content + /// via or . + /// When subscriber is OK with greedy reading entire content of the file, it can simplify subscribing to this event, + /// by using handler with same signature as handler for and wrapping it via + /// extension. + /// + /// + /// + /// private void OnStringReadDone(StringReadEventArgs e) + /// { + /// e.StringToBeUsed = e.StringToBeUsed.Replace("foo", "bar"); + /// } + /// + /// private void SubscribeToEvents() + /// { + /// reader.StringReadDone += OnStringReadDone; + /// reader.ArchiveFileEncountered += ((Action<StringReadEventArgs>)OnStringReadDone).ToArchiveFileHandler(); + /// } + /// + /// + /// + public event Action? ArchiveFileEncountered; +} diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/IEmbeddedContentSource.cs b/src/Build/Logging/BinaryLogger/Postprocessing/IEmbeddedContentSource.cs index e0080148b1a..eb9262939fa 100644 --- a/src/Build/Logging/BinaryLogger/Postprocessing/IEmbeddedContentSource.cs +++ b/src/Build/Logging/BinaryLogger/Postprocessing/IEmbeddedContentSource.cs @@ -5,9 +5,20 @@ namespace Microsoft.Build.Logging { - public interface IEmbeddedContentSource + + internal interface ILogVersionInfo + { + event Action FileFormatVersionRead; + } + + internal interface IEmbeddedContentSource : ILogVersionInfo { - int FileFormatVersion { get; } + + /// + /// Raised when the log reader encounters a project import archive (embedded content) in the stream. + /// The subscriber must read the exactly given length of binary data from the stream - otherwise exception is raised. + /// If no subscriber is attached, the data is skipped. + /// event Action EmbeddedContentRead; } } diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/SubStream.cs b/src/Build/Logging/BinaryLogger/Postprocessing/SubStream.cs new file mode 100644 index 00000000000..277c9ac66c0 --- /dev/null +++ b/src/Build/Logging/BinaryLogger/Postprocessing/SubStream.cs @@ -0,0 +1,56 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Microsoft.Build.Logging +{ + /// + /// Bounded read-only, forward-only view over an underlying stream. + /// + internal class SubStream : Stream + { + // Do not Dispose/Close on Dispose/Close !! + private readonly Stream _stream; + private readonly long _length; + private long _position; + + public SubStream(Stream stream, long length) + { + _stream = stream; + _length = length; + + if (!stream.CanRead) + { + throw new InvalidOperationException("Stream must be readable."); + } + } + + public override bool CanRead => true; + + public override bool CanSeek => false; + + public override bool CanWrite => false; + + public override long Length => _length; + + public override long Position { get => _position; set => throw new NotImplementedException(); } + + public override void Flush() { } + public override int Read(byte[] buffer, int offset, int count) + { + count = Math.Min((int)Math.Max(Length - _position, 0), count); + int read = _stream.Read(buffer, offset, count); + _position += read; + return read; + } + public override long Seek(long offset, SeekOrigin origin) => throw new NotImplementedException(); + public override void SetLength(long value) => throw new NotImplementedException(); + public override void Write(byte[] buffer, int offset, int count) => throw new NotImplementedException(); + } +} diff --git a/src/Build/Logging/BinaryLogger/ProjectImportsCollector.cs b/src/Build/Logging/BinaryLogger/ProjectImportsCollector.cs index 27ededae8cc..24e86af2991 100644 --- a/src/Build/Logging/BinaryLogger/ProjectImportsCollector.cs +++ b/src/Build/Logging/BinaryLogger/ProjectImportsCollector.cs @@ -8,8 +8,7 @@ using System.Text; using System.Threading.Tasks; using Microsoft.Build.Shared; - -#nullable disable +using Microsoft.Build.Shared.FileSystem; namespace Microsoft.Build.Logging { @@ -21,10 +20,10 @@ namespace Microsoft.Build.Logging /// internal class ProjectImportsCollector { - private Stream _fileStream; - private ZipArchive _zipArchive; - - public string ArchiveFilePath { get; } + private Stream? _fileStream; + private ZipArchive? _zipArchive; + private readonly string _archiveFilePath; + private readonly bool _runOnBackground; /// /// Avoid visiting each file more than once. @@ -34,12 +33,16 @@ internal class ProjectImportsCollector // this will form a chain of file write tasks, running sequentially on a background thread private Task _currentTask = Task.CompletedTask; - public ProjectImportsCollector(string logFilePath, bool createFile, string sourcesArchiveExtension = ".ProjectImports.zip") + public ProjectImportsCollector( + string logFilePath, + bool createFile, + string sourcesArchiveExtension = ".ProjectImports.zip", + bool runOnBackground = true) { if (createFile) { // Archive file will be stored alongside the binlog - ArchiveFilePath = Path.ChangeExtension(logFilePath, sourcesArchiveExtension); + _archiveFilePath = Path.ChangeExtension(logFilePath, sourcesArchiveExtension); } else { @@ -50,7 +53,7 @@ public ProjectImportsCollector(string logFilePath, bool createFile, string sourc } // Archive file will be temporarily stored in MSBuild cache folder and deleted when no longer needed - ArchiveFilePath = Path.Combine( + _archiveFilePath = Path.Combine( cacheDirectory, Path.ChangeExtension( Path.GetFileName(logFilePath), @@ -59,7 +62,7 @@ public ProjectImportsCollector(string logFilePath, bool createFile, string sourc try { - _fileStream = new FileStream(ArchiveFilePath, FileMode.Create, FileAccess.ReadWrite, FileShare.Delete); + _fileStream = new FileStream(_archiveFilePath, FileMode.Create, FileAccess.ReadWrite, FileShare.Delete); _zipArchive = new ZipArchive(_fileStream, ZipArchiveMode.Create); } catch @@ -69,50 +72,72 @@ public ProjectImportsCollector(string logFilePath, bool createFile, string sourc _fileStream = null; _zipArchive = null; } + _runOnBackground = runOnBackground; + } + + public void AddFile(string? filePath) + { + AddFileHelper(filePath, AddFileCore); } - public void AddFile(string filePath) + public void AddFileFromMemory( + string? filePath, + string data, + Encoding? encoding = null, + DateTimeOffset? entryCreationStamp = null, + bool makePathAbsolute = true) + { + AddFileHelper(filePath, path => + AddFileFromMemoryCore(path, data, encoding ?? Encoding.UTF8, makePathAbsolute, entryCreationStamp)); + } + + public void AddFileFromMemory( + string? filePath, + Stream data, + DateTimeOffset? entryCreationStamp = null, + bool makePathAbsolute = true) + { + AddFileHelper(filePath, path => AddFileFromMemoryCore(path, data, makePathAbsolute, entryCreationStamp)); + } + + private void AddFileHelper( + string? filePath, + Action addFileWorker) { if (filePath != null && _fileStream != null) { + Action addFileAction = WrapWithExceptionSwallowing(() => addFileWorker(filePath)); + lock (_fileStream) { - // enqueue the task to add a file and return quickly - // to avoid holding up the current thread - _currentTask = _currentTask.ContinueWith(t => + if (_runOnBackground) + { + // enqueue the task to add a file and return quickly + // to avoid holding up the current thread + _currentTask = _currentTask.ContinueWith( + t => { addFileAction(); }, + TaskScheduler.Default); + } + else { - try - { - AddFileCore(filePath); - } - catch - { - } - }, TaskScheduler.Default); + addFileAction(); + } } } } - public void AddFileFromMemory(string filePath, string data) + private Action WrapWithExceptionSwallowing(Action action) { - if (filePath != null && data != null && _fileStream != null) + return () => { - lock (_fileStream) + try { - // enqueue the task to add a file and return quickly - // to avoid holding up the current thread - _currentTask = _currentTask.ContinueWith(t => - { - try - { - AddFileFromMemoryCore(filePath, data); - } - catch - { - } - }, TaskScheduler.Default); + action(); } - } + catch + { + } + }; } /// @@ -122,61 +147,94 @@ public void AddFileFromMemory(string filePath, string data) private void AddFileCore(string filePath) { // quick check to avoid repeated disk access for Exists etc. - if (_processedFiles.Contains(filePath)) + if (!ShouldAddFile(ref filePath, true, true)) { return; } - if (!File.Exists(filePath)) + using FileStream content = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete); + AddFileData(filePath, content, null); + } + + /// + /// This method doesn't need locking/synchronization because it's only called + /// from a task that is chained linearly + /// + private void AddFileFromMemoryCore(string filePath, string data, Encoding encoding, bool makePathAbsolute, DateTimeOffset? entryCreationStamp) + { + // quick check to avoid repeated disk access for Exists etc. + if (!ShouldAddFile(ref filePath, false, makePathAbsolute)) { - _processedFiles.Add(filePath); return; } - filePath = Path.GetFullPath(filePath); + AddFileData(filePath, data, encoding, entryCreationStamp); + } - // if the file is already included, don't include it again - if (!_processedFiles.Add(filePath)) + private void AddFileFromMemoryCore(string filePath, Stream data, bool makePathAbsolute, DateTimeOffset? entryCreationStamp) + { + // quick check to avoid repeated disk access for Exists etc. + if (!ShouldAddFile(ref filePath, false, makePathAbsolute)) { return; } - using FileStream content = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete); - using Stream entryStream = OpenArchiveEntry(filePath); - content.CopyTo(entryStream); + AddFileData(filePath, data, entryCreationStamp); } - /// - /// This method doesn't need locking/synchronization because it's only called - /// from a task that is chained linearly - /// - private void AddFileFromMemoryCore(string filePath, string data) + private void AddFileData(string filePath, Stream data, DateTimeOffset? entryCreationStamp) + { + using Stream entryStream = OpenArchiveEntry(filePath, entryCreationStamp); + data.CopyTo(entryStream); + } + + private void AddFileData(string filePath, string data, Encoding encoding, DateTimeOffset? entryCreationStamp) + { + using Stream entryStream = OpenArchiveEntry(filePath, entryCreationStamp); + using MemoryStream memoryStream = new MemoryStream(); + // We need writer as encoding.GetBytes() isn't obliged to output preamble + // We cannot write directly to entryStream (preamble is written separately) as it's compressed differnetly, then writing the whole stream at once + using StreamWriter writer = new StreamWriter(memoryStream, encoding); + writer.Write(data); + writer.Flush(); + memoryStream.Position = 0; + memoryStream.CopyTo(entryStream); + } + + private bool ShouldAddFile(ref string filePath, bool checkFileExistence, bool makeAbsolute) { // quick check to avoid repeated disk access for Exists etc. if (_processedFiles.Contains(filePath)) { - return; + return false; } - filePath = Path.GetFullPath(filePath); - - // if the file is already included, don't include it again - if (!_processedFiles.Add(filePath)) + if (checkFileExistence && !File.Exists(filePath)) { - return; + _processedFiles.Add(filePath); + return false; } - using (Stream entryStream = OpenArchiveEntry(filePath)) - using (var content = new MemoryStream(Encoding.UTF8.GetBytes(data))) + // Only make the path absolute if it's request. In the replay scenario, the file entries + // are read from zip archive - where ':' is stripped and path can then seem relative. + if (makeAbsolute) { - content.CopyTo(entryStream); + filePath = Path.GetFullPath(filePath); } + + // if the file is already included, don't include it again + return _processedFiles.Add(filePath); } - private Stream OpenArchiveEntry(string filePath) + private Stream OpenArchiveEntry(string filePath, DateTimeOffset? entryCreationStamp) { string archivePath = CalculateArchivePath(filePath); - var archiveEntry = _zipArchive.CreateEntry(archivePath); + var archiveEntry = _zipArchive!.CreateEntry(archivePath); + if (entryCreationStamp.HasValue) + { + archiveEntry.LastWriteTime = entryCreationStamp.Value; + } + return archiveEntry.Open(); } @@ -191,6 +249,27 @@ private static string CalculateArchivePath(string filePath) return archivePath; } + public void ProcessResult(Action consumeStream, Action onError) + { + Close(); + + // It is possible that the archive couldn't be created for some reason. + // Only embed it if it actually exists. + if (FileSystems.Default.FileExists(_archiveFilePath)) + { + using FileStream fileStream = File.OpenRead(_archiveFilePath); + + if (fileStream.Length > int.MaxValue) + { + onError("Imported files archive exceeded 2GB limit and it's not embedded."); + } + else + { + consumeStream(fileStream); + } + } + } + public void Close() { // wait for all pending file writes to complete @@ -208,5 +287,11 @@ public void Close() _fileStream = null; } } + + public void DeleteArchive() + { + Close(); + File.Delete(_archiveFilePath); + } } } From 699e5d44b1f1fb2202f3821f4d0eb184b6a09cb9 Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Mon, 18 Sep 2023 14:26:55 +0200 Subject: [PATCH 03/49] Decrease scope where accidentally exposed --- .../BinaryLogger/Postprocessing/EmbeddedContentEventArgs.cs | 2 +- .../Logging/BinaryLogger/Postprocessing/EmbeddedContentKind.cs | 2 +- .../Postprocessing/IBuildEventArgsReaderNotifications.cs | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/EmbeddedContentEventArgs.cs b/src/Build/Logging/BinaryLogger/Postprocessing/EmbeddedContentEventArgs.cs index ce372c888d3..68969f2af4a 100644 --- a/src/Build/Logging/BinaryLogger/Postprocessing/EmbeddedContentEventArgs.cs +++ b/src/Build/Logging/BinaryLogger/Postprocessing/EmbeddedContentEventArgs.cs @@ -6,7 +6,7 @@ namespace Microsoft.Build.Logging { - public sealed class EmbeddedContentEventArgs : EventArgs + internal sealed class EmbeddedContentEventArgs : EventArgs { public EmbeddedContentEventArgs(EmbeddedContentKind contentKind, Stream contentStream, int length) { diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/EmbeddedContentKind.cs b/src/Build/Logging/BinaryLogger/Postprocessing/EmbeddedContentKind.cs index 7f694c91363..2fca5d7eaa3 100644 --- a/src/Build/Logging/BinaryLogger/Postprocessing/EmbeddedContentKind.cs +++ b/src/Build/Logging/BinaryLogger/Postprocessing/EmbeddedContentKind.cs @@ -9,7 +9,7 @@ namespace Microsoft.Build.Logging { - public enum EmbeddedContentKind + internal enum EmbeddedContentKind { Unknown = -1, ProjectImportArchive = 17, diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/IBuildEventArgsReaderNotifications.cs b/src/Build/Logging/BinaryLogger/Postprocessing/IBuildEventArgsReaderNotifications.cs index 5f49119570f..13bc343362a 100644 --- a/src/Build/Logging/BinaryLogger/Postprocessing/IBuildEventArgsReaderNotifications.cs +++ b/src/Build/Logging/BinaryLogger/Postprocessing/IBuildEventArgsReaderNotifications.cs @@ -6,7 +6,6 @@ namespace Microsoft.Build.Logging /// /// An interface for notifications from BuildEventArgsReader /// - // todo: IEmbeddedContentSource should not be here ideally public interface IBuildEventArgsReaderNotifications : IBuildEventStringsReader, IBuildFileReader { /* For future use */ From 2a75fb69deaf169e2134765f2460d63c92729d51 Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Tue, 19 Sep 2023 15:17:04 +0200 Subject: [PATCH 04/49] Sealing types --- src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFile.cs | 2 +- .../Logging/BinaryLogger/Postprocessing/ArchiveFileEventArgs.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFile.cs b/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFile.cs index 5f3f75ab761..6c6a9daf26c 100644 --- a/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFile.cs +++ b/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFile.cs @@ -8,7 +8,7 @@ namespace Microsoft.Build.Logging { - public class ArchiveFile + public sealed class ArchiveFile { public ArchiveFile(string fullPath, Stream contentStream) { diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFileEventArgs.cs b/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFileEventArgs.cs index 7045f261d86..45bc252a066 100644 --- a/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFileEventArgs.cs +++ b/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFileEventArgs.cs @@ -6,7 +6,7 @@ namespace Microsoft.Build.Logging; -public class ArchiveFileEventArgs : EventArgs +public sealed class ArchiveFileEventArgs : EventArgs { private ArchiveFile _archiveFile; private bool _resultSet; From 2b83376ec17545381e438ac1e5583384df3c8992 Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Wed, 20 Sep 2023 16:50:12 +0200 Subject: [PATCH 05/49] Adjust visibility, add comments --- .../BinaryLogger/Postprocessing/ArchiveFile.cs | 14 +++++++++++++- .../Postprocessing/ArchiveFileEventArgs.cs | 7 +++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFile.cs b/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFile.cs index 6c6a9daf26c..5e06a41362a 100644 --- a/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFile.cs +++ b/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFile.cs @@ -27,7 +27,7 @@ public ArchiveFile(string fullPath, string content, Encoding? contentEncoding = _stringEncoding = contentEncoding ?? Encoding.UTF8; } - public static ArchiveFile From(ZipArchiveEntry entry) + internal static ArchiveFile From(ZipArchiveEntry entry) { return new ArchiveFile(entry.FullName, entry.Open()); } @@ -39,6 +39,12 @@ public static ArchiveFile From(ZipArchiveEntry entry) public bool CanUseReader => !_stringAcquired; public bool CanUseString => !_streamAcquired; + /// + /// Fetches the file content as a stream reader (forward only). + /// This prevents the content to be read as string. + /// + /// + /// public StreamReader GetContentReader() { if (_stringAcquired) @@ -50,6 +56,12 @@ public StreamReader GetContentReader() return _contentReader; } + /// + /// Fetches the file content as a string. + /// This prevents the content to be fetched via StreamReader. + /// + /// + /// public string GetContent() { if (_streamAcquired) diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFileEventArgs.cs b/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFileEventArgs.cs index 45bc252a066..120362bcf55 100644 --- a/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFileEventArgs.cs +++ b/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFileEventArgs.cs @@ -15,6 +15,13 @@ public sealed class ArchiveFileEventArgs : EventArgs public ArchiveFileEventArgs(ArchiveFile archiveFile) => (_archiveFile, _resultSet, _disposeAction) = (archiveFile, true, archiveFile.Dispose); + /// + /// Acquires the instance. This method can only be called once and + /// or must be called afterwards + /// (this is because the embedded files are stored as forward only stream - reading them prevents re-reads). + /// + /// + /// public ArchiveFile ObtainArchiveFile() { if (!_resultSet) From b1c340eea759d16a307792619259287e7677960c Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Thu, 21 Sep 2023 12:07:47 +0200 Subject: [PATCH 06/49] Cache encoding --- .../Logging/BinaryLogger/Postprocessing/ArchiveFile.cs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFile.cs b/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFile.cs index 5e06a41362a..88e80cd9d59 100644 --- a/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFile.cs +++ b/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFile.cs @@ -10,12 +10,15 @@ namespace Microsoft.Build.Logging { public sealed class ArchiveFile { + // We need to specify encoding without preamble - as then StreamReader will + // automatically adjust the encoding to match the preamble (if present). + // It will as well change to other encoding if detected. + private static readonly Encoding s_utf8WithoutPreamble = new UTF8Encoding(false); + public ArchiveFile(string fullPath, Stream contentStream) { FullPath = fullPath; - // We need to specify encoding without preamble - as then StreamReader will - // automatically adjust the encoding to match the preamble (if present). - _contentReader = new StreamReader(contentStream, new System.Text.UTF8Encoding(false)); + _contentReader = new StreamReader(contentStream, s_utf8WithoutPreamble); } public ArchiveFile(string fullPath, string content, Encoding? contentEncoding = null) From cdca4e823b85f8ca92e256ed0e6b62a86a116e5d Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Mon, 2 Oct 2023 14:12:58 +0200 Subject: [PATCH 07/49] Fix code after merge --- .../Logging/BinaryLogger/BuildEventArgsReader.cs | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs b/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs index 6c53eeaa5d7..686152b6c62 100644 --- a/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs +++ b/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs @@ -87,12 +87,6 @@ public void Dispose() /// public event Action? StringReadDone; - /// - /// An event that allows the caller to be notified when a string is encountered in the binary log. - /// BinaryReader passed in ctor is at the beginning of the string at this point. - /// - public event Action? StringEncountered; - public int FileFormatVersion => fileFormatVersion; /// @@ -512,7 +506,7 @@ private BuildEventArgs ReadProjectEvaluationFinishedEventArgs() if (fileFormatVersion >= 12) { - if (fileFormatVersion < 17) + if (fileFormatVersion < 18) { // Throw away, but need to advance past it ReadBoolean(); @@ -568,7 +562,7 @@ private BuildEventArgs ReadProjectStartedEventArgs() if (fileFormatVersion > 6) { - if (fileFormatVersion < 17) + if (fileFormatVersion < 18) { // Throw away, but need to advance past it ReadBoolean(); @@ -1399,7 +1393,6 @@ private ITaskItem ReadTaskItem() private string ReadString() { - this.StringEncountered?.Invoke(); string text = binaryReader.ReadString(); if (this.StringReadDone != null) { From c8bf570f1fb4755a43897d895197315a3ace66c1 Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Wed, 4 Oct 2023 10:28:27 +0200 Subject: [PATCH 08/49] Fix after merge --- src/Build/Logging/BinaryLogger/BinaryLogger.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Build/Logging/BinaryLogger/BinaryLogger.cs b/src/Build/Logging/BinaryLogger/BinaryLogger.cs index dd8dae49445..94b28d10eb2 100644 --- a/src/Build/Logging/BinaryLogger/BinaryLogger.cs +++ b/src/Build/Logging/BinaryLogger/BinaryLogger.cs @@ -63,9 +63,10 @@ public sealed class BinaryLogger : ILogger // - AssemblyLoadBuildEventArgs // version 17: // - Added extended data for types implementing IExtendedBuildEventArgs + // version 18: // - Making ProjectStartedEventArgs, ProjectEvaluationFinishedEventArgs, AssemblyLoadBuildEventArgs equal // between de/serialization roundtrips. - internal const int FileFormatVersion = 17; + internal const int FileFormatVersion = 18; private Stream stream; private BinaryWriter binaryWriter; From b51f32078994d91f82d85a741e21afa6d5d2901d Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Fri, 22 Sep 2023 21:33:19 +0200 Subject: [PATCH 09/49] Add Binlog BuildEvents offsets for forward compatibility and event buffers reading --- .../BinaryLogReplayEventSource.cs | 249 +++++++++++-- .../BinaryLogger/BuildEventArgsReader.cs | 342 +++++++++++++----- .../BinaryLogger/BuildEventArgsWriter.cs | 167 +++++---- .../Postprocessing/StreamExtensions.cs | 57 +++ .../Postprocessing/TransparentReadStream.cs | 112 ++++++ 5 files changed, 734 insertions(+), 193 deletions(-) create mode 100644 src/Build/Logging/BinaryLogger/Postprocessing/StreamExtensions.cs create mode 100644 src/Build/Logging/BinaryLogger/Postprocessing/TransparentReadStream.cs diff --git a/src/Build/Logging/BinaryLogger/BinaryLogReplayEventSource.cs b/src/Build/Logging/BinaryLogger/BinaryLogReplayEventSource.cs index b6128c390d2..6524d3a81a8 100644 --- a/src/Build/Logging/BinaryLogger/BinaryLogReplayEventSource.cs +++ b/src/Build/Logging/BinaryLogger/BinaryLogReplayEventSource.cs @@ -16,25 +16,197 @@ namespace Microsoft.Build.Logging /// by implementing IEventSource and raising corresponding events. /// /// The class is public so that we can call it from MSBuild.exe when replaying a log file. - public sealed class BinaryLogReplayEventSource : EventArgsDispatcher, IEmbeddedContentSource + public sealed class BinaryLogReplayEventSource : BinaryLogReplayEventSourceBase, IEventSource, IEmbeddedContentSource { - /// Touches the static constructor - /// to ensure it initializes - /// and - static BinaryLogReplayEventSource() + public BinaryLogReplayEventSource() + : base(true) { } + + public bool AllowForwardCompatibility { - _ = ItemGroupLoggingHelper.ItemGroupIncludeLogMessagePrefix; + get => AllowForwardCompatibilityInternal; + set => AllowForwardCompatibilityInternal = value; + } + + public event Action? OnError + { + add => OnErrorInternal += value; + remove => OnErrorInternal -= value; + } + + public event Action? BuildEventReceived + { + add => BuildEventReceivedInternal += value; + remove => BuildEventReceivedInternal -= value; } /// - /// Read the provided binary log file and raise corresponding events for each BuildEventArgs + /// This event is raised for all BuildEventArgs objects after a more type-specific event /// - /// The full file path of the binary log file - public void Replay(string sourceFilePath) + public event AnyEventHandler? AnyEventRaised; + + /// + /// Raised for BuildStatusEventArgs instances + /// + public event BuildStatusEventHandler? StatusEventRaised; + + /// + /// Raised for CustomBuildEventArgs instances + /// + public event CustomBuildEventHandler? CustomEventRaised; + + /// + /// Raised for BuildStartedEventArgs instances + /// + public event BuildStartedEventHandler? BuildStarted; + + /// + /// Raised for BuildFinishedEventArgs instances + /// + public event BuildFinishedEventHandler? BuildFinished; + + /// + /// Raised for ProjectStartedEventArgs instances + /// + public event ProjectStartedEventHandler? ProjectStarted; + + /// + /// Raised for ProjectFinishedEventArgs instances + /// + public event ProjectFinishedEventHandler? ProjectFinished; + + /// + /// Raised for TargetStartedEventArgs instances + /// + public event TargetStartedEventHandler? TargetStarted; + + /// + /// Raised for TargetFinishedEventArgs instances + /// + public event TargetFinishedEventHandler? TargetFinished; + + /// + /// Raised for TaskStartedEventArgs instances + /// + public event TaskStartedEventHandler? TaskStarted; + + /// + /// Raised for TaskFinishedEventArgs instances + /// + public event TaskFinishedEventHandler? TaskFinished; + + /// + /// Raised for BuildErrorEventArgs instances + /// + public event BuildErrorEventHandler? ErrorRaised; + + /// + /// Raised for BuildWarningEventArgs instances + /// + public event BuildWarningEventHandler? WarningRaised; + + /// + /// Raised for BuildMessageEventArgs instances + /// + public event BuildMessageEventHandler? MessageRaised; + + /// + /// Raise one of the events that is appropriate for the type of the BuildEventArgs + /// + public void Dispatch(BuildEventArgs buildEvent) { - Replay(sourceFilePath, CancellationToken.None); + if (buildEvent is BuildMessageEventArgs buildMessageEventArgs) + { + MessageRaised?.Invoke(null, buildMessageEventArgs); + } + else if (buildEvent is TaskStartedEventArgs taskStartedEventArgs) + { + TaskStarted?.Invoke(null, taskStartedEventArgs); + } + else if (buildEvent is TaskFinishedEventArgs taskFinishedEventArgs) + { + TaskFinished?.Invoke(null, taskFinishedEventArgs); + } + else if (buildEvent is TargetStartedEventArgs targetStartedEventArgs) + { + TargetStarted?.Invoke(null, targetStartedEventArgs); + } + else if (buildEvent is TargetFinishedEventArgs targetFinishedEventArgs) + { + TargetFinished?.Invoke(null, targetFinishedEventArgs); + } + else if (buildEvent is ProjectStartedEventArgs projectStartedEventArgs) + { + ProjectStarted?.Invoke(null, projectStartedEventArgs); + } + else if (buildEvent is ProjectFinishedEventArgs projectFinishedEventArgs) + { + ProjectFinished?.Invoke(null, projectFinishedEventArgs); + } + else if (buildEvent is BuildStartedEventArgs buildStartedEventArgs) + { + BuildStarted?.Invoke(null, buildStartedEventArgs); + } + else if (buildEvent is BuildFinishedEventArgs buildFinishedEventArgs) + { + BuildFinished?.Invoke(null, buildFinishedEventArgs); + } + else if (buildEvent is CustomBuildEventArgs customBuildEventArgs) + { + CustomEventRaised?.Invoke(null, customBuildEventArgs); + } + else if (buildEvent is BuildStatusEventArgs buildStatusEventArgs) + { + StatusEventRaised?.Invoke(null, buildStatusEventArgs); + } + else if (buildEvent is BuildWarningEventArgs buildWarningEventArgs) + { + WarningRaised?.Invoke(null, buildWarningEventArgs); + } + else if (buildEvent is BuildErrorEventArgs buildErrorEventArgs) + { + ErrorRaised?.Invoke(null, buildErrorEventArgs); + } + + AnyEventRaised?.Invoke(null, buildEvent); + } + } + + public interface IRawLogEventsSource + { + public event Action>? BuildEventReceived; + } + + public sealed class BinaryLogReplayRawEventSource : BinaryLogReplayEventSourceBase, IRawLogEventsSource + { + public BinaryLogReplayRawEventSource() + : base(false) { } + + public event Action>? BuildEventReceived + { + add => RawBuildEventReceivedInternal += value; + remove => RawBuildEventReceivedInternal -= value; + } + } + + public class BinaryLogReplayEventSourceBase : IEmbeddedContentSource + { + /// Touches the static constructor + /// to ensure it initializes + /// and + static BinaryLogReplayEventSourceBase() + { + _ = ItemGroupLoggingHelper.ItemGroupIncludeLogMessagePrefix; } + internal BinaryLogReplayEventSourceBase(bool isStructured) + => _isStructuredReader = isStructured; + + private bool _isStructuredReader; + protected bool AllowForwardCompatibilityInternal { get; set; } = true; + protected event Action? OnErrorInternal; + protected event Action? BuildEventReceivedInternal; + protected event Action>? RawBuildEventReceivedInternal; + /// /// Creates a for the provided binary log file. /// Performs decompression and buffering in the optimal way. @@ -54,9 +226,9 @@ public static BinaryReader OpenReader(string sourceFilePath) // and the max throughput is reached with a 32K buffer. See details here: // https://github.com/dotnet/runtime/issues/39233#issuecomment-745598847 var bufferedStream = new BufferedStream(gzipStream, 32768); - return new BinaryReader(bufferedStream); + return new BinaryReader(bufferedStream.ToReadableSeekableStream()); } - catch(Exception) + catch (Exception) { stream?.Dispose(); throw; @@ -69,14 +241,15 @@ public static BinaryReader OpenReader(string sourceFilePath) /// /// /// Indicates whether the passed BinaryReader should be closed on disposing. + /// Indicates whether reading of future versions of logs should be allowed. /// BuildEventArgsReader over the given binlog file binary reader. - public static BuildEventArgsReader OpenBuildEventsReader(BinaryReader binaryReader, bool closeInput) + public static BuildEventArgsReader OpenBuildEventsReader(BinaryReader binaryReader, bool closeInput, bool allowForwardCompatibility = true) { int fileFormatVersion = binaryReader.ReadInt32(); // the log file is written using a newer version of file format // that we don't know how to read - if (fileFormatVersion > BinaryLogger.FileFormatVersion) + if (!allowForwardCompatibility && fileFormatVersion > BinaryLogger.FileFormatVersion) { var text = ResourceUtilities.FormatResourceStringStripCodeAndKeyword("UnsupportedLogFileFormat", fileFormatVersion, BinaryLogger.FileFormatVersion); throw new NotSupportedException(text); @@ -95,6 +268,15 @@ public static BuildEventArgsReader OpenBuildEventsReader(BinaryReader binaryRead public static BuildEventArgsReader OpenBuildEventsReader(string sourceFilePath) => OpenBuildEventsReader(OpenReader(sourceFilePath), true); + /// + /// Read the provided binary log file and raise corresponding events for each BuildEventArgs + /// + /// The full file path of the binary log file + public void Replay(string sourceFilePath) + { + Replay(sourceFilePath, CancellationToken.None); + } + /// /// Read the provided binary log file and raise corresponding events for each BuildEventArgs /// @@ -102,7 +284,7 @@ public static BuildEventArgsReader OpenBuildEventsReader(string sourceFilePath) /// A indicating the replay should stop as soon as possible. public void Replay(string sourceFilePath, CancellationToken cancellationToken) { - using var eventsReader = OpenBuildEventsReader(sourceFilePath); + using var eventsReader = BinaryLogReplayEventSource.OpenBuildEventsReader(sourceFilePath); Replay(eventsReader, cancellationToken); } @@ -114,23 +296,44 @@ public void Replay(string sourceFilePath, CancellationToken cancellationToken) /// A indicating the replay should stop as soon as possible. public void Replay(BinaryReader binaryReader, bool closeInput, CancellationToken cancellationToken) { - using var reader = OpenBuildEventsReader(binaryReader, closeInput); + using var reader = BinaryLogReplayEventSource.OpenBuildEventsReader(binaryReader, closeInput, AllowForwardCompatibilityInternal); Replay(reader, cancellationToken); } - /// - /// Read the provided binary log file and raise corresponding events for each BuildEventArgs - /// - /// The build events reader - caller is responsible for disposing. - /// A indicating the replay should stop as soon as possible. public void Replay(BuildEventArgsReader reader, CancellationToken cancellationToken) { _fileFormatVersionRead?.Invoke(reader.FileFormatVersion); reader.EmbeddedContentRead += _embeddedContentRead; - while (!cancellationToken.IsCancellationRequested && reader.Read() is { } instance) + if (_isStructuredReader) + { + ReplayStructured(reader, cancellationToken); + } + else + { + ReplayRaw(reader, cancellationToken); + } + } + + private void ReplayStructured(BuildEventArgsReader reader, CancellationToken cancellationToken) + { + while ( + !cancellationToken.IsCancellationRequested && + reader.Read(AllowForwardCompatibilityInternal, AllowForwardCompatibilityInternal, OnErrorInternal ?? (_ => { })) + is { } instance) + { + BuildEventReceivedInternal?.Invoke(instance); + } + } + + private void ReplayRaw(BuildEventArgsReader reader, CancellationToken cancellationToken) + { + _fileFormatVersionRead?.Invoke(reader.FileFormatVersion); + reader.EmbeddedContentRead += _embeddedContentRead; + + while (!cancellationToken.IsCancellationRequested && reader.ReadRaw() is { Count: > 0 } instance) { - Dispatch(instance); + RawBuildEventReceivedInternal?.Invoke(instance); } } diff --git a/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs b/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs index 686152b6c62..1b62948bc1d 100644 --- a/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs +++ b/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; +using System.Buffers; using System.Collections; using System.Collections.Generic; using System.IO; @@ -47,6 +48,11 @@ public class BuildEventArgsReader : IBuildEventArgsReaderNotifications, IDisposa /// private readonly StringStorage stringStorage = new StringStorage(); + /// + /// Is the enderlying stream seekable? + /// + private readonly bool _canSeek; + // reflection is needed to set these three fields because public constructors don't provide // a way to set these from the outside private static FieldInfo? buildEventArgsFieldThreadId = @@ -62,6 +68,7 @@ public class BuildEventArgsReader : IBuildEventArgsReaderNotifications, IDisposa public BuildEventArgsReader(BinaryReader binaryReader, int fileFormatVersion) { this.binaryReader = binaryReader; + this._canSeek = binaryReader.BaseStream.CanSeek; this.fileFormatVersion = fileFormatVersion; } @@ -99,6 +106,87 @@ public void Dispose() /// public event Action? ArchiveFileEncountered; + private byte[]? _pooledBuffer; + private BinaryWriter? _pooledBinaryWriter; + + /// + /// Reads the next serialized log record from the . + /// + /// ArraySegment containing serialized BuildEventArgs record + public ArraySegment ReadRaw() + { + // TODO: For even better performance experiment with returning a stream + // it would need to be concatenated Stream of a MemoryStream with kind and size + // and SubStream over the binaryReader.BaseStream of the given size. + // We'd need to validate that reader reads exactly the given size, or skip rest on their behalf. + + BinaryLogRecordKind recordKind = ReadTillNextEvent(); + + if (recordKind == BinaryLogRecordKind.EndOfFile) + { + ReturnPooledBuffer(); + return new ArraySegment(); + } + + int serializedEventLength = ReadInt32(); + + // The BinaryLogRecordKind and size needs to be added back to the buffer. + // We do not know the exact size needed to write those two upfront though, + // so we need to call memoryStream.Position to find out. + (BinaryWriter binaryWriter, MemoryStream memoryStream, byte[] buffer) = + RentPooledBinaryWriter(serializedEventLength + 2 * sizeof(int)); + binaryWriter.Write7BitEncodedInt((int)recordKind); + binaryWriter.Write7BitEncodedInt(serializedEventLength); + + binaryReader.BaseStream.ReadAtLeast(buffer, (int)memoryStream.Position, serializedEventLength, throwOnEndOfStream: true); + // Size of the raw message and the preceding type and size info + return new ArraySegment(buffer, 0, serializedEventLength + (int)memoryStream.Position); + } + + private byte[] RentPooledBuffer(int minimumBytes) + { + if (_pooledBuffer == null || _pooledBuffer.Length < minimumBytes) + { + ReturnPooledBuffer(); + _pooledBuffer = ArrayPool.Shared.Rent(minimumBytes); + } + + return _pooledBuffer; + } + + private (BinaryWriter binaryWriter, MemoryStream memoryStream, byte[] buffer) RentPooledBinaryWriter(int minimumBytes) + { + if (_pooledBuffer == null || _pooledBuffer.Length < minimumBytes) + { + ReturnPooledBuffer(); + _pooledBuffer = ArrayPool.Shared.Rent(minimumBytes); + _pooledBinaryWriter = null; + } + + MemoryStream ms; + if (_pooledBinaryWriter == null) + { + ms = new(_pooledBuffer); + _pooledBinaryWriter = new BinaryWriter(ms); + } + else + { + ms = (_pooledBinaryWriter.BaseStream as MemoryStream)!; + } + + return (_pooledBinaryWriter, ms, _pooledBuffer); + } + + private void ReturnPooledBuffer() + { + if (_pooledBuffer != null) + { + ArrayPool.Shared.Return(_pooledBuffer); + _pooledBuffer = null; + _pooledBinaryWriter = null; + } + } + /// /// Reads the next log record from the . /// @@ -107,6 +195,164 @@ public void Dispose() /// If there are no more records, returns . /// public BuildEventArgs? Read() + { + return Read(skipUnknownEvents: false, skipUnknownEventParts: false, _ => { }); + } + + /// + /// Reads the next log record from the . + /// + /// Indicates whether unknown BuildEvents should be silently skipped. Read returns null otherwise. + /// Indicates whether unread parts of BuildEvents (probably added in newer format of particular BuildEvent)should be silently skipped. Exception thrown otherwise. + /// Receives recoverable errors during reading. + /// + /// The next . + /// If there are no more records, returns . + /// + /// Thrown based on and arguments. + public BuildEventArgs? Read(bool skipUnknownEvents, bool skipUnknownEventParts, Action onError) + { + BuildEventArgs? result = null; + while (result == null) + { + BinaryLogRecordKind recordKind = ReadTillNextEvent(); + + if (recordKind == BinaryLogRecordKind.EndOfFile) + { + return null; + } + + int serializedEventLength = ReadInt32(); // record length + + long preEventPosition = _canSeek ? binaryReader.BaseStream.Position : 0; + + switch (recordKind) + { + case BinaryLogRecordKind.BuildStarted: + result = ReadBuildStartedEventArgs(); + break; + case BinaryLogRecordKind.BuildFinished: + result = ReadBuildFinishedEventArgs(); + break; + case BinaryLogRecordKind.ProjectStarted: + result = ReadProjectStartedEventArgs(); + break; + case BinaryLogRecordKind.ProjectFinished: + result = ReadProjectFinishedEventArgs(); + break; + case BinaryLogRecordKind.TargetStarted: + result = ReadTargetStartedEventArgs(); + break; + case BinaryLogRecordKind.TargetFinished: + result = ReadTargetFinishedEventArgs(); + break; + case BinaryLogRecordKind.TaskStarted: + result = ReadTaskStartedEventArgs(); + break; + case BinaryLogRecordKind.TaskFinished: + result = ReadTaskFinishedEventArgs(); + break; + case BinaryLogRecordKind.Error: + result = ReadBuildErrorEventArgs(); + break; + case BinaryLogRecordKind.Warning: + result = ReadBuildWarningEventArgs(); + break; + case BinaryLogRecordKind.Message: + result = ReadBuildMessageEventArgs(); + break; + case BinaryLogRecordKind.CriticalBuildMessage: + result = ReadCriticalBuildMessageEventArgs(); + break; + case BinaryLogRecordKind.TaskCommandLine: + result = ReadTaskCommandLineEventArgs(); + break; + case BinaryLogRecordKind.TaskParameter: + result = ReadTaskParameterEventArgs(); + break; + case BinaryLogRecordKind.ProjectEvaluationStarted: + result = ReadProjectEvaluationStartedEventArgs(); + break; + case BinaryLogRecordKind.ProjectEvaluationFinished: + result = ReadProjectEvaluationFinishedEventArgs(); + break; + case BinaryLogRecordKind.ProjectImported: + result = ReadProjectImportedEventArgs(); + break; + case BinaryLogRecordKind.TargetSkipped: + result = ReadTargetSkippedEventArgs(); + break; + case BinaryLogRecordKind.EnvironmentVariableRead: + result = ReadEnvironmentVariableReadEventArgs(); + break; + case BinaryLogRecordKind.ResponseFileUsed: + result = ReadResponseFileUsedEventArgs(); + break; + case BinaryLogRecordKind.PropertyReassignment: + result = ReadPropertyReassignmentEventArgs(); + break; + case BinaryLogRecordKind.UninitializedPropertyRead: + result = ReadUninitializedPropertyReadEventArgs(); + break; + case BinaryLogRecordKind.PropertyInitialValueSet: + result = ReadPropertyInitialValueSetEventArgs(); + break; + case BinaryLogRecordKind.AssemblyLoad: + result = ReadAssemblyLoadEventArgs(); + break; + default: + onError( + $"BuildEvent record number {recordNumber} (serialized size: {serializedEventLength}) is of unsupported type: {recordKind}.{(skipUnknownEvents ? " Skipping it." : string.Empty)}"); + if (skipUnknownEvents) + { + SkipBytes(serializedEventLength); + } + else + { + return null; + } + break; + } + + long postEventPosition = _canSeek ? binaryReader.BaseStream.Position : serializedEventLength; + int bytesRead = (int)(postEventPosition - preEventPosition); + if (bytesRead != serializedEventLength) + { + string error = + $"BuildEvent record number {recordNumber} was expected to read exactly {serializedEventLength} bytes from the stream, but read {bytesRead} instead."; + + if (skipUnknownEventParts && bytesRead < serializedEventLength) + { + onError(error); + SkipBytes(serializedEventLength - bytesRead); + } + else + { + throw new InvalidDataException(error); + } + } + + recordNumber += 1; + } + + return result; + } + + private void SkipBytes(int count) + { + if (_canSeek) + { + binaryReader.BaseStream.Seek(count, SeekOrigin.Current); + } + else + { + byte[] buffer = RentPooledBuffer(count); + binaryReader.BaseStream.ReadAtLeast(buffer, 0, count, throwOnEndOfStream: true); + ReturnPooledBuffer(); + } + } + + private BinaryLogRecordKind ReadTillNextEvent() { BinaryLogRecordKind recordKind = (BinaryLogRecordKind)ReadInt32(); @@ -133,88 +379,7 @@ public void Dispose() recordKind = (BinaryLogRecordKind)ReadInt32(); } - BuildEventArgs? result = null; - switch (recordKind) - { - case BinaryLogRecordKind.EndOfFile: - break; - case BinaryLogRecordKind.BuildStarted: - result = ReadBuildStartedEventArgs(); - break; - case BinaryLogRecordKind.BuildFinished: - result = ReadBuildFinishedEventArgs(); - break; - case BinaryLogRecordKind.ProjectStarted: - result = ReadProjectStartedEventArgs(); - break; - case BinaryLogRecordKind.ProjectFinished: - result = ReadProjectFinishedEventArgs(); - break; - case BinaryLogRecordKind.TargetStarted: - result = ReadTargetStartedEventArgs(); - break; - case BinaryLogRecordKind.TargetFinished: - result = ReadTargetFinishedEventArgs(); - break; - case BinaryLogRecordKind.TaskStarted: - result = ReadTaskStartedEventArgs(); - break; - case BinaryLogRecordKind.TaskFinished: - result = ReadTaskFinishedEventArgs(); - break; - case BinaryLogRecordKind.Error: - result = ReadBuildErrorEventArgs(); - break; - case BinaryLogRecordKind.Warning: - result = ReadBuildWarningEventArgs(); - break; - case BinaryLogRecordKind.Message: - result = ReadBuildMessageEventArgs(); - break; - case BinaryLogRecordKind.CriticalBuildMessage: - result = ReadCriticalBuildMessageEventArgs(); - break; - case BinaryLogRecordKind.TaskCommandLine: - result = ReadTaskCommandLineEventArgs(); - break; - case BinaryLogRecordKind.TaskParameter: - result = ReadTaskParameterEventArgs(); - break; - case BinaryLogRecordKind.ProjectEvaluationStarted: - result = ReadProjectEvaluationStartedEventArgs(); - break; - case BinaryLogRecordKind.ProjectEvaluationFinished: - result = ReadProjectEvaluationFinishedEventArgs(); - break; - case BinaryLogRecordKind.ProjectImported: - result = ReadProjectImportedEventArgs(); - break; - case BinaryLogRecordKind.TargetSkipped: - result = ReadTargetSkippedEventArgs(); - break; - case BinaryLogRecordKind.EnvironmentVariableRead: - result = ReadEnvironmentVariableReadEventArgs(); - break; - case BinaryLogRecordKind.ResponseFileUsed: - result = ReadResponseFileUsedEventArgs(); - break; - case BinaryLogRecordKind.PropertyReassignment: - result = ReadPropertyReassignmentEventArgs(); - break; - case BinaryLogRecordKind.UninitializedPropertyRead: - result = ReadUninitializedPropertyReadEventArgs(); - break; - case BinaryLogRecordKind.PropertyInitialValueSet: - result = ReadPropertyInitialValueSetEventArgs(); - break; - case BinaryLogRecordKind.AssemblyLoad: - result = ReadAssemblyLoadEventArgs(); - break; - } - - recordNumber += 1; - - return result; + return recordKind; } private static bool IsAuxiliaryRecord(BinaryLogRecordKind recordKind) @@ -241,7 +406,7 @@ private void ReadEmbeddedContent(BinaryLogRecordKind recordKind) new ProjectImportsCollector(Path.GetRandomFileName(), false, runOnBackground: false); } - Stream embeddedStream = new SubStream(binaryReader.BaseStream, length); + Stream embeddedStream = binaryReader.BaseStream.Slice(length); // We are intentionally not grace handling corrupt embedded stream @@ -291,18 +456,11 @@ private void ReadEmbeddedContent(BinaryLogRecordKind recordKind) { EmbeddedContentRead(new EmbeddedContentEventArgs( recordKind.ToEmbeddedContentKind(), - new SubStream(binaryReader.BaseStream, length))); + binaryReader.BaseStream.Slice(length))); } else { - if (binaryReader.BaseStream.CanSeek) - { - binaryReader.BaseStream.Seek(length, SeekOrigin.Current); - } - else - { - binaryReader.ReadBytes(length); - } + SkipBytes(length); } } diff --git a/src/Build/Logging/BinaryLogger/BuildEventArgsWriter.cs b/src/Build/Logging/BinaryLogger/BuildEventArgsWriter.cs index 99ffe22d96d..a0246b5d4fe 100644 --- a/src/Build/Logging/BinaryLogger/BuildEventArgsWriter.cs +++ b/src/Build/Logging/BinaryLogger/BuildEventArgsWriter.cs @@ -133,7 +133,10 @@ public BuildEventArgsWriter(BinaryWriter binaryWriter) /// public void Write(BuildEventArgs e) { - WriteCore(e); + BinaryLogRecordKind eventKind = WriteCore(e); + + Write(eventKind); + Write((int)currentRecordStream.Length); // flush the current record and clear the MemoryStream to prepare for next use currentRecordStream.WriteTo(originalStream); @@ -179,23 +182,23 @@ Base types and inheritance ("EventArgs" suffix omitted): ExtendedCustomBuild */ - private void WriteCore(BuildEventArgs e) + private BinaryLogRecordKind WriteCore(BuildEventArgs e) { switch (e) { - case BuildMessageEventArgs buildMessage: Write(buildMessage); break; - case TaskStartedEventArgs taskStarted: Write(taskStarted); break; - case TaskFinishedEventArgs taskFinished: Write(taskFinished); break; - case TargetStartedEventArgs targetStarted: Write(targetStarted); break; - case TargetFinishedEventArgs targetFinished: Write(targetFinished); break; - case BuildErrorEventArgs buildError: Write(buildError); break; - case BuildWarningEventArgs buildWarning: Write(buildWarning); break; - case ProjectStartedEventArgs projectStarted: Write(projectStarted); break; - case ProjectFinishedEventArgs projectFinished: Write(projectFinished); break; - case BuildStartedEventArgs buildStarted: Write(buildStarted); break; - case BuildFinishedEventArgs buildFinished: Write(buildFinished); break; - case ProjectEvaluationStartedEventArgs projectEvaluationStarted: Write(projectEvaluationStarted); break; - case ProjectEvaluationFinishedEventArgs projectEvaluationFinished: Write(projectEvaluationFinished); break; + case BuildMessageEventArgs buildMessage: return Write(buildMessage); + case TaskStartedEventArgs taskStarted: return Write(taskStarted); + case TaskFinishedEventArgs taskFinished: return Write(taskFinished); + case TargetStartedEventArgs targetStarted: return Write(targetStarted); + case TargetFinishedEventArgs targetFinished: return Write(targetFinished); + case BuildErrorEventArgs buildError: return Write(buildError); + case BuildWarningEventArgs buildWarning: return Write(buildWarning); + case ProjectStartedEventArgs projectStarted: return Write(projectStarted); + case ProjectFinishedEventArgs projectFinished: return Write(projectFinished); + case BuildStartedEventArgs buildStarted: return Write(buildStarted); + case BuildFinishedEventArgs buildFinished: return Write(buildFinished); + case ProjectEvaluationStartedEventArgs projectEvaluationStarted: return Write(projectEvaluationStarted); + case ProjectEvaluationFinishedEventArgs projectEvaluationFinished: return Write(projectEvaluationFinished); default: // convert all unrecognized objects to message // and just preserve the message @@ -225,8 +228,7 @@ private void WriteCore(BuildEventArgs e) e.Timestamp); } buildMessageEventArgs.BuildEventContext = e.BuildEventContext ?? BuildEventContext.Invalid; - Write(buildMessageEventArgs); - break; + return Write(buildMessageEventArgs); } } @@ -272,9 +274,8 @@ public void Dispose() } } - private void Write(BuildStartedEventArgs e) + private BinaryLogRecordKind Write(BuildStartedEventArgs e) { - Write(BinaryLogRecordKind.BuildStarted); WriteBuildEventArgsFields(e); if (Traits.LogAllEnvironmentVariables) { @@ -284,26 +285,27 @@ private void Write(BuildStartedEventArgs e) { Write(e.BuildEnvironment?.Where(kvp => EnvironmentUtilities.IsWellKnownEnvironmentDerivedProperty(kvp.Key))); } + + return BinaryLogRecordKind.BuildStarted; } - private void Write(BuildFinishedEventArgs e) + private BinaryLogRecordKind Write(BuildFinishedEventArgs e) { - Write(BinaryLogRecordKind.BuildFinished); WriteBuildEventArgsFields(e); Write(e.Succeeded); + + return BinaryLogRecordKind.BuildFinished; } - private void Write(ProjectEvaluationStartedEventArgs e) + private BinaryLogRecordKind Write(ProjectEvaluationStartedEventArgs e) { - Write(BinaryLogRecordKind.ProjectEvaluationStarted); WriteBuildEventArgsFields(e, writeMessage: false); WriteDeduplicatedString(e.ProjectFile); + return BinaryLogRecordKind.ProjectEvaluationStarted; } - private void Write(ProjectEvaluationFinishedEventArgs e) + private BinaryLogRecordKind Write(ProjectEvaluationFinishedEventArgs e) { - Write(BinaryLogRecordKind.ProjectEvaluationFinished); - WriteBuildEventArgsFields(e, writeMessage: false); WriteDeduplicatedString(e.ProjectFile); @@ -325,11 +327,12 @@ private void Write(ProjectEvaluationFinishedEventArgs e) Write(item.Value); } } + + return BinaryLogRecordKind.ProjectEvaluationFinished; } - private void Write(ProjectStartedEventArgs e) + private BinaryLogRecordKind Write(ProjectStartedEventArgs e) { - Write(BinaryLogRecordKind.ProjectStarted); WriteBuildEventArgsFields(e, writeMessage: false); if (e.ParentProjectBuildEventContext == null) @@ -353,62 +356,68 @@ private void Write(ProjectStartedEventArgs e) WriteProperties(e.Properties); WriteProjectItems(e.Items); + + return BinaryLogRecordKind.ProjectStarted; } - private void Write(ProjectFinishedEventArgs e) + private BinaryLogRecordKind Write(ProjectFinishedEventArgs e) { - Write(BinaryLogRecordKind.ProjectFinished); WriteBuildEventArgsFields(e, writeMessage: false); WriteDeduplicatedString(e.ProjectFile); Write(e.Succeeded); + + return BinaryLogRecordKind.ProjectFinished; } - private void Write(TargetStartedEventArgs e) + private BinaryLogRecordKind Write(TargetStartedEventArgs e) { - Write(BinaryLogRecordKind.TargetStarted); WriteBuildEventArgsFields(e, writeMessage: false); WriteDeduplicatedString(e.TargetName); WriteDeduplicatedString(e.ProjectFile); WriteDeduplicatedString(e.TargetFile); WriteDeduplicatedString(e.ParentTarget); Write((int)e.BuildReason); + + return BinaryLogRecordKind.TargetStarted; } - private void Write(TargetFinishedEventArgs e) + private BinaryLogRecordKind Write(TargetFinishedEventArgs e) { - Write(BinaryLogRecordKind.TargetFinished); WriteBuildEventArgsFields(e, writeMessage: false); Write(e.Succeeded); WriteDeduplicatedString(e.ProjectFile); WriteDeduplicatedString(e.TargetFile); WriteDeduplicatedString(e.TargetName); WriteTaskItemList(e.TargetOutputs); + + return BinaryLogRecordKind.TargetFinished; } - private void Write(TaskStartedEventArgs e) + private BinaryLogRecordKind Write(TaskStartedEventArgs e) { - Write(BinaryLogRecordKind.TaskStarted); WriteBuildEventArgsFields(e, writeMessage: false, writeLineAndColumn: true); Write(e.LineNumber); Write(e.ColumnNumber); WriteDeduplicatedString(e.TaskName); WriteDeduplicatedString(e.ProjectFile); WriteDeduplicatedString(e.TaskFile); + + return BinaryLogRecordKind.TaskStarted; } - private void Write(TaskFinishedEventArgs e) + private BinaryLogRecordKind Write(TaskFinishedEventArgs e) { - Write(BinaryLogRecordKind.TaskFinished); WriteBuildEventArgsFields(e, writeMessage: false); Write(e.Succeeded); WriteDeduplicatedString(e.TaskName); WriteDeduplicatedString(e.ProjectFile); WriteDeduplicatedString(e.TaskFile); + + return BinaryLogRecordKind.TaskFinished; } - private void Write(BuildErrorEventArgs e) + private BinaryLogRecordKind Write(BuildErrorEventArgs e) { - Write(BinaryLogRecordKind.Error); WriteBuildEventArgsFields(e); WriteArguments(e.RawArguments); WriteDeduplicatedString(e.Subcategory); @@ -419,11 +428,12 @@ private void Write(BuildErrorEventArgs e) Write(e.ColumnNumber); Write(e.EndLineNumber); Write(e.EndColumnNumber); + + return BinaryLogRecordKind.Error; } - private void Write(BuildWarningEventArgs e) + private BinaryLogRecordKind Write(BuildWarningEventArgs e) { - Write(BinaryLogRecordKind.Warning); WriteBuildEventArgsFields(e); WriteArguments(e.RawArguments); WriteDeduplicatedString(e.Subcategory); @@ -434,42 +444,42 @@ private void Write(BuildWarningEventArgs e) Write(e.ColumnNumber); Write(e.EndLineNumber); Write(e.EndColumnNumber); + + return BinaryLogRecordKind.Warning; } - private void Write(BuildMessageEventArgs e) + private BinaryLogRecordKind Write(BuildMessageEventArgs e) { switch (e) { - case ResponseFileUsedEventArgs responseFileUsed: Write(responseFileUsed); break; - case TaskParameterEventArgs taskParameter: Write(taskParameter); break; - case ProjectImportedEventArgs projectImported: Write(projectImported); break; - case TargetSkippedEventArgs targetSkipped: Write(targetSkipped); break; - case PropertyReassignmentEventArgs propertyReassignment: Write(propertyReassignment); break; - case TaskCommandLineEventArgs taskCommandLine: Write(taskCommandLine); break; - case UninitializedPropertyReadEventArgs uninitializedPropertyRead: Write(uninitializedPropertyRead); break; - case EnvironmentVariableReadEventArgs environmentVariableRead: Write(environmentVariableRead); break; - case PropertyInitialValueSetEventArgs propertyInitialValueSet: Write(propertyInitialValueSet); break; - case CriticalBuildMessageEventArgs criticalBuildMessage: Write(criticalBuildMessage); break; - case AssemblyLoadBuildEventArgs assemblyLoad: Write(assemblyLoad); break; + case ResponseFileUsedEventArgs responseFileUsed: return Write(responseFileUsed); + case TaskParameterEventArgs taskParameter: return Write(taskParameter); + case ProjectImportedEventArgs projectImported: return Write(projectImported); + case TargetSkippedEventArgs targetSkipped: return Write(targetSkipped); + case PropertyReassignmentEventArgs propertyReassignment: return Write(propertyReassignment); + case TaskCommandLineEventArgs taskCommandLine: return Write(taskCommandLine); + case UninitializedPropertyReadEventArgs uninitializedPropertyRead: return Write(uninitializedPropertyRead); + case EnvironmentVariableReadEventArgs environmentVariableRead: return Write(environmentVariableRead); + case PropertyInitialValueSetEventArgs propertyInitialValueSet: return Write(propertyInitialValueSet); + case CriticalBuildMessageEventArgs criticalBuildMessage: return Write(criticalBuildMessage); + case AssemblyLoadBuildEventArgs assemblyLoad: return Write(assemblyLoad); default: // actual BuildMessageEventArgs - Write(BinaryLogRecordKind.Message); WriteMessageFields(e, writeImportance: true); - break; + return BinaryLogRecordKind.Message; } } - private void Write(ProjectImportedEventArgs e) + private BinaryLogRecordKind Write(ProjectImportedEventArgs e) { - Write(BinaryLogRecordKind.ProjectImported); WriteMessageFields(e); Write(e.ImportIgnored); WriteDeduplicatedString(e.ImportedProjectFile); WriteDeduplicatedString(e.UnexpandedProject); + return BinaryLogRecordKind.ProjectImported; } - private void Write(TargetSkippedEventArgs e) + private BinaryLogRecordKind Write(TargetSkippedEventArgs e) { - Write(BinaryLogRecordKind.TargetSkipped); WriteMessageFields(e, writeMessage: false); WriteDeduplicatedString(e.TargetFile); WriteDeduplicatedString(e.TargetName); @@ -480,11 +490,11 @@ private void Write(TargetSkippedEventArgs e) Write((int)e.BuildReason); Write((int)e.SkipReason); binaryWriter.WriteOptionalBuildEventContext(e.OriginalBuildEventContext); + return BinaryLogRecordKind.TargetSkipped; } - private void Write(AssemblyLoadBuildEventArgs e) + private BinaryLogRecordKind Write(AssemblyLoadBuildEventArgs e) { - Write(BinaryLogRecordKind.AssemblyLoad); WriteMessageFields(e, writeMessage: false, writeImportance: false); Write((int)e.LoadingContext); WriteDeduplicatedString(e.LoadingInitiator); @@ -492,63 +502,63 @@ private void Write(AssemblyLoadBuildEventArgs e) WriteDeduplicatedString(e.AssemblyPath); Write(e.MVID); WriteDeduplicatedString(e.AppDomainDescriptor); + return BinaryLogRecordKind.AssemblyLoad; } - private void Write(CriticalBuildMessageEventArgs e) + private BinaryLogRecordKind Write(CriticalBuildMessageEventArgs e) { - Write(BinaryLogRecordKind.CriticalBuildMessage); WriteMessageFields(e); + return BinaryLogRecordKind.CriticalBuildMessage; } - private void Write(PropertyReassignmentEventArgs e) + private BinaryLogRecordKind Write(PropertyReassignmentEventArgs e) { - Write(BinaryLogRecordKind.PropertyReassignment); WriteMessageFields(e, writeMessage: false, writeImportance: true); WriteDeduplicatedString(e.PropertyName); WriteDeduplicatedString(e.PreviousValue); WriteDeduplicatedString(e.NewValue); WriteDeduplicatedString(e.Location); + return BinaryLogRecordKind.PropertyReassignment; } - private void Write(UninitializedPropertyReadEventArgs e) + private BinaryLogRecordKind Write(UninitializedPropertyReadEventArgs e) { - Write(BinaryLogRecordKind.UninitializedPropertyRead); WriteMessageFields(e, writeImportance: true); WriteDeduplicatedString(e.PropertyName); + return BinaryLogRecordKind.UninitializedPropertyRead; } - private void Write(PropertyInitialValueSetEventArgs e) + private BinaryLogRecordKind Write(PropertyInitialValueSetEventArgs e) { - Write(BinaryLogRecordKind.PropertyInitialValueSet); WriteMessageFields(e, writeImportance: true); WriteDeduplicatedString(e.PropertyName); WriteDeduplicatedString(e.PropertyValue); WriteDeduplicatedString(e.PropertySource); + return BinaryLogRecordKind.PropertyInitialValueSet; } - private void Write(EnvironmentVariableReadEventArgs e) + private BinaryLogRecordKind Write(EnvironmentVariableReadEventArgs e) { - Write(BinaryLogRecordKind.EnvironmentVariableRead); WriteMessageFields(e, writeImportance: true); WriteDeduplicatedString(e.EnvironmentVariableName); + return BinaryLogRecordKind.EnvironmentVariableRead; } - private void Write(ResponseFileUsedEventArgs e) + private BinaryLogRecordKind Write(ResponseFileUsedEventArgs e) { - Write(BinaryLogRecordKind.ResponseFileUsed); WriteMessageFields(e); WriteDeduplicatedString(e.ResponseFilePath); + return BinaryLogRecordKind.ResponseFileUsed; } - private void Write(TaskCommandLineEventArgs e) + private BinaryLogRecordKind Write(TaskCommandLineEventArgs e) { - Write(BinaryLogRecordKind.TaskCommandLine); WriteMessageFields(e, writeMessage: false, writeImportance: true); WriteDeduplicatedString(e.CommandLine); WriteDeduplicatedString(e.TaskName); + return BinaryLogRecordKind.TaskCommandLine; } - private void Write(TaskParameterEventArgs e) + private BinaryLogRecordKind Write(TaskParameterEventArgs e) { - Write(BinaryLogRecordKind.TaskParameter); WriteMessageFields(e, writeMessage: false); Write((int)e.Kind); WriteDeduplicatedString(e.ItemType); @@ -558,6 +568,7 @@ private void Write(TaskParameterEventArgs e) { CheckForFilesToEmbed(e.ItemType, e.Items); } + return BinaryLogRecordKind.TaskParameter; } private void WriteBuildEventArgsFields(BuildEventArgs e, bool writeMessage = true, bool writeLineAndColumn = false) diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/StreamExtensions.cs b/src/Build/Logging/BinaryLogger/Postprocessing/StreamExtensions.cs new file mode 100644 index 00000000000..83492b0af2d --- /dev/null +++ b/src/Build/Logging/BinaryLogger/Postprocessing/StreamExtensions.cs @@ -0,0 +1,57 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Microsoft.Build.Logging +{ + internal static class StreamExtensions + { + public static int ReadAtLeast(this Stream stream, byte[] buffer, int offset, int minimumBytes, bool throwOnEndOfStream) + { + Debug.Assert(offset + minimumBytes <= buffer.Length); + + int totalRead = 0; + while (totalRead < minimumBytes) + { + int read = stream.Read(buffer, offset, minimumBytes - totalRead); + if (read == 0) + { + if (throwOnEndOfStream) + { + throw new InvalidDataException("Unexpected end of stream."); + } + + return totalRead; + } + + totalRead += read; + offset += read; + } + + return totalRead; + } + + public static Stream ToReadableSeekableStream(this Stream stream) + { + return TransparentReadStream.CreateSeekableStream(stream); + } + + /// + /// Creates bounded read-only, forward-only view over an underlying stream. + /// + /// + /// + /// + public static Stream Slice(this Stream stream, long length) + { + return new SubStream(stream, length); + } + } +} diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/TransparentReadStream.cs b/src/Build/Logging/BinaryLogger/Postprocessing/TransparentReadStream.cs new file mode 100644 index 00000000000..5061a7c3b00 --- /dev/null +++ b/src/Build/Logging/BinaryLogger/Postprocessing/TransparentReadStream.cs @@ -0,0 +1,112 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Buffers; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Runtime.InteropServices.ComTypes; +using System.Text; +using System.Threading.Tasks; + +namespace Microsoft.Build.Logging +{ + /// + /// A wrapper stream that allows position tracking and forward seeking. + /// + internal class TransparentReadStream : Stream + { + private readonly Stream _stream; + private long _position; + + public static Stream CreateSeekableStream(Stream stream) + { + if (stream.CanSeek) + { + return stream; + } + + if(!stream.CanRead) + { + throw new InvalidOperationException("Stream must be readable."); + } + + return new TransparentReadStream(stream); + } + + private TransparentReadStream(Stream stream) + { + _stream = stream; + } + + public override bool CanRead => _stream.CanRead; + public override bool CanSeek => true; + public override bool CanWrite => false; + public override long Length => _stream.Length; + public override long Position + { + get => _position; + set => SkipBytes(value - _position); + } + + public override void Flush() + { + _stream.Flush(); + } + + public override int Read(byte[] buffer, int offset, int count) + { + int cnt = _stream.Read(buffer, offset, count); + _position += cnt; + return cnt; + } + + public override long Seek(long offset, SeekOrigin origin) + { + if(origin != SeekOrigin.Current) + { + throw new InvalidOperationException("Only seeking from SeekOrigin.Current is supported."); + } + + SkipBytes(offset); + + return _position; + } + + public override void SetLength(long value) + { + throw new InvalidOperationException("Expanding stream is not supported."); + } + + public override void Write(byte[] buffer, int offset, int count) + { + throw new InvalidOperationException("Writing is not supported."); + } + + private void SkipBytes(long count) + { + if(count < 0) + { + throw new InvalidOperationException("Seeking backwards is not supported."); + } + + if(count == 0) + { + return; + } + + byte[] buffer = ArrayPool.Shared.Rent((int)count); + try + { + _position += _stream.ReadAtLeast(buffer, 0, (int)count, throwOnEndOfStream: true); + } + finally + { + ArrayPool.Shared.Return(buffer); + } + } + + public override void Close() => _stream.Close(); + } +} From 5ae9bb100c5ebacedb39a7ad708f920a025d8d06 Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Thu, 5 Oct 2023 21:09:25 +0200 Subject: [PATCH 10/49] Add tests and fixes for forward compatibility reading --- src/Build.UnitTests/BinaryLogger_Tests.cs | 344 ++++++++++++++++- .../BinaryLogReplayEventSource.cs | 350 +++++++----------- .../Logging/BinaryLogger/BinaryLogger.cs | 70 ++-- .../BinaryLogger/BuildEventArgsDispatcher.cs | 16 + .../BinaryLogger/BuildEventArgsReader.cs | 190 +++++----- .../BinaryLogger/BuildEventArgsWriter.cs | 101 +++-- .../EmbeddedContentEventArgs.cs | 8 - .../Postprocessing/IEmbeddedContentSource.cs | 8 +- .../BinaryLogger/Postprocessing/SubStream.cs | 2 + 9 files changed, 666 insertions(+), 423 deletions(-) diff --git a/src/Build.UnitTests/BinaryLogger_Tests.cs b/src/Build.UnitTests/BinaryLogger_Tests.cs index 4928a8d29f8..6282d833a0c 100644 --- a/src/Build.UnitTests/BinaryLogger_Tests.cs +++ b/src/Build.UnitTests/BinaryLogger_Tests.cs @@ -5,8 +5,8 @@ using System.Collections.Generic; using System.IO; using System.IO.Compression; +using System.Reflection; using System.Text; - using Microsoft.Build.BackEnd.Logging; using Microsoft.Build.Evaluation; using Microsoft.Build.Execution; @@ -83,10 +83,21 @@ public BinaryLoggerTests(ITestOutputHelper output) _logFile = _env.ExpectFile(".binlog").Path; } + public enum BinlogRoundtripTestReplayMode + { + NoReplay, + Structured, + RawEvents + } + [Theory] - [InlineData(s_testProject)] - [InlineData(s_testProject2)] - public void TestBinaryLoggerRoundtrip(string projectText) + [InlineData(s_testProject, BinlogRoundtripTestReplayMode.NoReplay)] + [InlineData(s_testProject, BinlogRoundtripTestReplayMode.Structured)] + [InlineData(s_testProject, BinlogRoundtripTestReplayMode.RawEvents)] + [InlineData(s_testProject2, BinlogRoundtripTestReplayMode.NoReplay)] + [InlineData(s_testProject2, BinlogRoundtripTestReplayMode.Structured)] + [InlineData(s_testProject2, BinlogRoundtripTestReplayMode.RawEvents)] + public void TestBinaryLoggerRoundtrip(string projectText, BinlogRoundtripTestReplayMode replayMode) { var binaryLogger = new BinaryLogger(); @@ -110,6 +121,44 @@ public void TestBinaryLoggerRoundtrip(string projectText) project.Build(new ILogger[] { binaryLogger, mockLogFromBuild, serialFromBuild, parallelFromBuild }).ShouldBeTrue(); } + string fileToReplay; + switch (replayMode) + { + case BinlogRoundtripTestReplayMode.NoReplay: + fileToReplay = _logFile; + break; + case BinlogRoundtripTestReplayMode.Structured: + { + var logReader = new BinaryLogReplayEventSource(); + fileToReplay = _env.ExpectFile(".binlog").Path; + // need dummy handler to force structured replay + logReader.BuildFinished += (_, _) => { }; + BinaryLogger outputBinlog = new BinaryLogger() + { + Parameters = fileToReplay + }; + outputBinlog.Initialize(logReader); + logReader.Replay(_logFile); + outputBinlog.Shutdown(); + } + break; + case BinlogRoundtripTestReplayMode.RawEvents: + { + var logReader = new BinaryLogReplayEventSource(); + fileToReplay = _env.ExpectFile(".binlog").Path; + BinaryLogger outputBinlog = new BinaryLogger() + { + Parameters = fileToReplay + }; + outputBinlog.Initialize(logReader); + logReader.Replay(_logFile); + outputBinlog.Shutdown(); + } + break; + default: + throw new ArgumentOutOfRangeException(nameof(replayMode), replayMode, null); + } + var mockLogFromPlayback = new MockLogger(); var serialFromPlaybackText = new StringBuilder(); @@ -126,7 +175,10 @@ public void TestBinaryLoggerRoundtrip(string projectText) parallelFromPlayback.Initialize(binaryLogReader); // read the binary log and replay into mockLogger2 - binaryLogReader.Replay(_logFile); + binaryLogReader.Replay(fileToReplay); + mockLogFromPlayback.Shutdown(); + serialFromPlayback.Shutdown(); + parallelFromPlayback.Shutdown(); // the binlog will have more information than recorded by the text log mockLogFromPlayback.FullLog.ShouldContainWithoutWhitespace(mockLogFromBuild.FullLog); @@ -140,6 +192,288 @@ public void TestBinaryLoggerRoundtrip(string projectText) parallelActual.ShouldContainWithoutWhitespace(parallelExpected); } + [Theory] + [InlineData(s_testProject, BinlogRoundtripTestReplayMode.Structured)] + [InlineData(s_testProject, BinlogRoundtripTestReplayMode.RawEvents)] + [InlineData(s_testProject2, BinlogRoundtripTestReplayMode.Structured)] + [InlineData(s_testProject2, BinlogRoundtripTestReplayMode.RawEvents)] + public void TestBinaryLoggerRoundtripEquality(string projectText, BinlogRoundtripTestReplayMode replayMode) + { + // Make sure the env var will get transcribed to traits. + BuildEnvironmentState.s_runningTests = true; + _env.SetEnvironmentVariable("MSBUILDDETERMNISTICBINLOG", "1"); + + var binaryLogger = new BinaryLogger(); + + binaryLogger.Parameters = _logFile; + + // build and log into binary logger + using (ProjectCollection collection = new()) + { + Project project = ObjectModelHelpers.CreateInMemoryProject(collection, projectText); + project.Build(new ILogger[] { binaryLogger }).ShouldBeTrue(); + } + + var logReader = new BinaryLogReplayEventSource(); + string replayedLogFile = _env.ExpectFile(".binlog").Path; + if (replayMode == BinlogRoundtripTestReplayMode.Structured) + { + // need dummy handler to force structured replay + logReader.BuildFinished += (_, _) => { }; + } + + _logFile = @"C:\tmp\trash\console\packed.binlog"; + + BinaryLogger outputBinlog = new BinaryLogger() + { + Parameters = $"LogFile={replayedLogFile};ProjectImports=Replay;OmitInitialInfo" + }; + outputBinlog.Initialize(logReader); + logReader.Replay(_logFile); + // TODO: remove here + // logReader.Replay(originalBuildEventsReader, CancellationToken.None); + outputBinlog.Shutdown(); + + AssertBinlogsHaveEqualContent(_logFile, replayedLogFile); + // If this assertation complicates development - it can possibly be removed + // The structured equality above should be enough. + AssertFilesAreBinaryEqual(_logFile, replayedLogFile); + } + + private static void AssertFilesAreBinaryEqual(string firstPath, string secondPath) + { + FileInfo first = new(firstPath); + FileInfo second = new(secondPath); + + // Skipping shortcut test - so that we can better troubleshoot failures. + ////if (first.Length != second.Length) + ////{ + //// Assert.Fail($"Files differ in size ({first.Name}:{first.Length} and {second.Name}:{second.Length}"); + ////} + + if (string.Equals(first.FullName, second.FullName, StringComparison.OrdinalIgnoreCase)) + { + return; + } + + using FileStream fs1 = first.OpenRead(); + using FileStream fs2 = second.OpenRead(); + for (int i = 0; i < Math.Min(first.Length,second.Length); i++) + { + byte b1 = (byte)fs1.ReadByte(); + byte b2 = (byte)fs2.ReadByte(); + if (b1 != b2) + { + Assert.Fail( + $"Files ({first.Name}:{first.Length} and {second.Name}:{second.Length} sizes) are not equal at byte {i} ({b1} vs {b2})"); + } + } + + if (first.Length != second.Length) + { + Assert.Fail($"Files differ in size ({first.Name}:{first.Length} and {second.Name}:{second.Length}"); + } + } + + private static void AssertBinlogsHaveEqualContent(string firstPath, string secondPath) + { + using var reader1 = BinaryLogReplayEventSource.OpenBuildEventsReader(firstPath); + using var reader2 = BinaryLogReplayEventSource.OpenBuildEventsReader(secondPath); + + Dictionary embedFiles1 = new(); + Dictionary embedFiles2 = new(); + + reader1.ArchiveFileEncountered += arg + => AddArchiveFile(embedFiles1, arg); + + reader2.ArchiveFileEncountered += arg + => AddArchiveFile(embedFiles2, arg); + + int i = 0; + while (reader1.Read() is { } ev1) + { + i++; + var ev2 = reader2.Read(); + if (!Compare(ev1, ev2, out string diffReason, $"event arg {i}")) + { + Assert.Fail($"Binlogs ({firstPath} and {secondPath}) are not equal at event {i} ({diffReason})"); + } + } + // Read the second reader - to confirm there are no more events + // and to force the embedded files to be read. + reader2.Read().ShouldBeNull($"Binlogs ({firstPath} and {secondPath}) are not equal - second has more events >{i + 1}"); + + SweepArchiveFiles(); + + embedFiles1.ShouldBeEmpty(); + embedFiles2.ShouldBeEmpty(); + + void SweepArchiveFiles() + { + List toRemove = new(); + foreach (var file in embedFiles1) + { + if (embedFiles2.TryGetValue(file.Key, out string content)) + { + if (!string.Equals(file.Value, content)) + { + Assert.Fail($"Binlogs ({firstPath} and {secondPath}) are not equal at embedded file {file.Key}"); + } + toRemove.Add(file.Key); + embedFiles2.Remove(file.Key); + } + } + + foreach (var file in toRemove) + { + embedFiles1.Remove(file); + } + } + + void AddArchiveFile(Dictionary files, ArchiveFileEventArgs arg) + { + ArchiveFile embedFile = arg.ObtainArchiveFile(); + string content = embedFile.GetContent(); + files.Add(embedFile.FullPath, content); + arg.SetResult(embedFile.FullPath, content); + SweepArchiveFiles(); + } + } + + private static bool Compare(object left, object right, out string diffReason, string name = "", HashSet compared = null) + { + diffReason = null; + if (compared == null) + { + compared = new HashSet(); + } + else if(compared.Contains(left) && compared.Contains(right)) + { + return true; + } + else + { + compared.Add(left); + compared.Add(right); + } + + if (ReferenceEquals(left, right)) + { + return true; + } + + if ((left == null) ^ (right == null)) + { + diffReason = "One object is null and the other is not." + name; + return false; + } + + if (left.GetType() != right.GetType()) + { + diffReason = $"Object types are different ({left.GetType().FullName} vs {right.GetType().FullName})."; + return false; + } + + Type type = left.GetType(); + if (name == string.Empty) + { + name = type.Name; + } + + if (IsSimpleType(type)) + { + if (!left.Equals(right)) + { + diffReason = $"Objects are different ({left} vs {right}). " + name; + return false; + } + return true; + } + + if (type.IsArray) + { + Array first = left as Array; + Array second = right as Array; + if (first.Length != second.Length) + { + diffReason = $"{type.Name} : array size differs ({first.Length} vs {second.Length})"; + return false; + } + + var en = first.GetEnumerator(); + int i = 0; + while (en.MoveNext()) + { + if(!Compare(en.Current, second.GetValue(i), out diffReason, name, compared)) + { + diffReason += $" (Index {i})"; + return false; + } + i++; + } + } + else if (typeof(System.Collections.IEnumerable).IsAssignableFrom(type)) + { + System.Collections.IEnumerable first = left as System.Collections.IEnumerable; + System.Collections.IEnumerable second = right as System.Collections.IEnumerable; + + var en = first.GetEnumerator(); + var en2 = second.GetEnumerator(); + int i = 0; + while (en.MoveNext()) + { + if (!en2.MoveNext()) + { + diffReason = $"{name} : enumerable size differs"; + return false; + } + + if (!Compare(en.Current, en2.Current, out diffReason, name, compared)) + { + diffReason += $" (Position {i})"; + return false; + } + i++; + } + } + else + { + // Careful - the default argument-less impl gets the static properties as well (e.g. DateTime.Now) + foreach (PropertyInfo pi in type.GetProperties(BindingFlags.Instance | BindingFlags.Public)) + { + try + { + var val = pi.GetValue(left); + var tval = pi.GetValue(right); + var name1 = name + "." + pi.Name; + if (!Compare(val, tval, out diffReason, name1, compared)) + { + return false; + } + } + catch (TargetParameterCountException) + { + // index property + } + } + } + + return true; + } + + internal static bool IsSimpleType(Type type) + { + // Nullables + if (type.IsGenericType && type.GetGenericTypeDefinition() == typeof(Nullable<>)) + { + return IsSimpleType(type.GetGenericArguments()[0]); + } + return type.IsPrimitive + || type.IsEnum + || type == typeof(string) + || type == typeof(decimal); + } + [Fact] public void BinaryLoggerShouldSupportFilePathExplicitParameter() { diff --git a/src/Build/Logging/BinaryLogger/BinaryLogReplayEventSource.cs b/src/Build/Logging/BinaryLogger/BinaryLogReplayEventSource.cs index 6524d3a81a8..a4374b8bbdc 100644 --- a/src/Build/Logging/BinaryLogger/BinaryLogReplayEventSource.cs +++ b/src/Build/Logging/BinaryLogger/BinaryLogReplayEventSource.cs @@ -11,202 +11,71 @@ namespace Microsoft.Build.Logging { - /// - /// Provides a method to read a binary log file (*.binlog) and replay all stored BuildEventArgs - /// by implementing IEventSource and raising corresponding events. - /// - /// The class is public so that we can call it from MSBuild.exe when replaying a log file. - public sealed class BinaryLogReplayEventSource : BinaryLogReplayEventSourceBase, IEventSource, IEmbeddedContentSource + internal interface IRawLogEventsSource { - public BinaryLogReplayEventSource() - : base(true) { } - - public bool AllowForwardCompatibility - { - get => AllowForwardCompatibilityInternal; - set => AllowForwardCompatibilityInternal = value; - } - - public event Action? OnError - { - add => OnErrorInternal += value; - remove => OnErrorInternal -= value; - } - - public event Action? BuildEventReceived - { - add => BuildEventReceivedInternal += value; - remove => BuildEventReceivedInternal -= value; - } - - /// - /// This event is raised for all BuildEventArgs objects after a more type-specific event - /// - public event AnyEventHandler? AnyEventRaised; - - /// - /// Raised for BuildStatusEventArgs instances - /// - public event BuildStatusEventHandler? StatusEventRaised; - - /// - /// Raised for CustomBuildEventArgs instances - /// - public event CustomBuildEventHandler? CustomEventRaised; - - /// - /// Raised for BuildStartedEventArgs instances - /// - public event BuildStartedEventHandler? BuildStarted; - - /// - /// Raised for BuildFinishedEventArgs instances - /// - public event BuildFinishedEventHandler? BuildFinished; - - /// - /// Raised for ProjectStartedEventArgs instances - /// - public event ProjectStartedEventHandler? ProjectStarted; - - /// - /// Raised for ProjectFinishedEventArgs instances - /// - public event ProjectFinishedEventHandler? ProjectFinished; - - /// - /// Raised for TargetStartedEventArgs instances - /// - public event TargetStartedEventHandler? TargetStarted; - /// - /// Raised for TargetFinishedEventArgs instances + /// Event raised when non-textual log record is read. + /// This means all event args and key-value pairs. + /// Strings and Embedded files are not included. /// - public event TargetFinishedEventHandler? TargetFinished; + event Action? LogDataSliceReceived; /// - /// Raised for TaskStartedEventArgs instances + /// Enables initialization (e.g. subscription to events) - that is deferred until Replay is triggered. + /// At this point all other possible subscribers should be already subscribed - + /// so it can be determined if raw events or structured events should be replayed. /// - public event TaskStartedEventHandler? TaskStarted; + /// + /// + /// + void DeferredInitialize( + Action onFileFormatVersionRead, + Action onRawReadingPossible, + Action onStructuredReadingOnly); + } - /// - /// Raised for TaskFinishedEventArgs instances - /// - public event TaskFinishedEventHandler? TaskFinished; + /// + /// Interface for replaying a binary log file (*.binlog) + /// + internal interface IBinaryLogReplaySource : + IEventSource, IRawLogEventsSource, IBuildEventStringsReader, IEmbeddedContentSource + { } - /// - /// Raised for BuildErrorEventArgs instances - /// - public event BuildErrorEventHandler? ErrorRaised; + /// + /// Provides a method to read a binary log file (*.binlog) and replay all stored BuildEventArgs + /// by implementing IEventSource and raising corresponding events. + /// + /// The class is public so that we can call it from MSBuild.exe when replaying a log file. + public sealed class BinaryLogReplayEventSource : EventArgsDispatcher, + IBinaryLogReplaySource + { + /// Touches the static constructor + /// to ensure it initializes + /// and + static BinaryLogReplayEventSource() + { + _ = ItemGroupLoggingHelper.ItemGroupIncludeLogMessagePrefix; + } /// - /// Raised for BuildWarningEventArgs instances + /// Unknown build events or unknown parts of known build events will be ignored if this is set to true. /// - public event BuildWarningEventHandler? WarningRaised; + public bool AllowForwardCompatibility { private get; init; } = true; /// - /// Raised for BuildMessageEventArgs instances + /// Receives recoverable errors during reading. /// - public event BuildMessageEventHandler? MessageRaised; + public event Action? OnRecoverableReadError; /// - /// Raise one of the events that is appropriate for the type of the BuildEventArgs + /// Read the provided binary log file and raise corresponding events for each BuildEventArgs /// - public void Dispatch(BuildEventArgs buildEvent) - { - if (buildEvent is BuildMessageEventArgs buildMessageEventArgs) - { - MessageRaised?.Invoke(null, buildMessageEventArgs); - } - else if (buildEvent is TaskStartedEventArgs taskStartedEventArgs) - { - TaskStarted?.Invoke(null, taskStartedEventArgs); - } - else if (buildEvent is TaskFinishedEventArgs taskFinishedEventArgs) - { - TaskFinished?.Invoke(null, taskFinishedEventArgs); - } - else if (buildEvent is TargetStartedEventArgs targetStartedEventArgs) - { - TargetStarted?.Invoke(null, targetStartedEventArgs); - } - else if (buildEvent is TargetFinishedEventArgs targetFinishedEventArgs) - { - TargetFinished?.Invoke(null, targetFinishedEventArgs); - } - else if (buildEvent is ProjectStartedEventArgs projectStartedEventArgs) - { - ProjectStarted?.Invoke(null, projectStartedEventArgs); - } - else if (buildEvent is ProjectFinishedEventArgs projectFinishedEventArgs) - { - ProjectFinished?.Invoke(null, projectFinishedEventArgs); - } - else if (buildEvent is BuildStartedEventArgs buildStartedEventArgs) - { - BuildStarted?.Invoke(null, buildStartedEventArgs); - } - else if (buildEvent is BuildFinishedEventArgs buildFinishedEventArgs) - { - BuildFinished?.Invoke(null, buildFinishedEventArgs); - } - else if (buildEvent is CustomBuildEventArgs customBuildEventArgs) - { - CustomEventRaised?.Invoke(null, customBuildEventArgs); - } - else if (buildEvent is BuildStatusEventArgs buildStatusEventArgs) - { - StatusEventRaised?.Invoke(null, buildStatusEventArgs); - } - else if (buildEvent is BuildWarningEventArgs buildWarningEventArgs) - { - WarningRaised?.Invoke(null, buildWarningEventArgs); - } - else if (buildEvent is BuildErrorEventArgs buildErrorEventArgs) - { - ErrorRaised?.Invoke(null, buildErrorEventArgs); - } - - AnyEventRaised?.Invoke(null, buildEvent); - } - } - - public interface IRawLogEventsSource - { - public event Action>? BuildEventReceived; - } - - public sealed class BinaryLogReplayRawEventSource : BinaryLogReplayEventSourceBase, IRawLogEventsSource - { - public BinaryLogReplayRawEventSource() - : base(false) { } - - public event Action>? BuildEventReceived - { - add => RawBuildEventReceivedInternal += value; - remove => RawBuildEventReceivedInternal -= value; - } - } - - public class BinaryLogReplayEventSourceBase : IEmbeddedContentSource - { - /// Touches the static constructor - /// to ensure it initializes - /// and - static BinaryLogReplayEventSourceBase() + /// The full file path of the binary log file + public void Replay(string sourceFilePath) { - _ = ItemGroupLoggingHelper.ItemGroupIncludeLogMessagePrefix; + Replay(sourceFilePath, CancellationToken.None); } - internal BinaryLogReplayEventSourceBase(bool isStructured) - => _isStructuredReader = isStructured; - - private bool _isStructuredReader; - protected bool AllowForwardCompatibilityInternal { get; set; } = true; - protected event Action? OnErrorInternal; - protected event Action? BuildEventReceivedInternal; - protected event Action>? RawBuildEventReceivedInternal; - /// /// Creates a for the provided binary log file. /// Performs decompression and buffering in the optimal way. @@ -241,9 +110,14 @@ public static BinaryReader OpenReader(string sourceFilePath) /// /// /// Indicates whether the passed BinaryReader should be closed on disposing. - /// Indicates whether reading of future versions of logs should be allowed. + /// Unknown build events or unknown parts of known build events will be ignored if this is set to true. + /// Optional handler of recoverable errors during reading. /// BuildEventArgsReader over the given binlog file binary reader. - public static BuildEventArgsReader OpenBuildEventsReader(BinaryReader binaryReader, bool closeInput, bool allowForwardCompatibility = true) + public static BuildEventArgsReader OpenBuildEventsReader( + BinaryReader binaryReader, + bool closeInput, + bool allowForwardCompatibility = true, + Action? onRecoverableReadError = null) { int fileFormatVersion = binaryReader.ReadInt32(); @@ -255,7 +129,10 @@ public static BuildEventArgsReader OpenBuildEventsReader(BinaryReader binaryRead throw new NotSupportedException(text); } - return new BuildEventArgsReader(binaryReader, fileFormatVersion) { CloseInput = closeInput }; + return new BuildEventArgsReader(binaryReader, fileFormatVersion) + { + CloseInput = closeInput, + }; } /// @@ -272,77 +149,106 @@ public static BuildEventArgsReader OpenBuildEventsReader(string sourceFilePath) /// Read the provided binary log file and raise corresponding events for each BuildEventArgs /// /// The full file path of the binary log file - public void Replay(string sourceFilePath) + /// A indicating the replay should stop as soon as possible. + public void Replay(string sourceFilePath, CancellationToken cancellationToken) { - Replay(sourceFilePath, CancellationToken.None); + using var eventsReader = OpenBuildEventsReader(sourceFilePath); + Replay(eventsReader, cancellationToken); } /// /// Read the provided binary log file and raise corresponding events for each BuildEventArgs /// - /// The full file path of the binary log file + /// The binary log content binary reader - caller is responsible for disposing. /// A indicating the replay should stop as soon as possible. - public void Replay(string sourceFilePath, CancellationToken cancellationToken) - { - using var eventsReader = BinaryLogReplayEventSource.OpenBuildEventsReader(sourceFilePath); - Replay(eventsReader, cancellationToken); - } + public void Replay(BinaryReader binaryReader, CancellationToken cancellationToken) + => Replay(binaryReader, false, cancellationToken); /// /// Read the provided binary log file and raise corresponding events for each BuildEventArgs /// - /// The binary log content binary reader - caller is responsible for disposing. + /// The binary log content binary reader - caller is responsible for disposing, unless is set to true. /// Indicates whether the passed BinaryReader should be closed on disposing. /// A indicating the replay should stop as soon as possible. public void Replay(BinaryReader binaryReader, bool closeInput, CancellationToken cancellationToken) { - using var reader = BinaryLogReplayEventSource.OpenBuildEventsReader(binaryReader, closeInput, AllowForwardCompatibilityInternal); + using var reader = OpenBuildEventsReader(binaryReader, closeInput, AllowForwardCompatibility); Replay(reader, cancellationToken); } + /// + /// Read the provided binary log file and raise corresponding events for each BuildEventArgs + /// + /// The build events reader - caller is responsible for disposing. + /// A indicating the replay should stop as soon as possible. public void Replay(BuildEventArgsReader reader, CancellationToken cancellationToken) { - _fileFormatVersionRead?.Invoke(reader.FileFormatVersion); - reader.EmbeddedContentRead += _embeddedContentRead; + bool supportsForwardCompatibility = reader.FileFormatVersion >= 18; - if (_isStructuredReader) + // Allow any possible deferred subscriptions to be registered + if (HasStructuredEventsSubscribers || !supportsForwardCompatibility) { - ReplayStructured(reader, cancellationToken); + _onStructuredReadingOnly?.Invoke(); } else { - ReplayRaw(reader, cancellationToken); - } - } - - private void ReplayStructured(BuildEventArgsReader reader, CancellationToken cancellationToken) - { - while ( - !cancellationToken.IsCancellationRequested && - reader.Read(AllowForwardCompatibilityInternal, AllowForwardCompatibilityInternal, OnErrorInternal ?? (_ => { })) - is { } instance) - { - BuildEventReceivedInternal?.Invoke(instance); + _onRawReadingPossible?.Invoke(); } - } - private void ReplayRaw(BuildEventArgsReader reader, CancellationToken cancellationToken) - { _fileFormatVersionRead?.Invoke(reader.FileFormatVersion); reader.EmbeddedContentRead += _embeddedContentRead; + reader.StringReadDone += _stringReadDone; - while (!cancellationToken.IsCancellationRequested && reader.ReadRaw() is { Count: > 0 } instance) + if (HasStructuredEventsSubscribers || !supportsForwardCompatibility) + { + if (this._logDataSliceReceived != null) + { + throw new NotSupportedException( + "Structured events and raw events cannot be replayed at the same time."); + } + + // Forward compatibile reading makes sense only for structured events reading. + reader.SkipUnknownEvents = supportsForwardCompatibility && AllowForwardCompatibility; + reader.SkipUnknownEventParts = supportsForwardCompatibility && AllowForwardCompatibility; + reader.OnRecoverableReadError += OnRecoverableReadError; + + while (!cancellationToken.IsCancellationRequested && reader.Read() is { } instance) + { + Dispatch(instance); + } + } + else { - RawBuildEventReceivedInternal?.Invoke(instance); + if (this._logDataSliceReceived == null && + this._embeddedContentRead == null && + this._stringReadDone == null) + { + throw new NotSupportedException( + "No subscribers for any events."); + } + + while (!cancellationToken.IsCancellationRequested && reader.ReadRaw() is { } instance && + instance.RecordKind != BinaryLogRecordKind.EndOfFile) + { + _logDataSliceReceived?.Invoke(instance.RecordKind, instance.Stream); + } } } - private Action? _fileFormatVersionRead; - event Action ILogVersionInfo.FileFormatVersionRead + /// + void IRawLogEventsSource.DeferredInitialize( + Action onFileFormatVersionRead, + Action onRawReadingPossible, + Action onStructuredReadingOnly) { - add => _fileFormatVersionRead += value; - remove => _fileFormatVersionRead -= value; + this._fileFormatVersionRead += onFileFormatVersionRead; + this._onRawReadingPossible += onRawReadingPossible; + this._onStructuredReadingOnly += onStructuredReadingOnly; } + + private Action? _onRawReadingPossible; + private Action? _onStructuredReadingOnly; + private Action? _fileFormatVersionRead; private Action? _embeddedContentRead; /// event Action? IEmbeddedContentSource.EmbeddedContentRead @@ -352,5 +258,21 @@ event Action? IEmbeddedContentSource.EmbeddedContentRe add => _embeddedContentRead += value; remove => _embeddedContentRead -= value; } + + private Action? _stringReadDone; + /// + event Action? IBuildEventStringsReader.StringReadDone + { + add => _stringReadDone += value; + remove => _stringReadDone -= value; + } + + private Action? _logDataSliceReceived; + /// + event Action? IRawLogEventsSource.LogDataSliceReceived + { + add => _logDataSliceReceived += value; + remove => _logDataSliceReceived -= value; + } } } diff --git a/src/Build/Logging/BinaryLogger/BinaryLogger.cs b/src/Build/Logging/BinaryLogger/BinaryLogger.cs index 94b28d10eb2..f1f684d2a97 100644 --- a/src/Build/Logging/BinaryLogger/BinaryLogger.cs +++ b/src/Build/Logging/BinaryLogger/BinaryLogger.cs @@ -138,9 +138,7 @@ public void Initialize(IEventSource eventSource) Traits.Instance.EscapeHatches.LogProjectImports = true; bool logPropertiesAndItemsAfterEvaluation = Traits.Instance.EscapeHatches.LogPropertiesAndItemsAfterEvaluation ?? true; - bool replayInitialInfo; - ILogVersionInfo versionInfo = null; - ProcessParameters(out replayInitialInfo); + ProcessParameters(out bool omitInitialInfo); try { @@ -176,20 +174,6 @@ public void Initialize(IEventSource eventSource) { eventSource4.IncludeEvaluationPropertiesAndItems(); } - - if (eventSource is IEmbeddedContentSource embeddedFilesSource) - { - if (CollectProjectImports == ProjectImportsCollectionMode.Replay) - { - embeddedFilesSource.EmbeddedContentRead += args => - eventArgsWriter.WriteBlob(args.ContentKind.ToBinaryLogRecordKind(), args.ContentStream, args.Length); - } - - if (replayInitialInfo) - { - versionInfo = embeddedFilesSource; - } - } } catch (Exception e) { @@ -215,19 +199,46 @@ public void Initialize(IEventSource eventSource) eventArgsWriter.EmbedFile += EventArgsWriter_EmbedFile; } - if (versionInfo == null) + if (eventSource is IBinaryLogReplaySource replayEventsSource) { - binaryWriter.Write(FileFormatVersion); - LogInitialInfo(); + if (CollectProjectImports == ProjectImportsCollectionMode.Replay) + { + replayEventsSource.EmbeddedContentRead += args => + eventArgsWriter.WriteBlob(args.ContentKind.ToBinaryLogRecordKind(), args.ContentStream); + } + + // If raw events are provided - let's try to use the advantage. + // But other subscribers can later on subscribe to structured events - + // for this reason we do only subscribe delayed. + replayEventsSource.DeferredInitialize( + version => binaryWriter.Write(version), + // For raw events we cannot write the initial info - as we cannot write + // at the same time as raw events are being written - this would break the deduplicated strings store. + () => + { + replayEventsSource.LogDataSliceReceived += RawEvents_LogDataSliceReceived; + // Replay separated strings here as well (and do not deduplicate! It would skew string indexes) + replayEventsSource.StringReadDone += strArg => eventArgsWriter.WriteStringRecord(strArg.StringToBeUsed); + }, + SubscribeToStructuredEvents); } else { - versionInfo.FileFormatVersionRead += version => binaryWriter.Write(version); + binaryWriter.Write(FileFormatVersion); + SubscribeToStructuredEvents(); } - eventSource.AnyEventRaised += EventSource_AnyEventRaised; - KnownTelemetry.LoggingConfigurationTelemetry.BinaryLogger = true; + + void SubscribeToStructuredEvents() + { + if (!omitInitialInfo) + { + LogInitialInfo(); + } + + eventSource.AnyEventRaised += EventSource_AnyEventRaised; + } } private void EventArgsWriter_EmbedFile(string filePath) @@ -290,6 +301,11 @@ public void Shutdown() } } + private void RawEvents_LogDataSliceReceived(BinaryLogRecordKind recordKind, Stream stream) + { + eventArgsWriter.WriteBlob(recordKind, stream); + } + private void EventSource_AnyEventRaised(object sender, BuildEventArgs e) { Write(e); @@ -337,14 +353,14 @@ private void CollectImports(BuildEventArgs e) /// /// /// - private void ProcessParameters(out bool replayInitialInfo) + private void ProcessParameters(out bool omitInitialInfo) { if (Parameters == null) { throw new LoggerException(ResourceUtilities.FormatResourceStringStripCodeAndKeyword("InvalidBinaryLoggerParameters", "")); } - replayInitialInfo = false; + omitInitialInfo = false; var parameters = Parameters.Split(MSBuildConstants.SemicolonChar, StringSplitOptions.RemoveEmptyEntries); foreach (var parameter in parameters) { @@ -364,9 +380,9 @@ private void ProcessParameters(out bool replayInitialInfo) { CollectProjectImports = ProjectImportsCollectionMode.Replay; } - else if (string.Equals(parameter, "ReplayInitialInfo", StringComparison.OrdinalIgnoreCase)) + else if (string.Equals(parameter, "OmitInitialInfo", StringComparison.OrdinalIgnoreCase)) { - replayInitialInfo = true; + omitInitialInfo = true; } else if (parameter.EndsWith(".binlog", StringComparison.OrdinalIgnoreCase)) { diff --git a/src/Build/Logging/BinaryLogger/BuildEventArgsDispatcher.cs b/src/Build/Logging/BinaryLogger/BuildEventArgsDispatcher.cs index 4a50a3d8e8b..db24d791ed2 100644 --- a/src/Build/Logging/BinaryLogger/BuildEventArgsDispatcher.cs +++ b/src/Build/Logging/BinaryLogger/BuildEventArgsDispatcher.cs @@ -85,6 +85,22 @@ public class EventArgsDispatcher : IEventSource /// public event BuildMessageEventHandler MessageRaised; + internal bool HasStructuredEventsSubscribers => + AnyEventRaised != null || + StatusEventRaised != null || + CustomEventRaised != null || + BuildStarted != null || + BuildFinished != null || + ProjectStarted != null || + ProjectFinished != null || + TargetStarted != null || + TargetFinished != null || + TaskStarted != null || + TaskFinished != null || + ErrorRaised != null || + WarningRaised != null || + MessageRaised != null; + /// /// Raise one of the events that is appropriate for the type of the BuildEventArgs /// diff --git a/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs b/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs index 1b62948bc1d..245d9adcdd4 100644 --- a/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs +++ b/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs @@ -78,6 +78,24 @@ public BuildEventArgsReader(BinaryReader binaryReader, int fileFormatVersion) /// public bool CloseInput { private get; set; } = false; + /// + /// Indicates whether unknown BuildEvents should be silently skipped. Read returns null otherwise. + /// Parameter is supported only if the file format supports forward compatible reading (version is 18 or higher). + /// + public bool SkipUnknownEvents { private get; set; } = false; + + /// + /// Indicates whether unread parts of BuildEvents (probably added in newer format of particular BuildEvent)should be silently skipped. Exception thrown otherwise. + /// Parameter is supported only if the file format supports forward compatible reading (version is 18 or higher). + /// + public bool SkipUnknownEventParts { private get; set; } = false; + + /// + /// Receives recoverable errors during reading. + /// Applicable mainly when or is set to true."/> + /// + public event Action? OnRecoverableReadError; + public void Dispose() { stringStorage.Dispose(); @@ -106,85 +124,41 @@ public void Dispose() /// public event Action? ArchiveFileEncountered; - private byte[]? _pooledBuffer; - private BinaryWriter? _pooledBinaryWriter; + private SubStream? _lastSubStream; /// /// Reads the next serialized log record from the . /// /// ArraySegment containing serialized BuildEventArgs record - public ArraySegment ReadRaw() + internal (BinaryLogRecordKind RecordKind, Stream Stream) ReadRaw() { - // TODO: For even better performance experiment with returning a stream - // it would need to be concatenated Stream of a MemoryStream with kind and size - // and SubStream over the binaryReader.BaseStream of the given size. - // We'd need to validate that reader reads exactly the given size, or skip rest on their behalf. - - BinaryLogRecordKind recordKind = ReadTillNextEvent(); + // This method is internal and condition is checked once before calling in loop, + // so avoiding it here on each call. + // But keeping it for documentation purposes - in case someone will try to call it and debug issues. + ////if (fileFormatVersion < 18) + ////{ + //// throw new InvalidOperationException( + //// $"Raw data reading is not supported for file format version {fileFormatVersion} (needs >=18)."); + ////} - if (recordKind == BinaryLogRecordKind.EndOfFile) + if (!(_lastSubStream?.IsAtEnd ?? true)) { - ReturnPooledBuffer(); - return new ArraySegment(); + throw new InvalidDataException($"Raw data slice for record {recordNumber} was not fully read."); } - int serializedEventLength = ReadInt32(); - - // The BinaryLogRecordKind and size needs to be added back to the buffer. - // We do not know the exact size needed to write those two upfront though, - // so we need to call memoryStream.Position to find out. - (BinaryWriter binaryWriter, MemoryStream memoryStream, byte[] buffer) = - RentPooledBinaryWriter(serializedEventLength + 2 * sizeof(int)); - binaryWriter.Write7BitEncodedInt((int)recordKind); - binaryWriter.Write7BitEncodedInt(serializedEventLength); - - binaryReader.BaseStream.ReadAtLeast(buffer, (int)memoryStream.Position, serializedEventLength, throwOnEndOfStream: true); - // Size of the raw message and the preceding type and size info - return new ArraySegment(buffer, 0, serializedEventLength + (int)memoryStream.Position); - } + BinaryLogRecordKind recordKind = ReadTillNextEvent(IsTextualDataRecord); - private byte[] RentPooledBuffer(int minimumBytes) - { - if (_pooledBuffer == null || _pooledBuffer.Length < minimumBytes) - { - ReturnPooledBuffer(); - _pooledBuffer = ArrayPool.Shared.Rent(minimumBytes); - } - - return _pooledBuffer; - } - - private (BinaryWriter binaryWriter, MemoryStream memoryStream, byte[] buffer) RentPooledBinaryWriter(int minimumBytes) - { - if (_pooledBuffer == null || _pooledBuffer.Length < minimumBytes) + if (recordKind == BinaryLogRecordKind.EndOfFile) { - ReturnPooledBuffer(); - _pooledBuffer = ArrayPool.Shared.Rent(minimumBytes); - _pooledBinaryWriter = null; + return new(recordKind, Stream.Null); } - MemoryStream ms; - if (_pooledBinaryWriter == null) - { - ms = new(_pooledBuffer); - _pooledBinaryWriter = new BinaryWriter(ms); - } - else - { - ms = (_pooledBinaryWriter.BaseStream as MemoryStream)!; - } + int serializedEventLength = ReadInt32(); + Stream stream = binaryReader.BaseStream.Slice(serializedEventLength); - return (_pooledBinaryWriter, ms, _pooledBuffer); - } + _lastSubStream = stream as SubStream; - private void ReturnPooledBuffer() - { - if (_pooledBuffer != null) - { - ArrayPool.Shared.Return(_pooledBuffer); - _pooledBuffer = null; - _pooledBinaryWriter = null; - } + return new(recordKind, stream); } /// @@ -196,35 +170,30 @@ private void ReturnPooledBuffer() /// public BuildEventArgs? Read() { - return Read(skipUnknownEvents: false, skipUnknownEventParts: false, _ => { }); - } + // todo - flip this into a properties - not to check on each call and to avoid params passing + if ((SkipUnknownEvents || SkipUnknownEventParts) && fileFormatVersion < 18) + { + throw new InvalidOperationException( + $"Forward compatible reading is not supported for file format version {fileFormatVersion} (needs >=18)."); + } - /// - /// Reads the next log record from the . - /// - /// Indicates whether unknown BuildEvents should be silently skipped. Read returns null otherwise. - /// Indicates whether unread parts of BuildEvents (probably added in newer format of particular BuildEvent)should be silently skipped. Exception thrown otherwise. - /// Receives recoverable errors during reading. - /// - /// The next . - /// If there are no more records, returns . - /// - /// Thrown based on and arguments. - public BuildEventArgs? Read(bool skipUnknownEvents, bool skipUnknownEventParts, Action onError) - { BuildEventArgs? result = null; while (result == null) { - BinaryLogRecordKind recordKind = ReadTillNextEvent(); + BinaryLogRecordKind recordKind = ReadTillNextEvent(IsAuxiliaryRecord); if (recordKind == BinaryLogRecordKind.EndOfFile) { return null; } - int serializedEventLength = ReadInt32(); // record length - - long preEventPosition = _canSeek ? binaryReader.BaseStream.Position : 0; + int serializedEventLength = 0; + long preEventPosition = 0; + if (fileFormatVersion >= 18) + { + serializedEventLength = ReadInt32(); // record length + preEventPosition = _canSeek ? binaryReader.BaseStream.Position : 0; + } switch (recordKind) { @@ -301,9 +270,9 @@ private void ReturnPooledBuffer() result = ReadAssemblyLoadEventArgs(); break; default: - onError( - $"BuildEvent record number {recordNumber} (serialized size: {serializedEventLength}) is of unsupported type: {recordKind}.{(skipUnknownEvents ? " Skipping it." : string.Empty)}"); - if (skipUnknownEvents) + OnRecoverableReadError?.Invoke( + $"BuildEvent record number {recordNumber} (serialized size: {serializedEventLength}) is of unsupported type: {recordKind}.{(SkipUnknownEvents ? " Skipping it." : string.Empty)}"); + if (SkipUnknownEvents) { SkipBytes(serializedEventLength); } @@ -314,21 +283,24 @@ private void ReturnPooledBuffer() break; } - long postEventPosition = _canSeek ? binaryReader.BaseStream.Position : serializedEventLength; - int bytesRead = (int)(postEventPosition - preEventPosition); - if (bytesRead != serializedEventLength) + if (fileFormatVersion >= 18) { - string error = - $"BuildEvent record number {recordNumber} was expected to read exactly {serializedEventLength} bytes from the stream, but read {bytesRead} instead."; - - if (skipUnknownEventParts && bytesRead < serializedEventLength) + long postEventPosition = _canSeek ? binaryReader.BaseStream.Position : serializedEventLength; + int bytesRead = (int)(postEventPosition - preEventPosition); + if (bytesRead != serializedEventLength) { - onError(error); - SkipBytes(serializedEventLength - bytesRead); - } - else - { - throw new InvalidDataException(error); + string error = + $"BuildEvent record number {recordNumber} was expected to read exactly {serializedEventLength} bytes from the stream, but read {bytesRead} instead."; + + if (SkipUnknownEventParts && bytesRead < serializedEventLength) + { + OnRecoverableReadError?.Invoke(error); + SkipBytes(serializedEventLength - bytesRead); + } + else + { + throw new InvalidDataException(error); + } } } @@ -346,19 +318,19 @@ private void SkipBytes(int count) } else { - byte[] buffer = RentPooledBuffer(count); + byte[] buffer = ArrayPool.Shared.Rent(count); + using var _ = new CleanupScope(() => ArrayPool.Shared.Return(buffer)); binaryReader.BaseStream.ReadAtLeast(buffer, 0, count, throwOnEndOfStream: true); - ReturnPooledBuffer(); } } - private BinaryLogRecordKind ReadTillNextEvent() + private BinaryLogRecordKind ReadTillNextEvent(Func isPreprocessRecord) { BinaryLogRecordKind recordKind = (BinaryLogRecordKind)ReadInt32(); // Skip over data storage records since they don't result in a BuildEventArgs. // just ingest their data and continue. - while (IsAuxiliaryRecord(recordKind)) + while (isPreprocessRecord(recordKind)) { // these are ordered by commonality if (recordKind == BinaryLogRecordKind.String) @@ -389,6 +361,12 @@ private static bool IsAuxiliaryRecord(BinaryLogRecordKind recordKind) || recordKind == BinaryLogRecordKind.ProjectImportArchive; } + private static bool IsTextualDataRecord(BinaryLogRecordKind recordKind) + { + return recordKind == BinaryLogRecordKind.String + || recordKind == BinaryLogRecordKind.ProjectImportArchive; + } + private void ReadEmbeddedContent(BinaryLogRecordKind recordKind) { int length = ReadInt32(); @@ -444,6 +422,7 @@ private void ReadEmbeddedContent(BinaryLogRecordKind recordKind) } } + // Once embedded files are replayed one by one - we can send the resulting stream to subscriber if (EmbeddedContentRead != null) { projectImportsCollector!.ProcessResult( @@ -466,6 +445,11 @@ private void ReadEmbeddedContent(BinaryLogRecordKind recordKind) private void ReadNameValueList() { + if (fileFormatVersion >= 18) + { + _ = ReadInt32(); // buffer size, not used in structured reading + } + int count = ReadInt32(); var list = new (int, int)[count]; @@ -507,7 +491,7 @@ private IDictionary GetNameValueList(int id) // this should never happen for valid binlogs throw new InvalidDataException( - $"NameValueList record number {recordNumber} is invalid: index {id} is not within {stringRecords.Count}."); + $"NameValueList record number {recordNumber} is invalid: index {id} is not within {nameValueListRecords.Count}."); } private readonly StringReadEventArgs stringReadEventArgs = new StringReadEventArgs(string.Empty); diff --git a/src/Build/Logging/BinaryLogger/BuildEventArgsWriter.cs b/src/Build/Logging/BinaryLogger/BuildEventArgsWriter.cs index a0246b5d4fe..503c185791e 100644 --- a/src/Build/Logging/BinaryLogger/BuildEventArgsWriter.cs +++ b/src/Build/Logging/BinaryLogger/BuildEventArgsWriter.cs @@ -36,6 +36,13 @@ internal class BuildEventArgsWriter /// private readonly MemoryStream currentRecordStream; + /// + /// For NameValueList we need to prefix the storage size + /// (distinct from values count due to variable int encoding) + /// So using same technique as with 'currentRecordStream'. + /// + private readonly MemoryStream nameValueListStream = new MemoryStream(256); + /// /// The binary writer around the originalStream. /// @@ -135,12 +142,16 @@ public void Write(BuildEventArgs e) { BinaryLogRecordKind eventKind = WriteCore(e); - Write(eventKind); - Write((int)currentRecordStream.Length); + FlushRecordToFinalStream(eventKind, currentRecordStream); + } - // flush the current record and clear the MemoryStream to prepare for next use - currentRecordStream.WriteTo(originalStream); - currentRecordStream.SetLength(0); + private void FlushRecordToFinalStream(BinaryLogRecordKind recordKind, MemoryStream recordStream) + { + using var redirectionScope = RedirectWritesToOriginalWriter(); + Write(recordKind); + Write((int)recordStream.Length); + recordStream.WriteTo(originalStream); + recordStream.SetLength(0); } /* @@ -232,7 +243,7 @@ private BinaryLogRecordKind WriteCore(BuildEventArgs e) } } - public void WriteBlob(BinaryLogRecordKind kind, Stream stream, int? length = null) + public void WriteBlob(BinaryLogRecordKind kind, Stream stream) { if (stream.CanSeek && stream.Length > int.MaxValue) { @@ -244,8 +255,8 @@ public void WriteBlob(BinaryLogRecordKind kind, Stream stream, int? length = nul using var redirection = RedirectWritesToOriginalWriter(); Write(kind); - Write(length ?? (int)stream.Length); - Write(stream, length); + Write((int)stream.Length); + Write(stream); } /// @@ -255,23 +266,13 @@ public void WriteBlob(BinaryLogRecordKind kind, Stream stream, int? length = nul /// private IDisposable RedirectWritesToOriginalWriter() { - binaryWriter = originalBinaryWriter; - return new RedirectionScope(this); + return RedirectWritesToDifferentWriter(originalBinaryWriter, currentRecordWriter); } - private struct RedirectionScope : IDisposable + private IDisposable RedirectWritesToDifferentWriter(BinaryWriter inScopeWriter, BinaryWriter afterScopeWriter) { - private readonly BuildEventArgsWriter _writer; - - public RedirectionScope(BuildEventArgsWriter buildEventArgsWriter) - { - _writer = buildEventArgsWriter; - } - - public void Dispose() - { - _writer.binaryWriter = _writer.currentRecordWriter; - } + binaryWriter = inScopeWriter; + return new CleanupScope(() => binaryWriter = afterScopeWriter); } private BinaryLogRecordKind Write(BuildStartedEventArgs e) @@ -1063,19 +1064,26 @@ private void WriteNameValueList() /// private void WriteNameValueListRecord() { - // Switch the binaryWriter used by the Write* methods to the direct underlying stream writer. // We want this record to precede the record we're currently writing to currentRecordWriter - // which is backed by a MemoryStream buffer - using var redirectionScope = RedirectWritesToOriginalWriter(); + // We as well want to know the storage size (differs from nameValueIndexListBuffer.Count as + // we use variable integer encoding). + // So we redirect the writes to a MemoryStream and then flush the record to the final stream. + // All that is redirected away from the 'currentRecordStream' - that will be flushed last + + var nameValueListBw = new BinaryWriter(nameValueListStream); - Write(BinaryLogRecordKind.NameValueList); - Write(nameValueIndexListBuffer.Count); - for (int i = 0; i < nameValueListBuffer.Count; i++) + using (var _ = RedirectWritesToDifferentWriter(nameValueListBw, binaryWriter)) { - var kvp = nameValueIndexListBuffer[i]; - Write(kvp.Key); - Write(kvp.Value); + Write(nameValueIndexListBuffer.Count); + for (int i = 0; i < nameValueListBuffer.Count; i++) + { + var kvp = nameValueIndexListBuffer[i]; + Write(kvp.Key); + Write(kvp.Value); + } } + + FlushRecordToFinalStream(BinaryLogRecordKind.NameValueList, nameValueListStream); } /// @@ -1124,35 +1132,10 @@ private void Write(byte[] bytes) binaryWriter.Write(bytes); } - private void Write(Stream stream, int? length) + private void Write(Stream stream) { Stream destinationStream = binaryWriter.BaseStream; - if (length == null) - { - stream.CopyTo(destinationStream); - return; - } - - // borrowed from runtime from Stream.cs - const int defaultCopyBufferSize = 81920; - int bufferSize = Math.Min(defaultCopyBufferSize, length.Value); - - byte[] buffer = ArrayPool.Shared.Rent(bufferSize); - try - { - int bytesRead; - while ( - length > 0 && - (bytesRead = stream.Read(buffer, 0, Math.Min(buffer.Length, length.Value))) != 0) - { - destinationStream.Write(buffer, 0, bytesRead); - length -= bytesRead; - } - } - finally - { - ArrayPool.Shared.Return(buffer); - } + stream.CopyTo(destinationStream); } private void Write(byte b) @@ -1209,7 +1192,7 @@ private void WriteDeduplicatedString(string text) return (recordId, hash); } - private void WriteStringRecord(string text) + internal void WriteStringRecord(string text) { using var redirectionScope = RedirectWritesToOriginalWriter(); diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/EmbeddedContentEventArgs.cs b/src/Build/Logging/BinaryLogger/Postprocessing/EmbeddedContentEventArgs.cs index 68969f2af4a..4ddd1ae456e 100644 --- a/src/Build/Logging/BinaryLogger/Postprocessing/EmbeddedContentEventArgs.cs +++ b/src/Build/Logging/BinaryLogger/Postprocessing/EmbeddedContentEventArgs.cs @@ -8,13 +8,6 @@ namespace Microsoft.Build.Logging { internal sealed class EmbeddedContentEventArgs : EventArgs { - public EmbeddedContentEventArgs(EmbeddedContentKind contentKind, Stream contentStream, int length) - { - ContentKind = contentKind; - ContentStream = contentStream; - Length = length; - } - public EmbeddedContentEventArgs(EmbeddedContentKind contentKind, Stream contentStream) { ContentKind = contentKind; @@ -23,6 +16,5 @@ public EmbeddedContentEventArgs(EmbeddedContentKind contentKind, Stream contentS public EmbeddedContentKind ContentKind { get; } public Stream ContentStream { get; } - public int? Length { get; } } } diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/IEmbeddedContentSource.cs b/src/Build/Logging/BinaryLogger/Postprocessing/IEmbeddedContentSource.cs index eb9262939fa..c07c3b2ee78 100644 --- a/src/Build/Logging/BinaryLogger/Postprocessing/IEmbeddedContentSource.cs +++ b/src/Build/Logging/BinaryLogger/Postprocessing/IEmbeddedContentSource.cs @@ -5,13 +5,7 @@ namespace Microsoft.Build.Logging { - - internal interface ILogVersionInfo - { - event Action FileFormatVersionRead; - } - - internal interface IEmbeddedContentSource : ILogVersionInfo + internal interface IEmbeddedContentSource { /// diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/SubStream.cs b/src/Build/Logging/BinaryLogger/Postprocessing/SubStream.cs index 277c9ac66c0..b35e4e7e97b 100644 --- a/src/Build/Logging/BinaryLogger/Postprocessing/SubStream.cs +++ b/src/Build/Logging/BinaryLogger/Postprocessing/SubStream.cs @@ -31,6 +31,8 @@ public SubStream(Stream stream, long length) } } + public bool IsAtEnd => _position >= _length; + public override bool CanRead => true; public override bool CanSeek => false; From 4726733acf7f4f49631329dad52e7e395bcb1d02 Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Fri, 6 Oct 2023 09:45:17 +0200 Subject: [PATCH 11/49] Bugfix failing tests --- src/Build.UnitTests/BinaryLogger_Tests.cs | 4 ---- .../Logging/BinaryLogger/BinaryLogReplayEventSource.cs | 6 ------ src/Build/Logging/BinaryLogger/BinaryLogger.cs | 4 ++-- src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs | 2 +- 4 files changed, 3 insertions(+), 13 deletions(-) diff --git a/src/Build.UnitTests/BinaryLogger_Tests.cs b/src/Build.UnitTests/BinaryLogger_Tests.cs index 6282d833a0c..655102699a7 100644 --- a/src/Build.UnitTests/BinaryLogger_Tests.cs +++ b/src/Build.UnitTests/BinaryLogger_Tests.cs @@ -222,16 +222,12 @@ public void TestBinaryLoggerRoundtripEquality(string projectText, BinlogRoundtri logReader.BuildFinished += (_, _) => { }; } - _logFile = @"C:\tmp\trash\console\packed.binlog"; - BinaryLogger outputBinlog = new BinaryLogger() { Parameters = $"LogFile={replayedLogFile};ProjectImports=Replay;OmitInitialInfo" }; outputBinlog.Initialize(logReader); logReader.Replay(_logFile); - // TODO: remove here - // logReader.Replay(originalBuildEventsReader, CancellationToken.None); outputBinlog.Shutdown(); AssertBinlogsHaveEqualContent(_logFile, replayedLogFile); diff --git a/src/Build/Logging/BinaryLogger/BinaryLogReplayEventSource.cs b/src/Build/Logging/BinaryLogger/BinaryLogReplayEventSource.cs index a4374b8bbdc..087ccebc898 100644 --- a/src/Build/Logging/BinaryLogger/BinaryLogReplayEventSource.cs +++ b/src/Build/Logging/BinaryLogger/BinaryLogReplayEventSource.cs @@ -25,11 +25,9 @@ internal interface IRawLogEventsSource /// At this point all other possible subscribers should be already subscribed - /// so it can be determined if raw events or structured events should be replayed. /// - /// /// /// void DeferredInitialize( - Action onFileFormatVersionRead, Action onRawReadingPossible, Action onStructuredReadingOnly); } @@ -195,7 +193,6 @@ public void Replay(BuildEventArgsReader reader, CancellationToken cancellationTo _onRawReadingPossible?.Invoke(); } - _fileFormatVersionRead?.Invoke(reader.FileFormatVersion); reader.EmbeddedContentRead += _embeddedContentRead; reader.StringReadDone += _stringReadDone; @@ -237,18 +234,15 @@ public void Replay(BuildEventArgsReader reader, CancellationToken cancellationTo /// void IRawLogEventsSource.DeferredInitialize( - Action onFileFormatVersionRead, Action onRawReadingPossible, Action onStructuredReadingOnly) { - this._fileFormatVersionRead += onFileFormatVersionRead; this._onRawReadingPossible += onRawReadingPossible; this._onStructuredReadingOnly += onStructuredReadingOnly; } private Action? _onRawReadingPossible; private Action? _onStructuredReadingOnly; - private Action? _fileFormatVersionRead; private Action? _embeddedContentRead; /// event Action? IEmbeddedContentSource.EmbeddedContentRead diff --git a/src/Build/Logging/BinaryLogger/BinaryLogger.cs b/src/Build/Logging/BinaryLogger/BinaryLogger.cs index f1f684d2a97..fd3c8d5e3ca 100644 --- a/src/Build/Logging/BinaryLogger/BinaryLogger.cs +++ b/src/Build/Logging/BinaryLogger/BinaryLogger.cs @@ -199,6 +199,8 @@ public void Initialize(IEventSource eventSource) eventArgsWriter.EmbedFile += EventArgsWriter_EmbedFile; } + binaryWriter.Write(FileFormatVersion); + if (eventSource is IBinaryLogReplaySource replayEventsSource) { if (CollectProjectImports == ProjectImportsCollectionMode.Replay) @@ -211,7 +213,6 @@ public void Initialize(IEventSource eventSource) // But other subscribers can later on subscribe to structured events - // for this reason we do only subscribe delayed. replayEventsSource.DeferredInitialize( - version => binaryWriter.Write(version), // For raw events we cannot write the initial info - as we cannot write // at the same time as raw events are being written - this would break the deduplicated strings store. () => @@ -224,7 +225,6 @@ public void Initialize(IEventSource eventSource) } else { - binaryWriter.Write(FileFormatVersion); SubscribeToStructuredEvents(); } diff --git a/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs b/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs index 245d9adcdd4..2ca7dbbbb87 100644 --- a/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs +++ b/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs @@ -272,7 +272,7 @@ public void Dispose() default: OnRecoverableReadError?.Invoke( $"BuildEvent record number {recordNumber} (serialized size: {serializedEventLength}) is of unsupported type: {recordKind}.{(SkipUnknownEvents ? " Skipping it." : string.Empty)}"); - if (SkipUnknownEvents) + if (SkipUnknownEvents && serializedEventLength > 0) { SkipBytes(serializedEventLength); } From d4e72f67ebf29faf695f569b3f13543c07ee8a5e Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Fri, 6 Oct 2023 11:41:26 +0200 Subject: [PATCH 12/49] Adjust unit test to compare unpacked binlogs --- src/Build.UnitTests/BinaryLogger_Tests.cs | 24 +++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/src/Build.UnitTests/BinaryLogger_Tests.cs b/src/Build.UnitTests/BinaryLogger_Tests.cs index 655102699a7..bb4ee876aac 100644 --- a/src/Build.UnitTests/BinaryLogger_Tests.cs +++ b/src/Build.UnitTests/BinaryLogger_Tests.cs @@ -5,6 +5,7 @@ using System.Collections.Generic; using System.IO; using System.IO.Compression; +using System.Linq; using System.Reflection; using System.Text; using Microsoft.Build.BackEnd.Logging; @@ -233,7 +234,26 @@ public void TestBinaryLoggerRoundtripEquality(string projectText, BinlogRoundtri AssertBinlogsHaveEqualContent(_logFile, replayedLogFile); // If this assertation complicates development - it can possibly be removed // The structured equality above should be enough. - AssertFilesAreBinaryEqual(_logFile, replayedLogFile); + AssertFilesAreBinaryEqualAfterUnpack(_logFile, replayedLogFile); + } + + private static void AssertFilesAreBinaryEqualAfterUnpack(string firstPath, string secondPath) + { + using var br1 = BinaryLogReplayEventSource.OpenReader(firstPath); + using var br2 = BinaryLogReplayEventSource.OpenReader(secondPath); + const int bufferSize = 4096; + + int readCount = 0; + while (br1.ReadBytes(bufferSize) is { Length: > 0 } bytes1) + { + var bytes2 = br2.ReadBytes(bufferSize); + + bytes1.SequenceEqual(bytes2).ShouldBeTrue(() => + $"Buffers starting at position {readCount} differ. First:{Environment.NewLine}{string.Join(",", bytes1)}{Environment.NewLine}Second:{Environment.NewLine}{string.Join(",", bytes2)}"); + readCount += bufferSize; + } + + br2.ReadBytes(bufferSize).Length.ShouldBe(0, "Second buffer contains byt after first end"); } private static void AssertFilesAreBinaryEqual(string firstPath, string secondPath) @@ -254,7 +274,7 @@ private static void AssertFilesAreBinaryEqual(string firstPath, string secondPat using FileStream fs1 = first.OpenRead(); using FileStream fs2 = second.OpenRead(); - for (int i = 0; i < Math.Min(first.Length,second.Length); i++) + for (int i = 0; i < Math.Min(first.Length, second.Length); i++) { byte b1 = (byte)fs1.ReadByte(); byte b2 = (byte)fs2.ReadByte(); From 3d947b8544bfdca1fde5f660e6197b57a9e15d8f Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Mon, 9 Oct 2023 16:31:54 +0200 Subject: [PATCH 13/49] Add version description --- src/Build/Logging/BinaryLogger/BinaryLogger.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Build/Logging/BinaryLogger/BinaryLogger.cs b/src/Build/Logging/BinaryLogger/BinaryLogger.cs index fd3c8d5e3ca..9da915cb2d6 100644 --- a/src/Build/Logging/BinaryLogger/BinaryLogger.cs +++ b/src/Build/Logging/BinaryLogger/BinaryLogger.cs @@ -66,6 +66,7 @@ public sealed class BinaryLogger : ILogger // version 18: // - Making ProjectStartedEventArgs, ProjectEvaluationFinishedEventArgs, AssemblyLoadBuildEventArgs equal // between de/serialization roundtrips. + // - Adding serialized events lengths - to support forward compatible reading internal const int FileFormatVersion = 18; private Stream stream; From dd6b0ada030ea2d8c19154bad5c1ad25cf9b145e Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Tue, 10 Oct 2023 22:04:09 +0200 Subject: [PATCH 14/49] Reflecting PR comments --- src/Build.UnitTests/BinaryLogger_Tests.cs | 41 ++++----- .../BinaryLogger/BinaryLogRecordKind.cs | 2 +- .../BinaryLogReplayEventSource.cs | 36 ++++++-- .../Logging/BinaryLogger/BinaryLogger.cs | 26 +++--- .../BinaryLogger/BuildEventArgsReader.cs | 35 ++++---- .../Postprocessing/ArchiveFile.cs | 16 +--- .../Postprocessing/ArchiveFileEventArgs.cs | 2 +- .../Postprocessing/CleanupScope.cs | 2 +- .../Postprocessing/EmbeddedContentKind.cs | 8 +- .../Postprocessing/GreedyBufferedStream.cs | 83 ------------------- .../IBuildEventStringsReader.cs | 8 ++ .../Postprocessing/StreamExtensions.cs | 34 +++++++- .../BinaryLogger/ProjectImportsCollector.cs | 39 +++------ src/Build/Microsoft.Build.csproj | 2 +- src/Framework/Traits.cs | 6 -- 15 files changed, 130 insertions(+), 210 deletions(-) delete mode 100644 src/Build/Logging/BinaryLogger/Postprocessing/GreedyBufferedStream.cs diff --git a/src/Build.UnitTests/BinaryLogger_Tests.cs b/src/Build.UnitTests/BinaryLogger_Tests.cs index bb4ee876aac..7e5762f0768 100644 --- a/src/Build.UnitTests/BinaryLogger_Tests.cs +++ b/src/Build.UnitTests/BinaryLogger_Tests.cs @@ -129,24 +129,16 @@ public void TestBinaryLoggerRoundtrip(string projectText, BinlogRoundtripTestRep fileToReplay = _logFile; break; case BinlogRoundtripTestReplayMode.Structured: - { - var logReader = new BinaryLogReplayEventSource(); - fileToReplay = _env.ExpectFile(".binlog").Path; - // need dummy handler to force structured replay - logReader.BuildFinished += (_, _) => { }; - BinaryLogger outputBinlog = new BinaryLogger() - { - Parameters = fileToReplay - }; - outputBinlog.Initialize(logReader); - logReader.Replay(_logFile); - outputBinlog.Shutdown(); - } - break; case BinlogRoundtripTestReplayMode.RawEvents: { var logReader = new BinaryLogReplayEventSource(); fileToReplay = _env.ExpectFile(".binlog").Path; + if (replayMode == BinlogRoundtripTestReplayMode.Structured) + { + // need dummy handler to force structured replay + logReader.BuildFinished += (_, _) => { }; + } + BinaryLogger outputBinlog = new BinaryLogger() { Parameters = fileToReplay @@ -193,6 +185,21 @@ public void TestBinaryLoggerRoundtrip(string projectText, BinlogRoundtripTestRep parallelActual.ShouldContainWithoutWhitespace(parallelExpected); } + /// + /// This test validate then binlog file content is identical upon replaying. + /// The identity can be defined via 3 ways: + /// * byte-for-byte equality + /// * byte-for-byte equality of unzipped content + /// * structured equality of events + /// + /// They are ordered by their strength (the byte-for-byte equality implies the other two, etc.), + /// but we mainly care about the structured equality. If the more strong equalities are broken - + /// the assertions can be simply removed. + /// However the structured equality is important - it guarantees that binlog reading and writing functionality + /// is not dropping or altering any information. + /// + /// + /// [Theory] [InlineData(s_testProject, BinlogRoundtripTestReplayMode.Structured)] [InlineData(s_testProject, BinlogRoundtripTestReplayMode.RawEvents)] @@ -200,10 +207,6 @@ public void TestBinaryLoggerRoundtrip(string projectText, BinlogRoundtripTestRep [InlineData(s_testProject2, BinlogRoundtripTestReplayMode.RawEvents)] public void TestBinaryLoggerRoundtripEquality(string projectText, BinlogRoundtripTestReplayMode replayMode) { - // Make sure the env var will get transcribed to traits. - BuildEnvironmentState.s_runningTests = true; - _env.SetEnvironmentVariable("MSBUILDDETERMNISTICBINLOG", "1"); - var binaryLogger = new BinaryLogger(); binaryLogger.Parameters = _logFile; @@ -225,7 +228,7 @@ public void TestBinaryLoggerRoundtripEquality(string projectText, BinlogRoundtri BinaryLogger outputBinlog = new BinaryLogger() { - Parameters = $"LogFile={replayedLogFile};ProjectImports=Replay;OmitInitialInfo" + Parameters = $"LogFile={replayedLogFile};OmitInitialInfo" }; outputBinlog.Initialize(logReader); logReader.Replay(_logFile); diff --git a/src/Build/Logging/BinaryLogger/BinaryLogRecordKind.cs b/src/Build/Logging/BinaryLogger/BinaryLogRecordKind.cs index 28333110721..9fe1638fd3a 100644 --- a/src/Build/Logging/BinaryLogger/BinaryLogRecordKind.cs +++ b/src/Build/Logging/BinaryLogger/BinaryLogRecordKind.cs @@ -24,7 +24,7 @@ internal enum BinaryLogRecordKind ProjectEvaluationStarted, ProjectEvaluationFinished, ProjectImported, - ProjectImportArchive = 17, + ProjectImportArchive, TargetSkipped, PropertyReassignment, UninitializedPropertyRead, diff --git a/src/Build/Logging/BinaryLogger/BinaryLogReplayEventSource.cs b/src/Build/Logging/BinaryLogger/BinaryLogReplayEventSource.cs index 087ccebc898..afc16c43a1a 100644 --- a/src/Build/Logging/BinaryLogger/BinaryLogReplayEventSource.cs +++ b/src/Build/Logging/BinaryLogger/BinaryLogReplayEventSource.cs @@ -18,7 +18,7 @@ internal interface IRawLogEventsSource /// This means all event args and key-value pairs. /// Strings and Embedded files are not included. /// - event Action? LogDataSliceReceived; + event Action? RawLogRecordReceived; /// /// Enables initialization (e.g. subscription to events) - that is deferred until Replay is triggered. @@ -65,6 +65,13 @@ static BinaryLogReplayEventSource() /// public event Action? OnRecoverableReadError; + /// + /// WARNING: This event is under low support and low maintenance - please use events directly exposed by instead. + /// + /// Raised once is created during replaying + /// + public event Action? NotificationsSourceCreated; + /// /// Read the provided binary log file and raise corresponding events for each BuildEventArgs /// @@ -171,6 +178,7 @@ public void Replay(BinaryReader binaryReader, CancellationToken cancellationToke public void Replay(BinaryReader binaryReader, bool closeInput, CancellationToken cancellationToken) { using var reader = OpenBuildEventsReader(binaryReader, closeInput, AllowForwardCompatibility); + NotificationsSourceCreated?.Invoke(reader); Replay(reader, cancellationToken); } @@ -195,10 +203,11 @@ public void Replay(BuildEventArgsReader reader, CancellationToken cancellationTo reader.EmbeddedContentRead += _embeddedContentRead; reader.StringReadDone += _stringReadDone; + reader.StringEncountered += _stringEncountered; if (HasStructuredEventsSubscribers || !supportsForwardCompatibility) { - if (this._logDataSliceReceived != null) + if (this._rawLogRecordReceived != null) { throw new NotSupportedException( "Structured events and raw events cannot be replayed at the same time."); @@ -216,9 +225,10 @@ public void Replay(BuildEventArgsReader reader, CancellationToken cancellationTo } else { - if (this._logDataSliceReceived == null && + if (this._rawLogRecordReceived == null && this._embeddedContentRead == null && - this._stringReadDone == null) + this._stringReadDone == null && + this._stringEncountered == null) { throw new NotSupportedException( "No subscribers for any events."); @@ -227,7 +237,7 @@ public void Replay(BuildEventArgsReader reader, CancellationToken cancellationTo while (!cancellationToken.IsCancellationRequested && reader.ReadRaw() is { } instance && instance.RecordKind != BinaryLogRecordKind.EndOfFile) { - _logDataSliceReceived?.Invoke(instance.RecordKind, instance.Stream); + _rawLogRecordReceived?.Invoke(instance.RecordKind, instance.Stream); } } } @@ -261,12 +271,20 @@ event Action? IBuildEventStringsReader.StringReadDone remove => _stringReadDone -= value; } - private Action? _logDataSliceReceived; + private Action? _stringEncountered; + /// + event Action? IBuildEventStringsReader.StringEncountered + { + add => _stringEncountered += value; + remove => _stringEncountered -= value; + } + + private Action? _rawLogRecordReceived; /// - event Action? IRawLogEventsSource.LogDataSliceReceived + event Action? IRawLogEventsSource.RawLogRecordReceived { - add => _logDataSliceReceived += value; - remove => _logDataSliceReceived -= value; + add => _rawLogRecordReceived += value; + remove => _rawLogRecordReceived -= value; } } } diff --git a/src/Build/Logging/BinaryLogger/BinaryLogger.cs b/src/Build/Logging/BinaryLogger/BinaryLogger.cs index 9da915cb2d6..1df77d877e7 100644 --- a/src/Build/Logging/BinaryLogger/BinaryLogger.cs +++ b/src/Build/Logging/BinaryLogger/BinaryLogger.cs @@ -97,11 +97,6 @@ public enum ProjectImportsCollectionMode /// Create an external .ProjectImports.zip archive for the project files. /// ZipFile, - - /// - /// Don't collect any files from build events, but instead replay them from the given event source (if that one supports it). - /// - Replay, } /// @@ -140,6 +135,7 @@ public void Initialize(IEventSource eventSource) bool logPropertiesAndItemsAfterEvaluation = Traits.Instance.EscapeHatches.LogPropertiesAndItemsAfterEvaluation ?? true; ProcessParameters(out bool omitInitialInfo); + var replayEventsSource = eventSource as IBinaryLogReplaySource; try { @@ -161,7 +157,7 @@ public void Initialize(IEventSource eventSource) stream = new FileStream(FilePath, FileMode.Create); - if (CollectProjectImports != ProjectImportsCollectionMode.None && CollectProjectImports != ProjectImportsCollectionMode.Replay) + if (CollectProjectImports != ProjectImportsCollectionMode.None && replayEventsSource == null) { projectImportsCollector = new ProjectImportsCollector(FilePath, CollectProjectImports == ProjectImportsCollectionMode.ZipFile); } @@ -189,9 +185,7 @@ public void Initialize(IEventSource eventSource) // wrapping the GZipStream in a buffered stream significantly improves performance // and the max throughput is reached with a 32K buffer. See details here: // https://github.com/dotnet/runtime/issues/39233#issuecomment-745598847 - stream = Traits.Instance.DeterministicBinlogStreamBuffering ? - new GreedyBufferedStream(stream, bufferSize: 32768) : - new BufferedStream(stream, bufferSize: 32768); + stream = new BufferedStream(stream, bufferSize: 32768); binaryWriter = new BinaryWriter(stream); eventArgsWriter = new BuildEventArgsWriter(binaryWriter); @@ -202,13 +196,17 @@ public void Initialize(IEventSource eventSource) binaryWriter.Write(FileFormatVersion); - if (eventSource is IBinaryLogReplaySource replayEventsSource) + if (replayEventsSource != null) { - if (CollectProjectImports == ProjectImportsCollectionMode.Replay) + if (CollectProjectImports == ProjectImportsCollectionMode.Embed) { replayEventsSource.EmbeddedContentRead += args => eventArgsWriter.WriteBlob(args.ContentKind.ToBinaryLogRecordKind(), args.ContentStream); } + else if (CollectProjectImports != ProjectImportsCollectionMode.None) + { + throw new LoggerException($"ProjectImports={CollectProjectImports} not supported in reply mode - only Embed or None are supported."); + } // If raw events are provided - let's try to use the advantage. // But other subscribers can later on subscribe to structured events - @@ -218,7 +216,7 @@ public void Initialize(IEventSource eventSource) // at the same time as raw events are being written - this would break the deduplicated strings store. () => { - replayEventsSource.LogDataSliceReceived += RawEvents_LogDataSliceReceived; + replayEventsSource.RawLogRecordReceived += RawEvents_LogDataSliceReceived; // Replay separated strings here as well (and do not deduplicate! It would skew string indexes) replayEventsSource.StringReadDone += strArg => eventArgsWriter.WriteStringRecord(strArg.StringToBeUsed); }, @@ -377,10 +375,6 @@ private void ProcessParameters(out bool omitInitialInfo) { CollectProjectImports = ProjectImportsCollectionMode.ZipFile; } - else if (string.Equals(parameter, "ProjectImports=Replay", StringComparison.OrdinalIgnoreCase)) - { - CollectProjectImports = ProjectImportsCollectionMode.Replay; - } else if (string.Equals(parameter, "OmitInitialInfo", StringComparison.OrdinalIgnoreCase)) { omitInitialInfo = true; diff --git a/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs b/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs index 2ca7dbbbb87..5c213896b7c 100644 --- a/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs +++ b/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs @@ -105,20 +105,15 @@ public void Dispose() } } - /// - /// An event that allows the subscriber to be notified when a string is read from the binary log. - /// Subscriber may adjust the string by setting property. - /// The passed event arg can be reused and should not be stored. - /// + /// public event Action? StringReadDone; + /// + public event Action? StringEncountered; + public int FileFormatVersion => fileFormatVersion; - /// - /// Raised when the log reader encounters a project import archive (embedded content) in the stream. - /// The subscriber must read the exactly given length of binary data from the stream - otherwise exception is raised. - /// If no subscriber is attached, the data is skipped. - /// + /// internal event Action? EmbeddedContentRead; /// @@ -146,7 +141,7 @@ public void Dispose() throw new InvalidDataException($"Raw data slice for record {recordNumber} was not fully read."); } - BinaryLogRecordKind recordKind = ReadTillNextEvent(IsTextualDataRecord); + BinaryLogRecordKind recordKind = PreprocessRecordsTillNextEvent(IsTextualDataRecord); if (recordKind == BinaryLogRecordKind.EndOfFile) { @@ -180,7 +175,7 @@ public void Dispose() BuildEventArgs? result = null; while (result == null) { - BinaryLogRecordKind recordKind = ReadTillNextEvent(IsAuxiliaryRecord); + BinaryLogRecordKind recordKind = PreprocessRecordsTillNextEvent(IsAuxiliaryRecord); if (recordKind == BinaryLogRecordKind.EndOfFile) { @@ -270,15 +265,17 @@ public void Dispose() result = ReadAssemblyLoadEventArgs(); break; default: - OnRecoverableReadError?.Invoke( - $"BuildEvent record number {recordNumber} (serialized size: {serializedEventLength}) is of unsupported type: {recordKind}.{(SkipUnknownEvents ? " Skipping it." : string.Empty)}"); + string error = + $"BuildEvent record number {recordNumber} (serialized size: {serializedEventLength}) is of unsupported type: {recordKind}.{(SkipUnknownEvents ? " Skipping it." : string.Empty)}"; + if (SkipUnknownEvents && serializedEventLength > 0) { + OnRecoverableReadError?.Invoke(error); SkipBytes(serializedEventLength); } else { - return null; + throw new InvalidDataException(error); } break; } @@ -318,13 +315,11 @@ private void SkipBytes(int count) } else { - byte[] buffer = ArrayPool.Shared.Rent(count); - using var _ = new CleanupScope(() => ArrayPool.Shared.Return(buffer)); - binaryReader.BaseStream.ReadAtLeast(buffer, 0, count, throwOnEndOfStream: true); + binaryReader.BaseStream.SkipBytes(count, true); } } - private BinaryLogRecordKind ReadTillNextEvent(Func isPreprocessRecord) + private BinaryLogRecordKind PreprocessRecordsTillNextEvent(Func isPreprocessRecord) { BinaryLogRecordKind recordKind = (BinaryLogRecordKind)ReadInt32(); @@ -415,7 +410,6 @@ private void ReadEmbeddedContent(BinaryLogRecordKind recordKind) projectImportsCollector.AddFileFromMemory( resultFile.FullPath, resultFile.GetContent(), - encoding: resultFile.Encoding, makePathAbsolute: false, entryCreationStamp: entry.LastWriteTime); } @@ -1535,6 +1529,7 @@ private ITaskItem ReadTaskItem() private string ReadString() { + this.StringEncountered?.Invoke(); string text = binaryReader.ReadString(); if (this.StringReadDone != null) { diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFile.cs b/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFile.cs index 88e80cd9d59..d88b4191ef5 100644 --- a/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFile.cs +++ b/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFile.cs @@ -10,24 +10,18 @@ namespace Microsoft.Build.Logging { public sealed class ArchiveFile { - // We need to specify encoding without preamble - as then StreamReader will - // automatically adjust the encoding to match the preamble (if present). - // It will as well change to other encoding if detected. - private static readonly Encoding s_utf8WithoutPreamble = new UTF8Encoding(false); - public ArchiveFile(string fullPath, Stream contentStream) { FullPath = fullPath; - _contentReader = new StreamReader(contentStream, s_utf8WithoutPreamble); + _contentReader = new StreamReader(contentStream); } - public ArchiveFile(string fullPath, string content, Encoding? contentEncoding = null) + public ArchiveFile(string fullPath, string content) { FullPath = fullPath; _content = content; _stringAcquired = true; _contentReader = StreamReader.Null; - _stringEncoding = contentEncoding ?? Encoding.UTF8; } internal static ArchiveFile From(ZipArchiveEntry entry) @@ -36,9 +30,6 @@ internal static ArchiveFile From(ZipArchiveEntry entry) } public string FullPath { get; } - - public Encoding Encoding => _stringEncoding ?? _contentReader.CurrentEncoding; - public bool CanUseReader => !_stringAcquired; public bool CanUseString => !_streamAcquired; @@ -69,7 +60,7 @@ public string GetContent() { if (_streamAcquired) { - throw new InvalidOperationException("Content already acquired as StreamReader via GetContnetReader."); + throw new InvalidOperationException("Content already acquired as StreamReader via GetContentReader."); } if (!_stringAcquired) @@ -85,7 +76,6 @@ public string GetContent() private bool _stringAcquired; private readonly StreamReader _contentReader; private string? _content; - private readonly Encoding? _stringEncoding; // Intentionally not exposing this publicly (e.g. as IDisposable implementation) // as we don't want to user to be bothered with ownership and disposing concerns. diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFileEventArgs.cs b/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFileEventArgs.cs index 120362bcf55..fa1da7e68fc 100644 --- a/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFileEventArgs.cs +++ b/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFileEventArgs.cs @@ -43,7 +43,7 @@ public void SetResult(string resultPath, Stream resultStream) public void SetResult(string resultPath, string resultContent) { - _archiveFile = new ArchiveFile(resultPath, resultContent, _archiveFile.Encoding); + _archiveFile = new ArchiveFile(resultPath, resultContent); _disposeAction += _archiveFile.Dispose; _resultSet = true; } diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/CleanupScope.cs b/src/Build/Logging/BinaryLogger/Postprocessing/CleanupScope.cs index bef26ff4d13..3045f914fe7 100644 --- a/src/Build/Logging/BinaryLogger/Postprocessing/CleanupScope.cs +++ b/src/Build/Logging/BinaryLogger/Postprocessing/CleanupScope.cs @@ -5,7 +5,7 @@ namespace Microsoft.Build.Logging; -internal class CleanupScope : IDisposable +internal readonly struct CleanupScope : IDisposable { private readonly Action _disposeAction; diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/EmbeddedContentKind.cs b/src/Build/Logging/BinaryLogger/Postprocessing/EmbeddedContentKind.cs index 2fca5d7eaa3..592733afbff 100644 --- a/src/Build/Logging/BinaryLogger/Postprocessing/EmbeddedContentKind.cs +++ b/src/Build/Logging/BinaryLogger/Postprocessing/EmbeddedContentKind.cs @@ -1,17 +1,11 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - namespace Microsoft.Build.Logging { internal enum EmbeddedContentKind { Unknown = -1, - ProjectImportArchive = 17, + ProjectImportArchive = BinaryLogRecordKind.ProjectImportArchive, } } diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/GreedyBufferedStream.cs b/src/Build/Logging/BinaryLogger/Postprocessing/GreedyBufferedStream.cs deleted file mode 100644 index e334eac4b8f..00000000000 --- a/src/Build/Logging/BinaryLogger/Postprocessing/GreedyBufferedStream.cs +++ /dev/null @@ -1,83 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; -using System.IO; - -namespace Microsoft.Build.Logging -{ - /// - /// This is write-only, append-only stream that always buffers the wrapped stream - /// into the chunks of the same size (except the possible shorter last chunk). - /// So unlike the it never writes to the wrapped stream - /// until it has full chunk or is closing. - /// - /// This is not supposed to bring performance benefits, but it allows to avoid nondeterministic - /// GZipStream output for the identical input. - /// - internal class GreedyBufferedStream : Stream - { - private readonly Stream _stream; - private readonly byte[] _buffer; - private int _position; - - public GreedyBufferedStream(Stream stream, int bufferSize) - { - _stream = stream; - _buffer = new byte[bufferSize]; - } - - public override void Flush() - { - _stream.Write(_buffer, 0, _position); - _position = 0; - } - - public override int Read(byte[] buffer, int offset, int count) => throw UnsupportedException; - - public override long Seek(long offset, SeekOrigin origin) => throw UnsupportedException; - - public override void SetLength(long value) => throw UnsupportedException; - - public override void Write(byte[] buffer, int offset, int count) - { - // Appends input to the buffer until it is full - then flushes it to the wrapped stream. - // Repeat above until all input is processed. - - int srcOffset = offset; - do - { - int currentCount = Math.Min(count, _buffer.Length - _position); - Buffer.BlockCopy(buffer, srcOffset, _buffer, _position, currentCount); - _position += currentCount; - count -= currentCount; - srcOffset += currentCount; - - if (_position == _buffer.Length) - { - Flush(); - } - } while (count > 0); - } - - public override bool CanRead => false; - public override bool CanSeek => false; - public override bool CanWrite => _stream.CanWrite; - public override long Length => _stream.Length + _position; - - public override long Position - { - get => _stream.Position + _position; - set => throw UnsupportedException; - } - - public override void Close() - { - Flush(); - _stream.Close(); - base.Close(); - } - - private Exception UnsupportedException => new NotSupportedException("GreedyBufferedStream is write-only, append-only"); - } -} diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/IBuildEventStringsReader.cs b/src/Build/Logging/BinaryLogger/Postprocessing/IBuildEventStringsReader.cs index bf3d54f8ff8..9870ab6b587 100644 --- a/src/Build/Logging/BinaryLogger/Postprocessing/IBuildEventStringsReader.cs +++ b/src/Build/Logging/BinaryLogger/Postprocessing/IBuildEventStringsReader.cs @@ -16,5 +16,13 @@ public interface IBuildEventStringsReader /// The passed event arg can be reused and should not be stored. /// public event Action? StringReadDone; + + /// + /// WARNING: This event is under low support and low maintenance - please use instead. + /// + /// An event that allows the caller to be notified when a string is encountered in the binary log. + /// BinaryReader passed in ctor is at the beginning of the string at this point. + /// + public event Action? StringEncountered; } } diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/StreamExtensions.cs b/src/Build/Logging/BinaryLogger/Postprocessing/StreamExtensions.cs index 83492b0af2d..aa6b97e7869 100644 --- a/src/Build/Logging/BinaryLogger/Postprocessing/StreamExtensions.cs +++ b/src/Build/Logging/BinaryLogger/Postprocessing/StreamExtensions.cs @@ -2,12 +2,9 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; -using System.Collections.Generic; +using System.Buffers; using System.Diagnostics; using System.IO; -using System.Linq; -using System.Text; -using System.Threading.Tasks; namespace Microsoft.Build.Logging { @@ -38,6 +35,35 @@ public static int ReadAtLeast(this Stream stream, byte[] buffer, int offset, int return totalRead; } + public static int SkipBytes(this Stream stream, int bytesCount, bool throwOnEndOfStream) + { + byte[] buffer = ArrayPool.Shared.Rent(4096); + using var _ = new CleanupScope(() => ArrayPool.Shared.Return(buffer)); + return SkipBytes(stream, bytesCount, throwOnEndOfStream, buffer); + } + + public static int SkipBytes(this Stream stream, int bytesCount, bool throwOnEndOfStream, byte[] buffer) + { + int totalRead = 0; + while (totalRead < bytesCount) + { + int read = stream.Read(buffer, 0, Math.Min(bytesCount - totalRead, buffer.Length)); + if (read == 0) + { + if (throwOnEndOfStream) + { + throw new InvalidDataException("Unexpected end of stream."); + } + + return totalRead; + } + + totalRead += read; + } + + return totalRead; + } + public static Stream ToReadableSeekableStream(this Stream stream) { return TransparentReadStream.CreateSeekableStream(stream); diff --git a/src/Build/Logging/BinaryLogger/ProjectImportsCollector.cs b/src/Build/Logging/BinaryLogger/ProjectImportsCollector.cs index 24e86af2991..714eb3332e3 100644 --- a/src/Build/Logging/BinaryLogger/ProjectImportsCollector.cs +++ b/src/Build/Logging/BinaryLogger/ProjectImportsCollector.cs @@ -83,12 +83,11 @@ public void AddFile(string? filePath) public void AddFileFromMemory( string? filePath, string data, - Encoding? encoding = null, DateTimeOffset? entryCreationStamp = null, bool makePathAbsolute = true) { AddFileHelper(filePath, path => - AddFileFromMemoryCore(path, data, encoding ?? Encoding.UTF8, makePathAbsolute, entryCreationStamp)); + AddFileFromMemoryCore(path, data, makePathAbsolute, entryCreationStamp)); } public void AddFileFromMemory( @@ -106,8 +105,6 @@ private void AddFileHelper( { if (filePath != null && _fileStream != null) { - Action addFileAction = WrapWithExceptionSwallowing(() => addFileWorker(filePath)); - lock (_fileStream) { if (_runOnBackground) @@ -115,29 +112,25 @@ private void AddFileHelper( // enqueue the task to add a file and return quickly // to avoid holding up the current thread _currentTask = _currentTask.ContinueWith( - t => { addFileAction(); }, + t => { TryAddFile(); }, TaskScheduler.Default); } else { - addFileAction(); + TryAddFile(); } } } - } - private Action WrapWithExceptionSwallowing(Action action) - { - return () => + void TryAddFile() { try { - action(); + addFileWorker(filePath); } catch - { - } - }; + { } + } } /// @@ -160,7 +153,7 @@ private void AddFileCore(string filePath) /// This method doesn't need locking/synchronization because it's only called /// from a task that is chained linearly /// - private void AddFileFromMemoryCore(string filePath, string data, Encoding encoding, bool makePathAbsolute, DateTimeOffset? entryCreationStamp) + private void AddFileFromMemoryCore(string filePath, string data, bool makePathAbsolute, DateTimeOffset? entryCreationStamp) { // quick check to avoid repeated disk access for Exists etc. if (!ShouldAddFile(ref filePath, false, makePathAbsolute)) @@ -168,7 +161,8 @@ private void AddFileFromMemoryCore(string filePath, string data, Encoding encodi return; } - AddFileData(filePath, data, encoding, entryCreationStamp); + using var content = new MemoryStream(Encoding.UTF8.GetBytes(data)); + AddFileData(filePath, content, entryCreationStamp); } private void AddFileFromMemoryCore(string filePath, Stream data, bool makePathAbsolute, DateTimeOffset? entryCreationStamp) @@ -188,19 +182,6 @@ private void AddFileData(string filePath, Stream data, DateTimeOffset? entryCrea data.CopyTo(entryStream); } - private void AddFileData(string filePath, string data, Encoding encoding, DateTimeOffset? entryCreationStamp) - { - using Stream entryStream = OpenArchiveEntry(filePath, entryCreationStamp); - using MemoryStream memoryStream = new MemoryStream(); - // We need writer as encoding.GetBytes() isn't obliged to output preamble - // We cannot write directly to entryStream (preamble is written separately) as it's compressed differnetly, then writing the whole stream at once - using StreamWriter writer = new StreamWriter(memoryStream, encoding); - writer.Write(data); - writer.Flush(); - memoryStream.Position = 0; - memoryStream.CopyTo(entryStream); - } - private bool ShouldAddFile(ref string filePath, bool checkFileExistence, bool makeAbsolute) { // quick check to avoid repeated disk access for Exists etc. diff --git a/src/Build/Microsoft.Build.csproj b/src/Build/Microsoft.Build.csproj index 81439958264..18ab8b06ba0 100644 --- a/src/Build/Microsoft.Build.csproj +++ b/src/Build/Microsoft.Build.csproj @@ -1,4 +1,4 @@ - + diff --git a/src/Framework/Traits.cs b/src/Framework/Traits.cs index 3c81043f963..04b2fc90237 100644 --- a/src/Framework/Traits.cs +++ b/src/Framework/Traits.cs @@ -120,12 +120,6 @@ public Traits() /// public readonly int LogPropertyTracking = ParseIntFromEnvironmentVariableOrDefault("MsBuildLogPropertyTracking", 0); // Default to logging nothing via the property tracker. - /// - /// Turn on greedy buffering stream decorator for binlog writer. - /// This will ensure that 2 identical binlog contents will result into identical binlog files (as writing different chunks to GZipStream can lead to different result). - /// - public readonly bool DeterministicBinlogStreamBuffering = !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("MSBUILDDETERMNISTICBINLOG")); - /// /// When evaluating items, this is the minimum number of items on the running list to use a dictionary-based remove optimization. /// From 0d2317eb2868e1c0799116d75b428ceecddf30f7 Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Tue, 10 Oct 2023 22:27:06 +0200 Subject: [PATCH 15/49] Add support for minimum supported reader version --- .../BinaryLogReplayEventSource.cs | 18 ++++++++----- .../Logging/BinaryLogger/BinaryLogger.cs | 3 +++ .../BinaryLogger/BuildEventArgsReader.cs | 6 ++--- .../Postprocessing/IBinlogReaderErrors.cs | 27 +++++++++++++++++++ .../IBuildEventArgsReaderNotifications.cs | 5 +++- src/Build/Resources/Strings.resx | 2 +- src/Build/Resources/xlf/Strings.cs.xlf | 4 +-- src/Build/Resources/xlf/Strings.de.xlf | 4 +-- src/Build/Resources/xlf/Strings.es.xlf | 4 +-- src/Build/Resources/xlf/Strings.fr.xlf | 4 +-- src/Build/Resources/xlf/Strings.it.xlf | 4 +-- src/Build/Resources/xlf/Strings.ja.xlf | 4 +-- src/Build/Resources/xlf/Strings.ko.xlf | 4 +-- src/Build/Resources/xlf/Strings.pl.xlf | 4 +-- src/Build/Resources/xlf/Strings.pt-BR.xlf | 4 +-- src/Build/Resources/xlf/Strings.ru.xlf | 4 +-- src/Build/Resources/xlf/Strings.tr.xlf | 4 +-- src/Build/Resources/xlf/Strings.zh-Hans.xlf | 4 +-- src/Build/Resources/xlf/Strings.zh-Hant.xlf | 4 +-- 19 files changed, 75 insertions(+), 38 deletions(-) create mode 100644 src/Build/Logging/BinaryLogger/Postprocessing/IBinlogReaderErrors.cs diff --git a/src/Build/Logging/BinaryLogger/BinaryLogReplayEventSource.cs b/src/Build/Logging/BinaryLogger/BinaryLogReplayEventSource.cs index afc16c43a1a..3963e78afb4 100644 --- a/src/Build/Logging/BinaryLogger/BinaryLogReplayEventSource.cs +++ b/src/Build/Logging/BinaryLogger/BinaryLogReplayEventSource.cs @@ -36,7 +36,11 @@ void DeferredInitialize( /// Interface for replaying a binary log file (*.binlog) /// internal interface IBinaryLogReplaySource : - IEventSource, IRawLogEventsSource, IBuildEventStringsReader, IEmbeddedContentSource + IEventSource, + IRawLogEventsSource, + IBuildEventStringsReader, + IEmbeddedContentSource, + IBinlogReaderErrors { } /// @@ -60,10 +64,8 @@ static BinaryLogReplayEventSource() /// public bool AllowForwardCompatibility { private get; init; } = true; - /// - /// Receives recoverable errors during reading. - /// - public event Action? OnRecoverableReadError; + /// + public event Action? OnRecoverableReadError; /// /// WARNING: This event is under low support and low maintenance - please use events directly exposed by instead. @@ -125,12 +127,14 @@ public static BuildEventArgsReader OpenBuildEventsReader( Action? onRecoverableReadError = null) { int fileFormatVersion = binaryReader.ReadInt32(); + int minimumReaderVersion = binaryReader.ReadInt32(); // the log file is written using a newer version of file format // that we don't know how to read - if (!allowForwardCompatibility && fileFormatVersion > BinaryLogger.FileFormatVersion) + if (fileFormatVersion > BinaryLogger.FileFormatVersion && + (!allowForwardCompatibility || minimumReaderVersion > BinaryLogger.FileFormatVersion)) { - var text = ResourceUtilities.FormatResourceStringStripCodeAndKeyword("UnsupportedLogFileFormat", fileFormatVersion, BinaryLogger.FileFormatVersion); + var text = ResourceUtilities.FormatResourceStringStripCodeAndKeyword("UnsupportedLogFileFormat", fileFormatVersion, minimumReaderVersion, BinaryLogger.FileFormatVersion); throw new NotSupportedException(text); } diff --git a/src/Build/Logging/BinaryLogger/BinaryLogger.cs b/src/Build/Logging/BinaryLogger/BinaryLogger.cs index 1df77d877e7..21a51e94332 100644 --- a/src/Build/Logging/BinaryLogger/BinaryLogger.cs +++ b/src/Build/Logging/BinaryLogger/BinaryLogger.cs @@ -68,6 +68,8 @@ public sealed class BinaryLogger : ILogger // between de/serialization roundtrips. // - Adding serialized events lengths - to support forward compatible reading internal const int FileFormatVersion = 18; + // The minimum version of the binary log reader that can read log of above version. + internal const int MinimumReaderVersion = 18; private Stream stream; private BinaryWriter binaryWriter; @@ -195,6 +197,7 @@ public void Initialize(IEventSource eventSource) } binaryWriter.Write(FileFormatVersion); + binaryWriter.Write(MinimumReaderVersion); if (replayEventsSource != null) { diff --git a/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs b/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs index 5c213896b7c..bafd0971cc4 100644 --- a/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs +++ b/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs @@ -94,7 +94,7 @@ public BuildEventArgsReader(BinaryReader binaryReader, int fileFormatVersion) /// Receives recoverable errors during reading. /// Applicable mainly when or is set to true."/> /// - public event Action? OnRecoverableReadError; + public event Action? OnRecoverableReadError; public void Dispose() { @@ -270,7 +270,7 @@ public void Dispose() if (SkipUnknownEvents && serializedEventLength > 0) { - OnRecoverableReadError?.Invoke(error); + OnRecoverableReadError?.Invoke(ReaderErrorType.UnkownEventType, error); SkipBytes(serializedEventLength); } else @@ -291,7 +291,7 @@ public void Dispose() if (SkipUnknownEventParts && bytesRead < serializedEventLength) { - OnRecoverableReadError?.Invoke(error); + OnRecoverableReadError?.Invoke(ReaderErrorType.UnknownEventData, error); SkipBytes(serializedEventLength - bytesRead); } else diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/IBinlogReaderErrors.cs b/src/Build/Logging/BinaryLogger/Postprocessing/IBinlogReaderErrors.cs new file mode 100644 index 00000000000..67d457e26ff --- /dev/null +++ b/src/Build/Logging/BinaryLogger/Postprocessing/IBinlogReaderErrors.cs @@ -0,0 +1,27 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Microsoft.Build.Logging +{ + public enum ReaderErrorType + { + UnsupportedFileFormat, + UnkownEventType, + UnknownEventData, + UnknownFormatOfEventData, + } + + public interface IBinlogReaderErrors + { + /// + /// Receives recoverable errors during reading. + /// + event Action? OnRecoverableReadError; + } +} diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/IBuildEventArgsReaderNotifications.cs b/src/Build/Logging/BinaryLogger/Postprocessing/IBuildEventArgsReaderNotifications.cs index 13bc343362a..b5940c07e52 100644 --- a/src/Build/Logging/BinaryLogger/Postprocessing/IBuildEventArgsReaderNotifications.cs +++ b/src/Build/Logging/BinaryLogger/Postprocessing/IBuildEventArgsReaderNotifications.cs @@ -6,7 +6,10 @@ namespace Microsoft.Build.Logging /// /// An interface for notifications from BuildEventArgsReader /// - public interface IBuildEventArgsReaderNotifications : IBuildEventStringsReader, IBuildFileReader + public interface IBuildEventArgsReaderNotifications : + IBuildEventStringsReader, + IBuildFileReader, + IBinlogReaderErrors { /* For future use */ } diff --git a/src/Build/Resources/Strings.resx b/src/Build/Resources/Strings.resx index 3e949a272e4..6b931215fe9 100644 --- a/src/Build/Resources/Strings.resx +++ b/src/Build/Resources/Strings.resx @@ -1346,7 +1346,7 @@ {StrBegin="MSB4068: "} - MSB4235: The log file format version is {0}, whereas this version of MSBuild only supports versions up to {1}. + MSB4235: The log file format version is {0} and minimum required read version is {1}, whereas this version of MSBuild only supports versions up to {2}. {StrBegin="MSB4235: "} diff --git a/src/Build/Resources/xlf/Strings.cs.xlf b/src/Build/Resources/xlf/Strings.cs.xlf index 5fb7069d762..83cf0c28464 100644 --- a/src/Build/Resources/xlf/Strings.cs.xlf +++ b/src/Build/Resources/xlf/Strings.cs.xlf @@ -2480,8 +2480,8 @@ Využití: Průměrné využití {0}: {1:###.0} {StrBegin="MSB4238: "} - MSB4235: The log file format version is {0}, whereas this version of MSBuild only supports versions up to {1}. - MSB4235: Verze formátu souboru protokolu je {0}, ale tato verze nástroje MSBuild podporuje jenom verze do {1}. + MSB4235: The log file format version is {0} and minimum required read version is {1}, whereas this version of MSBuild only supports versions up to {2}. + MSB4235: The log file format version is {0} and minimum required read version is {1}, whereas this version of MSBuild only supports versions up to {2}. {StrBegin="MSB4235: "} diff --git a/src/Build/Resources/xlf/Strings.de.xlf b/src/Build/Resources/xlf/Strings.de.xlf index 6d1b74b7103..252bb80ca56 100644 --- a/src/Build/Resources/xlf/Strings.de.xlf +++ b/src/Build/Resources/xlf/Strings.de.xlf @@ -2480,8 +2480,8 @@ Auslastung: {0} Durchschnittliche Auslastung: {1:###.0} {StrBegin="MSB4238: "} - MSB4235: The log file format version is {0}, whereas this version of MSBuild only supports versions up to {1}. - MSB4235: Die Protokolldatei hat die Formatversion {0}, diese Version von MSBuild unterstützt jedoch nur Versionen bis zu {1}. + MSB4235: The log file format version is {0} and minimum required read version is {1}, whereas this version of MSBuild only supports versions up to {2}. + MSB4235: The log file format version is {0} and minimum required read version is {1}, whereas this version of MSBuild only supports versions up to {2}. {StrBegin="MSB4235: "} diff --git a/src/Build/Resources/xlf/Strings.es.xlf b/src/Build/Resources/xlf/Strings.es.xlf index 671bdf72929..052bcc949cf 100644 --- a/src/Build/Resources/xlf/Strings.es.xlf +++ b/src/Build/Resources/xlf/Strings.es.xlf @@ -2480,8 +2480,8 @@ Utilización: Utilización media de {0}: {1:###.0} {StrBegin="MSB4238: "} - MSB4235: The log file format version is {0}, whereas this version of MSBuild only supports versions up to {1}. - MSB4235: La versión del formato del archivo de registro es {0}, mientras que la versión de MSBuild solo admite versiones hasta {1}. + MSB4235: The log file format version is {0} and minimum required read version is {1}, whereas this version of MSBuild only supports versions up to {2}. + MSB4235: The log file format version is {0} and minimum required read version is {1}, whereas this version of MSBuild only supports versions up to {2}. {StrBegin="MSB4235: "} diff --git a/src/Build/Resources/xlf/Strings.fr.xlf b/src/Build/Resources/xlf/Strings.fr.xlf index ea57ce8c804..a6c964784fa 100644 --- a/src/Build/Resources/xlf/Strings.fr.xlf +++ b/src/Build/Resources/xlf/Strings.fr.xlf @@ -2480,8 +2480,8 @@ Utilisation : {0} Utilisation moyenne : {1:###.0} {StrBegin="MSB4238: "} - MSB4235: The log file format version is {0}, whereas this version of MSBuild only supports versions up to {1}. - MSB4235: Le format du fichier journal correspond à la version {0}, alors que cette version de MSBuild ne prend en charge que les versions allant jusqu'à {1}. + MSB4235: The log file format version is {0} and minimum required read version is {1}, whereas this version of MSBuild only supports versions up to {2}. + MSB4235: The log file format version is {0} and minimum required read version is {1}, whereas this version of MSBuild only supports versions up to {2}. {StrBegin="MSB4235: "} diff --git a/src/Build/Resources/xlf/Strings.it.xlf b/src/Build/Resources/xlf/Strings.it.xlf index 2055daa2c3d..2b0888d467a 100644 --- a/src/Build/Resources/xlf/Strings.it.xlf +++ b/src/Build/Resources/xlf/Strings.it.xlf @@ -2480,8 +2480,8 @@ Utilizzo: {0} Utilizzo medio: {1:###.0} {StrBegin="MSB4238: "} - MSB4235: The log file format version is {0}, whereas this version of MSBuild only supports versions up to {1}. - MSB4235: la versione del formato del file di log è la {0}, ma questa versione di MSBuild supporta solo le versioni fino alla {1}. + MSB4235: The log file format version is {0} and minimum required read version is {1}, whereas this version of MSBuild only supports versions up to {2}. + MSB4235: The log file format version is {0} and minimum required read version is {1}, whereas this version of MSBuild only supports versions up to {2}. {StrBegin="MSB4235: "} diff --git a/src/Build/Resources/xlf/Strings.ja.xlf b/src/Build/Resources/xlf/Strings.ja.xlf index 9cddd84c5e9..a9a3a9a11d4 100644 --- a/src/Build/Resources/xlf/Strings.ja.xlf +++ b/src/Build/Resources/xlf/Strings.ja.xlf @@ -2480,8 +2480,8 @@ Utilization: {0} Average Utilization: {1:###.0} {StrBegin="MSB4238: "} - MSB4235: The log file format version is {0}, whereas this version of MSBuild only supports versions up to {1}. - MSB4235: ログ ファイル形式のバージョンは {0} ですが、このバージョンの MSBuild がサポートしているのは {1} のバージョンまでのみです。 + MSB4235: The log file format version is {0} and minimum required read version is {1}, whereas this version of MSBuild only supports versions up to {2}. + MSB4235: The log file format version is {0} and minimum required read version is {1}, whereas this version of MSBuild only supports versions up to {2}. {StrBegin="MSB4235: "} diff --git a/src/Build/Resources/xlf/Strings.ko.xlf b/src/Build/Resources/xlf/Strings.ko.xlf index 584b7efaf00..b6b54e1b302 100644 --- a/src/Build/Resources/xlf/Strings.ko.xlf +++ b/src/Build/Resources/xlf/Strings.ko.xlf @@ -2480,8 +2480,8 @@ Utilization: {0} Average Utilization: {1:###.0} {StrBegin="MSB4238: "} - MSB4235: The log file format version is {0}, whereas this version of MSBuild only supports versions up to {1}. - MSB4235: 로그 파일 형식 버전은 {0}이지만, MSBuild의 이 버전은 {1} 버전까지만 지원합니다. + MSB4235: The log file format version is {0} and minimum required read version is {1}, whereas this version of MSBuild only supports versions up to {2}. + MSB4235: The log file format version is {0} and minimum required read version is {1}, whereas this version of MSBuild only supports versions up to {2}. {StrBegin="MSB4235: "} diff --git a/src/Build/Resources/xlf/Strings.pl.xlf b/src/Build/Resources/xlf/Strings.pl.xlf index 1878d231009..a8098fe843c 100644 --- a/src/Build/Resources/xlf/Strings.pl.xlf +++ b/src/Build/Resources/xlf/Strings.pl.xlf @@ -2480,8 +2480,8 @@ Wykorzystanie: Średnie wykorzystanie {0}: {1:###.0} {StrBegin="MSB4238: "} - MSB4235: The log file format version is {0}, whereas this version of MSBuild only supports versions up to {1}. - MSB4235: Wersja formatu pliku dziennika to {0}, podczas gdy ta wersja programu MSBuild obsługuje wyłącznie wersje do {1}. + MSB4235: The log file format version is {0} and minimum required read version is {1}, whereas this version of MSBuild only supports versions up to {2}. + MSB4235: The log file format version is {0} and minimum required read version is {1}, whereas this version of MSBuild only supports versions up to {2}. {StrBegin="MSB4235: "} diff --git a/src/Build/Resources/xlf/Strings.pt-BR.xlf b/src/Build/Resources/xlf/Strings.pt-BR.xlf index cdc4b13c11f..3fbb723baab 100644 --- a/src/Build/Resources/xlf/Strings.pt-BR.xlf +++ b/src/Build/Resources/xlf/Strings.pt-BR.xlf @@ -2480,8 +2480,8 @@ Utilização: {0} Utilização Média: {1:###.0} {StrBegin="MSB4238: "} - MSB4235: The log file format version is {0}, whereas this version of MSBuild only supports versions up to {1}. - MSB4235: a versão de formato do arquivo de log é {0}, enquanto esta versão do MSBuild dá suporte apenas para versões até {1}. + MSB4235: The log file format version is {0} and minimum required read version is {1}, whereas this version of MSBuild only supports versions up to {2}. + MSB4235: The log file format version is {0} and minimum required read version is {1}, whereas this version of MSBuild only supports versions up to {2}. {StrBegin="MSB4235: "} diff --git a/src/Build/Resources/xlf/Strings.ru.xlf b/src/Build/Resources/xlf/Strings.ru.xlf index 3e7014a5cf7..838bc5d0dc4 100644 --- a/src/Build/Resources/xlf/Strings.ru.xlf +++ b/src/Build/Resources/xlf/Strings.ru.xlf @@ -2480,8 +2480,8 @@ Utilization: {0} Average Utilization: {1:###.0} {StrBegin="MSB4238: "} - MSB4235: The log file format version is {0}, whereas this version of MSBuild only supports versions up to {1}. - MSB4235: версия формата файла журнала: {0}, тогда как эта версия MSBuild поддерживает только версии до {1}. + MSB4235: The log file format version is {0} and minimum required read version is {1}, whereas this version of MSBuild only supports versions up to {2}. + MSB4235: The log file format version is {0} and minimum required read version is {1}, whereas this version of MSBuild only supports versions up to {2}. {StrBegin="MSB4235: "} diff --git a/src/Build/Resources/xlf/Strings.tr.xlf b/src/Build/Resources/xlf/Strings.tr.xlf index 303b551c45e..4efc77d16a2 100644 --- a/src/Build/Resources/xlf/Strings.tr.xlf +++ b/src/Build/Resources/xlf/Strings.tr.xlf @@ -2480,8 +2480,8 @@ Kullanım: {0} Ortalama Kullanım: {1:###.0} {StrBegin="MSB4238: "} - MSB4235: The log file format version is {0}, whereas this version of MSBuild only supports versions up to {1}. - MSB4235: Günlük dosyasının biçim sürümü {0} ancak bu MSBuild sürümünün desteklediği en yüksek sürüm {1}. + MSB4235: The log file format version is {0} and minimum required read version is {1}, whereas this version of MSBuild only supports versions up to {2}. + MSB4235: The log file format version is {0} and minimum required read version is {1}, whereas this version of MSBuild only supports versions up to {2}. {StrBegin="MSB4235: "} diff --git a/src/Build/Resources/xlf/Strings.zh-Hans.xlf b/src/Build/Resources/xlf/Strings.zh-Hans.xlf index 3d8e217d6e6..9576a8eb423 100644 --- a/src/Build/Resources/xlf/Strings.zh-Hans.xlf +++ b/src/Build/Resources/xlf/Strings.zh-Hans.xlf @@ -2480,8 +2480,8 @@ Utilization: {0} Average Utilization: {1:###.0} {StrBegin="MSB4238: "} - MSB4235: The log file format version is {0}, whereas this version of MSBuild only supports versions up to {1}. - MSB4235: 日志文件格式版本为 {0}，但此版本的 MSBuild 仅支持 {1} 及更低版本。 + MSB4235: The log file format version is {0} and minimum required read version is {1}, whereas this version of MSBuild only supports versions up to {2}. + MSB4235: The log file format version is {0} and minimum required read version is {1}, whereas this version of MSBuild only supports versions up to {2}. {StrBegin="MSB4235: "} diff --git a/src/Build/Resources/xlf/Strings.zh-Hant.xlf b/src/Build/Resources/xlf/Strings.zh-Hant.xlf index dd551490a7a..f608e019738 100644 --- a/src/Build/Resources/xlf/Strings.zh-Hant.xlf +++ b/src/Build/Resources/xlf/Strings.zh-Hant.xlf @@ -2480,8 +2480,8 @@ Utilization: {0} Average Utilization: {1:###.0} {StrBegin="MSB4238: "} - MSB4235: The log file format version is {0}, whereas this version of MSBuild only supports versions up to {1}. - MSB4235: 記錄檔格式版本為 {0}，但此版 MSBuild 僅支援 {1} (含) 之前的版本。 + MSB4235: The log file format version is {0} and minimum required read version is {1}, whereas this version of MSBuild only supports versions up to {2}. + MSB4235: The log file format version is {0} and minimum required read version is {1}, whereas this version of MSBuild only supports versions up to {2}. {StrBegin="MSB4235: "} From 651ab345ed0c7c64aef6bb4ab154decc68a0d3c3 Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Wed, 11 Oct 2023 13:48:24 +0200 Subject: [PATCH 16/49] Support ProjectImports=ZipFile in Replay mode --- .../Logging/BinaryLogger/BinaryLogger.cs | 5 +++-- .../BinaryLogger/ProjectImportsCollector.cs | 21 ++++++++++++++++--- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/src/Build/Logging/BinaryLogger/BinaryLogger.cs b/src/Build/Logging/BinaryLogger/BinaryLogger.cs index 21a51e94332..142bd50b8ed 100644 --- a/src/Build/Logging/BinaryLogger/BinaryLogger.cs +++ b/src/Build/Logging/BinaryLogger/BinaryLogger.cs @@ -206,9 +206,10 @@ public void Initialize(IEventSource eventSource) replayEventsSource.EmbeddedContentRead += args => eventArgsWriter.WriteBlob(args.ContentKind.ToBinaryLogRecordKind(), args.ContentStream); } - else if (CollectProjectImports != ProjectImportsCollectionMode.None) + else if (CollectProjectImports == ProjectImportsCollectionMode.ZipFile) { - throw new LoggerException($"ProjectImports={CollectProjectImports} not supported in reply mode - only Embed or None are supported."); + replayEventsSource.EmbeddedContentRead += args => + ProjectImportsCollector.FlushBlobToFile(FilePath, args.ContentStream); } // If raw events are provided - let's try to use the advantage. diff --git a/src/Build/Logging/BinaryLogger/ProjectImportsCollector.cs b/src/Build/Logging/BinaryLogger/ProjectImportsCollector.cs index 714eb3332e3..a1f37876ca9 100644 --- a/src/Build/Logging/BinaryLogger/ProjectImportsCollector.cs +++ b/src/Build/Logging/BinaryLogger/ProjectImportsCollector.cs @@ -24,6 +24,7 @@ internal class ProjectImportsCollector private ZipArchive? _zipArchive; private readonly string _archiveFilePath; private readonly bool _runOnBackground; + private const string DefaultSourcesArchiveExtension = ".ProjectImports.zip"; /// /// Avoid visiting each file more than once. @@ -33,16 +34,30 @@ internal class ProjectImportsCollector // this will form a chain of file write tasks, running sequentially on a background thread private Task _currentTask = Task.CompletedTask; + internal static void FlushBlobToFile( + string logFilePath, + Stream contentStream) + { + string archiveFilePath = GetArchiveFilePath(logFilePath, DefaultSourcesArchiveExtension); + + using var fileStream = new FileStream(archiveFilePath, FileMode.Create, FileAccess.ReadWrite, FileShare.Delete); + contentStream.CopyTo(fileStream); + } + + // Archive file will be stored alongside the binlog + private static string GetArchiveFilePath(string logFilePath, string sourcesArchiveExtension) + => Path.ChangeExtension(logFilePath, sourcesArchiveExtension); + + public ProjectImportsCollector( string logFilePath, bool createFile, - string sourcesArchiveExtension = ".ProjectImports.zip", + string sourcesArchiveExtension = DefaultSourcesArchiveExtension, bool runOnBackground = true) { if (createFile) { - // Archive file will be stored alongside the binlog - _archiveFilePath = Path.ChangeExtension(logFilePath, sourcesArchiveExtension); + _archiveFilePath = GetArchiveFilePath(logFilePath, sourcesArchiveExtension); } else { From 37f74fb3b92db8c37eef453cd4a5630e47f46e7b Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Wed, 11 Oct 2023 17:23:56 +0200 Subject: [PATCH 17/49] Add more granular error information about recoverable errors --- .../BinaryLogger/BuildEventArgsReader.cs | 251 ++++++++++-------- .../StreamChunkOverreadException.cs | 22 ++ .../Postprocessing/StreamExtensions.cs | 2 +- .../Postprocessing/TransparentReadStream.cs | 33 ++- 4 files changed, 191 insertions(+), 117 deletions(-) create mode 100644 src/Build/Logging/BinaryLogger/Postprocessing/StreamChunkOverreadException.cs diff --git a/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs b/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs index bafd0971cc4..43924ebf9cf 100644 --- a/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs +++ b/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs @@ -24,8 +24,12 @@ namespace Microsoft.Build.Logging public class BuildEventArgsReader : IBuildEventArgsReaderNotifications, IDisposable { private readonly BinaryReader binaryReader; + // This is used to verify that events deserialization is not overreading expected size. + private readonly TransparentReadStream _readStream; private readonly int fileFormatVersion; private long recordNumber = 0; + private bool _skipUnknownEvents; + private bool _skipUnknownEventParts; /// /// A list of string records we've encountered so far. If it's a small string, it will be the string directly. @@ -69,6 +73,7 @@ public BuildEventArgsReader(BinaryReader binaryReader, int fileFormatVersion) { this.binaryReader = binaryReader; this._canSeek = binaryReader.BaseStream.CanSeek; + this._readStream = TransparentReadStream.EnsureTransparentReadStream(binaryReader.BaseStream); this.fileFormatVersion = fileFormatVersion; } @@ -82,13 +87,43 @@ public BuildEventArgsReader(BinaryReader binaryReader, int fileFormatVersion) /// Indicates whether unknown BuildEvents should be silently skipped. Read returns null otherwise. /// Parameter is supported only if the file format supports forward compatible reading (version is 18 or higher). /// - public bool SkipUnknownEvents { private get; set; } = false; + public bool SkipUnknownEvents + { + set + { + if (value) + { + EnsureForwardCompatibleReadingSupported(); + } + + _skipUnknownEvents = value; + } + } /// /// Indicates whether unread parts of BuildEvents (probably added in newer format of particular BuildEvent)should be silently skipped. Exception thrown otherwise. /// Parameter is supported only if the file format supports forward compatible reading (version is 18 or higher). /// - public bool SkipUnknownEventParts { private get; set; } = false; + public bool SkipUnknownEventParts + { + set + { + if (value) + { + EnsureForwardCompatibleReadingSupported(); + } + _skipUnknownEventParts = value; + } + } + + private void EnsureForwardCompatibleReadingSupported() + { + if (fileFormatVersion < 18) + { + throw new InvalidOperationException( + $"Forward compatible reading is not supported for file format version {fileFormatVersion} (needs >=18)."); + } + } /// /// Receives recoverable errors during reading. @@ -165,13 +200,6 @@ public void Dispose() /// public BuildEventArgs? Read() { - // todo - flip this into a properties - not to check on each call and to avoid params passing - if ((SkipUnknownEvents || SkipUnknownEventParts) && fileFormatVersion < 18) - { - throw new InvalidOperationException( - $"Forward compatible reading is not supported for file format version {fileFormatVersion} (needs >=18)."); - } - BuildEventArgs? result = null; while (result == null) { @@ -183,128 +211,118 @@ public void Dispose() } int serializedEventLength = 0; - long preEventPosition = 0; if (fileFormatVersion >= 18) { serializedEventLength = ReadInt32(); // record length - preEventPosition = _canSeek ? binaryReader.BaseStream.Position : 0; + _readStream.BytesCountAllowedToRead = serializedEventLength; } - switch (recordKind) + bool hasError = false; + try { - case BinaryLogRecordKind.BuildStarted: - result = ReadBuildStartedEventArgs(); - break; - case BinaryLogRecordKind.BuildFinished: - result = ReadBuildFinishedEventArgs(); - break; - case BinaryLogRecordKind.ProjectStarted: - result = ReadProjectStartedEventArgs(); - break; - case BinaryLogRecordKind.ProjectFinished: - result = ReadProjectFinishedEventArgs(); - break; - case BinaryLogRecordKind.TargetStarted: - result = ReadTargetStartedEventArgs(); - break; - case BinaryLogRecordKind.TargetFinished: - result = ReadTargetFinishedEventArgs(); - break; - case BinaryLogRecordKind.TaskStarted: - result = ReadTaskStartedEventArgs(); - break; - case BinaryLogRecordKind.TaskFinished: - result = ReadTaskFinishedEventArgs(); - break; - case BinaryLogRecordKind.Error: - result = ReadBuildErrorEventArgs(); - break; - case BinaryLogRecordKind.Warning: - result = ReadBuildWarningEventArgs(); - break; - case BinaryLogRecordKind.Message: - result = ReadBuildMessageEventArgs(); - break; - case BinaryLogRecordKind.CriticalBuildMessage: - result = ReadCriticalBuildMessageEventArgs(); - break; - case BinaryLogRecordKind.TaskCommandLine: - result = ReadTaskCommandLineEventArgs(); - break; - case BinaryLogRecordKind.TaskParameter: - result = ReadTaskParameterEventArgs(); - break; - case BinaryLogRecordKind.ProjectEvaluationStarted: - result = ReadProjectEvaluationStartedEventArgs(); - break; - case BinaryLogRecordKind.ProjectEvaluationFinished: - result = ReadProjectEvaluationFinishedEventArgs(); - break; - case BinaryLogRecordKind.ProjectImported: - result = ReadProjectImportedEventArgs(); - break; - case BinaryLogRecordKind.TargetSkipped: - result = ReadTargetSkippedEventArgs(); - break; - case BinaryLogRecordKind.EnvironmentVariableRead: - result = ReadEnvironmentVariableReadEventArgs(); - break; - case BinaryLogRecordKind.ResponseFileUsed: - result = ReadResponseFileUsedEventArgs(); - break; - case BinaryLogRecordKind.PropertyReassignment: - result = ReadPropertyReassignmentEventArgs(); - break; - case BinaryLogRecordKind.UninitializedPropertyRead: - result = ReadUninitializedPropertyReadEventArgs(); - break; - case BinaryLogRecordKind.PropertyInitialValueSet: - result = ReadPropertyInitialValueSetEventArgs(); - break; - case BinaryLogRecordKind.AssemblyLoad: - result = ReadAssemblyLoadEventArgs(); - break; - default: - string error = - $"BuildEvent record number {recordNumber} (serialized size: {serializedEventLength}) is of unsupported type: {recordKind}.{(SkipUnknownEvents ? " Skipping it." : string.Empty)}"; + result = ReadBuildEventArgs(recordKind); + } + catch (Exception e) when (e is InvalidDataException dataException || + e is FormatException formatException || + e is StreamChunkOverReadException overReadException) + { + hasError = true; + string error = + $"BuildEvent record number {recordNumber} (serialized size: {serializedEventLength}) attempted to perform disallowed reads (error: {e.Message}).{(_skipUnknownEventParts ? " Skipping it." : string.Empty)}"; - if (SkipUnknownEvents && serializedEventLength > 0) - { - OnRecoverableReadError?.Invoke(ReaderErrorType.UnkownEventType, error); - SkipBytes(serializedEventLength); - } - else - { - throw new InvalidDataException(error); - } - break; + HandleError(error, _skipUnknownEventParts, ReaderErrorType.UnknownFormatOfEventData, e); } - if (fileFormatVersion >= 18) + if (result == null && !hasError) { - long postEventPosition = _canSeek ? binaryReader.BaseStream.Position : serializedEventLength; - int bytesRead = (int)(postEventPosition - preEventPosition); - if (bytesRead != serializedEventLength) - { - string error = - $"BuildEvent record number {recordNumber} was expected to read exactly {serializedEventLength} bytes from the stream, but read {bytesRead} instead."; + string error = + $"BuildEvent record number {recordNumber} (serialized size: {serializedEventLength}) is of unsupported type: {recordKind}.{(_skipUnknownEvents ? " Skipping it." : string.Empty)}"; - if (SkipUnknownEventParts && bytesRead < serializedEventLength) - { - OnRecoverableReadError?.Invoke(ReaderErrorType.UnknownEventData, error); - SkipBytes(serializedEventLength - bytesRead); - } - else - { - throw new InvalidDataException(error); - } - } + HandleError(error, _skipUnknownEvents, ReaderErrorType.UnkownEventType); + } + + if (_readStream.BytesCountAllowedToReadRemaining > 0) + { + string error = + $"BuildEvent record number {recordNumber} was expected to read exactly {serializedEventLength} bytes from the stream, but read {serializedEventLength - _readStream.BytesCountAllowedToReadRemaining} instead."; + + HandleError(error, _skipUnknownEventParts, ReaderErrorType.UnknownEventData); } recordNumber += 1; } return result; + + void HandleError(string msg, bool noThrow, ReaderErrorType readerErrorType, Exception? innerException = null) + { + if (noThrow) + { + OnRecoverableReadError?.Invoke(readerErrorType, msg); + SkipBytes(_readStream.BytesCountAllowedToReadRemaining); + } + else + { + throw new InvalidDataException(msg, innerException); + } + } + } + + private BuildEventArgs? ReadBuildEventArgs(BinaryLogRecordKind recordKind) + { + switch (recordKind) + { + case BinaryLogRecordKind.BuildStarted: + return ReadBuildStartedEventArgs(); + case BinaryLogRecordKind.BuildFinished: + return ReadBuildFinishedEventArgs(); + case BinaryLogRecordKind.ProjectStarted: + return ReadProjectStartedEventArgs(); + case BinaryLogRecordKind.ProjectFinished: + return ReadProjectFinishedEventArgs(); + case BinaryLogRecordKind.TargetStarted: + return ReadTargetStartedEventArgs(); + case BinaryLogRecordKind.TargetFinished: + return ReadTargetFinishedEventArgs(); + case BinaryLogRecordKind.TaskStarted: + return ReadTaskStartedEventArgs(); + case BinaryLogRecordKind.TaskFinished: + return ReadTaskFinishedEventArgs(); + case BinaryLogRecordKind.Error: + return ReadBuildErrorEventArgs(); + case BinaryLogRecordKind.Warning: + return ReadBuildWarningEventArgs(); + case BinaryLogRecordKind.Message: + return ReadBuildMessageEventArgs(); + case BinaryLogRecordKind.CriticalBuildMessage: + return ReadCriticalBuildMessageEventArgs(); + case BinaryLogRecordKind.TaskCommandLine: + return ReadTaskCommandLineEventArgs(); + case BinaryLogRecordKind.TaskParameter: + return ReadTaskParameterEventArgs(); + case BinaryLogRecordKind.ProjectEvaluationStarted: + return ReadProjectEvaluationStartedEventArgs(); + case BinaryLogRecordKind.ProjectEvaluationFinished: + return ReadProjectEvaluationFinishedEventArgs(); + case BinaryLogRecordKind.ProjectImported: + return ReadProjectImportedEventArgs(); + case BinaryLogRecordKind.TargetSkipped: + return ReadTargetSkippedEventArgs(); + case BinaryLogRecordKind.EnvironmentVariableRead: + return ReadEnvironmentVariableReadEventArgs(); + case BinaryLogRecordKind.ResponseFileUsed: + return ReadResponseFileUsedEventArgs(); + case BinaryLogRecordKind.PropertyReassignment: + return ReadPropertyReassignmentEventArgs(); + case BinaryLogRecordKind.UninitializedPropertyRead: + return ReadUninitializedPropertyReadEventArgs(); + case BinaryLogRecordKind.PropertyInitialValueSet: + return ReadPropertyInitialValueSetEventArgs(); + case BinaryLogRecordKind.AssemblyLoad: + return ReadAssemblyLoadEventArgs(); + default: + return null; + } } private void SkipBytes(int count) @@ -321,6 +339,8 @@ private void SkipBytes(int count) private BinaryLogRecordKind PreprocessRecordsTillNextEvent(Func isPreprocessRecord) { + _readStream.BytesCountAllowedToRead = null; + BinaryLogRecordKind recordKind = (BinaryLogRecordKind)ReadInt32(); // Skip over data storage records since they don't result in a BuildEventArgs. @@ -335,6 +355,7 @@ private BinaryLogRecordKind PreprocessRecordsTillNextEvent(Func= 18) { - _ = ReadInt32(); // buffer size, not used in structured reading + _readStream.BytesCountAllowedToRead = ReadInt32(); } int count = ReadInt32(); diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/StreamChunkOverreadException.cs b/src/Build/Logging/BinaryLogger/Postprocessing/StreamChunkOverreadException.cs new file mode 100644 index 00000000000..265a888062a --- /dev/null +++ b/src/Build/Logging/BinaryLogger/Postprocessing/StreamChunkOverreadException.cs @@ -0,0 +1,22 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; + +namespace Microsoft.Build.Logging +{ + public class StreamChunkOverReadException : Exception + { + public StreamChunkOverReadException() + { + } + + public StreamChunkOverReadException(string message) : base(message) + { + } + + public StreamChunkOverReadException(string message, Exception inner) : base(message, inner) + { + } + } +} diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/StreamExtensions.cs b/src/Build/Logging/BinaryLogger/Postprocessing/StreamExtensions.cs index aa6b97e7869..2083e9e8665 100644 --- a/src/Build/Logging/BinaryLogger/Postprocessing/StreamExtensions.cs +++ b/src/Build/Logging/BinaryLogger/Postprocessing/StreamExtensions.cs @@ -66,7 +66,7 @@ public static int SkipBytes(this Stream stream, int bytesCount, bool throwOnEndO public static Stream ToReadableSeekableStream(this Stream stream) { - return TransparentReadStream.CreateSeekableStream(stream); + return TransparentReadStream.EnsureSeekableStream(stream); } /// diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/TransparentReadStream.cs b/src/Build/Logging/BinaryLogger/Postprocessing/TransparentReadStream.cs index 5061a7c3b00..ec327528d13 100644 --- a/src/Build/Logging/BinaryLogger/Postprocessing/TransparentReadStream.cs +++ b/src/Build/Logging/BinaryLogger/Postprocessing/TransparentReadStream.cs @@ -20,7 +20,7 @@ internal class TransparentReadStream : Stream private readonly Stream _stream; private long _position; - public static Stream CreateSeekableStream(Stream stream) + public static Stream EnsureSeekableStream(Stream stream) { if (stream.CanSeek) { @@ -35,11 +35,36 @@ public static Stream CreateSeekableStream(Stream stream) return new TransparentReadStream(stream); } + public static TransparentReadStream EnsureTransparentReadStream(Stream stream) + { + if (stream is TransparentReadStream transparentReadStream) + { + return transparentReadStream; + } + + if (!stream.CanRead) + { + throw new InvalidOperationException("Stream must be readable."); + } + + return new TransparentReadStream(stream); + } + private TransparentReadStream(Stream stream) { _stream = stream; } + public int? BytesCountAllowedToRead + { + set { _maxAllowedPosition = value.HasValue ? _position + value.Value : long.MaxValue; } + } + + // if we haven't constrained the allowed read size - do not report it being unfinished either. + public int BytesCountAllowedToReadRemaining => + _maxAllowedPosition == long.MaxValue ? 0 : (int)(_maxAllowedPosition - _position); + + private long _maxAllowedPosition = long.MaxValue; public override bool CanRead => _stream.CanRead; public override bool CanSeek => true; public override bool CanWrite => false; @@ -57,6 +82,12 @@ public override void Flush() public override int Read(byte[] buffer, int offset, int count) { + if (_position + count > _maxAllowedPosition) + { + throw new StreamChunkOverReadException( + $"Attempt to read {count} bytes, when only {_maxAllowedPosition - _position} are allowed to be read."); + } + int cnt = _stream.Read(buffer, offset, count); _position += cnt; return cnt; From 07411676b2c1c486a76cab5b571a2a68fbd0afa4 Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Wed, 11 Oct 2023 21:22:43 +0200 Subject: [PATCH 18/49] Fix tests --- .../BinaryLogger/BuildEventArgsReader.cs | 20 ++++---------- .../Postprocessing/StreamExtensions.cs | 8 +++--- .../Postprocessing/TransparentReadStream.cs | 27 ++----------------- 3 files changed, 11 insertions(+), 44 deletions(-) diff --git a/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs b/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs index 43924ebf9cf..b45cf895ce5 100644 --- a/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs +++ b/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs @@ -52,11 +52,6 @@ public class BuildEventArgsReader : IBuildEventArgsReaderNotifications, IDisposa /// private readonly StringStorage stringStorage = new StringStorage(); - /// - /// Is the enderlying stream seekable? - /// - private readonly bool _canSeek; - // reflection is needed to set these three fields because public constructors don't provide // a way to set these from the outside private static FieldInfo? buildEventArgsFieldThreadId = @@ -71,9 +66,11 @@ public class BuildEventArgsReader : IBuildEventArgsReaderNotifications, IDisposa /// The file format version of the log file being read. public BuildEventArgsReader(BinaryReader binaryReader, int fileFormatVersion) { - this.binaryReader = binaryReader; - this._canSeek = binaryReader.BaseStream.CanSeek; this._readStream = TransparentReadStream.EnsureTransparentReadStream(binaryReader.BaseStream); + // make sure the reader we're going to use wraps the transparent stream wrapper + this.binaryReader = binaryReader.BaseStream == _readStream + ? binaryReader + : new BinaryReader(_readStream); this.fileFormatVersion = fileFormatVersion; } @@ -327,14 +324,7 @@ void HandleError(string msg, bool noThrow, ReaderErrorType readerErrorType, Exce private void SkipBytes(int count) { - if (_canSeek) - { - binaryReader.BaseStream.Seek(count, SeekOrigin.Current); - } - else - { - binaryReader.BaseStream.SkipBytes(count, true); - } + binaryReader.BaseStream.Seek(count, SeekOrigin.Current); } private BinaryLogRecordKind PreprocessRecordsTillNextEvent(Func isPreprocessRecord) diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/StreamExtensions.cs b/src/Build/Logging/BinaryLogger/Postprocessing/StreamExtensions.cs index 2083e9e8665..b2104f2dc03 100644 --- a/src/Build/Logging/BinaryLogger/Postprocessing/StreamExtensions.cs +++ b/src/Build/Logging/BinaryLogger/Postprocessing/StreamExtensions.cs @@ -35,19 +35,19 @@ public static int ReadAtLeast(this Stream stream, byte[] buffer, int offset, int return totalRead; } - public static int SkipBytes(this Stream stream, int bytesCount, bool throwOnEndOfStream) + public static long SkipBytes(this Stream stream, long bytesCount, bool throwOnEndOfStream) { byte[] buffer = ArrayPool.Shared.Rent(4096); using var _ = new CleanupScope(() => ArrayPool.Shared.Return(buffer)); return SkipBytes(stream, bytesCount, throwOnEndOfStream, buffer); } - public static int SkipBytes(this Stream stream, int bytesCount, bool throwOnEndOfStream, byte[] buffer) + public static long SkipBytes(this Stream stream, long bytesCount, bool throwOnEndOfStream, byte[] buffer) { - int totalRead = 0; + long totalRead = 0; while (totalRead < bytesCount) { - int read = stream.Read(buffer, 0, Math.Min(bytesCount - totalRead, buffer.Length)); + int read = stream.Read(buffer, 0, (int)Math.Min(bytesCount - totalRead, buffer.Length)); if (read == 0) { if (throwOnEndOfStream) diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/TransparentReadStream.cs b/src/Build/Logging/BinaryLogger/Postprocessing/TransparentReadStream.cs index ec327528d13..931a2294197 100644 --- a/src/Build/Logging/BinaryLogger/Postprocessing/TransparentReadStream.cs +++ b/src/Build/Logging/BinaryLogger/Postprocessing/TransparentReadStream.cs @@ -72,7 +72,7 @@ public int? BytesCountAllowedToRead public override long Position { get => _position; - set => SkipBytes(value - _position); + set => this.SkipBytes((int)(value - _position), true); } public override void Flush() @@ -100,7 +100,7 @@ public override long Seek(long offset, SeekOrigin origin) throw new InvalidOperationException("Only seeking from SeekOrigin.Current is supported."); } - SkipBytes(offset); + this.SkipBytes((int)offset, true); return _position; } @@ -115,29 +115,6 @@ public override void Write(byte[] buffer, int offset, int count) throw new InvalidOperationException("Writing is not supported."); } - private void SkipBytes(long count) - { - if(count < 0) - { - throw new InvalidOperationException("Seeking backwards is not supported."); - } - - if(count == 0) - { - return; - } - - byte[] buffer = ArrayPool.Shared.Rent((int)count); - try - { - _position += _stream.ReadAtLeast(buffer, 0, (int)count, throwOnEndOfStream: true); - } - finally - { - ArrayPool.Shared.Return(buffer); - } - } - public override void Close() => _stream.Close(); } } From d328c4269d8598d8f828c02674b5983c52a01f8b Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Wed, 11 Oct 2023 22:11:39 +0200 Subject: [PATCH 19/49] Localizing strings --- .../BinaryLogReplayEventSource.cs | 4 +- .../BinaryLogger/BuildEventArgsReader.cs | 18 +++- .../Postprocessing/ArchiveFile.cs | 5 +- .../Postprocessing/ArchiveFileEventArgs.cs | 3 +- .../BinaryLogger/Postprocessing/SubStream.cs | 3 +- .../Postprocessing/TransparentReadStream.cs | 14 +-- .../BinaryLogger/ProjectImportsCollector.cs | 2 +- src/Build/Resources/Strings.resx | 69 +++++++++++++ src/Build/Resources/xlf/Strings.cs.xlf | 97 +++++++++++++++++++ src/Build/Resources/xlf/Strings.de.xlf | 97 +++++++++++++++++++ src/Build/Resources/xlf/Strings.es.xlf | 97 +++++++++++++++++++ src/Build/Resources/xlf/Strings.fr.xlf | 97 +++++++++++++++++++ src/Build/Resources/xlf/Strings.it.xlf | 97 +++++++++++++++++++ src/Build/Resources/xlf/Strings.ja.xlf | 97 +++++++++++++++++++ src/Build/Resources/xlf/Strings.ko.xlf | 97 +++++++++++++++++++ src/Build/Resources/xlf/Strings.pl.xlf | 97 +++++++++++++++++++ src/Build/Resources/xlf/Strings.pt-BR.xlf | 97 +++++++++++++++++++ src/Build/Resources/xlf/Strings.ru.xlf | 97 +++++++++++++++++++ src/Build/Resources/xlf/Strings.tr.xlf | 97 +++++++++++++++++++ src/Build/Resources/xlf/Strings.zh-Hans.xlf | 97 +++++++++++++++++++ src/Build/Resources/xlf/Strings.zh-Hant.xlf | 97 +++++++++++++++++++ 21 files changed, 1361 insertions(+), 18 deletions(-) diff --git a/src/Build/Logging/BinaryLogger/BinaryLogReplayEventSource.cs b/src/Build/Logging/BinaryLogger/BinaryLogReplayEventSource.cs index 3963e78afb4..71102153d5e 100644 --- a/src/Build/Logging/BinaryLogger/BinaryLogReplayEventSource.cs +++ b/src/Build/Logging/BinaryLogger/BinaryLogReplayEventSource.cs @@ -214,7 +214,7 @@ public void Replay(BuildEventArgsReader reader, CancellationToken cancellationTo if (this._rawLogRecordReceived != null) { throw new NotSupportedException( - "Structured events and raw events cannot be replayed at the same time."); + ResourceUtilities.GetResourceString("Binlog_Source_MultiSubscribeError")); } // Forward compatibile reading makes sense only for structured events reading. @@ -235,7 +235,7 @@ public void Replay(BuildEventArgsReader reader, CancellationToken cancellationTo this._stringEncountered == null) { throw new NotSupportedException( - "No subscribers for any events."); + ResourceUtilities.GetResourceString("Binlog_Source_MissingSubscribeError")); } while (!cancellationToken.IsCancellationRequested && reader.ReadRaw() is { } instance && diff --git a/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs b/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs index b45cf895ce5..d07b11f5e48 100644 --- a/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs +++ b/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs @@ -118,7 +118,8 @@ private void EnsureForwardCompatibleReadingSupported() if (fileFormatVersion < 18) { throw new InvalidOperationException( - $"Forward compatible reading is not supported for file format version {fileFormatVersion} (needs >=18)."); + ResourceUtilities.FormatResourceStringStripCodeAndKeyword("Binlog_FwdCompatUnsupported", + fileFormatVersion)); } } @@ -170,7 +171,7 @@ public void Dispose() if (!(_lastSubStream?.IsAtEnd ?? true)) { - throw new InvalidDataException($"Raw data slice for record {recordNumber} was not fully read."); + throw new InvalidDataException(ResourceUtilities.FormatResourceStringStripCodeAndKeyword("Binlog_RawDataUnread", recordNumber)); } BinaryLogRecordKind recordKind = PreprocessRecordsTillNextEvent(IsTextualDataRecord); @@ -225,7 +226,10 @@ e is FormatException formatException || { hasError = true; string error = - $"BuildEvent record number {recordNumber} (serialized size: {serializedEventLength}) attempted to perform disallowed reads (error: {e.Message}).{(_skipUnknownEventParts ? " Skipping it." : string.Empty)}"; + ResourceUtilities.FormatResourceStringStripCodeAndKeyword("Binlog_ReaderMismatchedRead", + recordNumber, serializedEventLength, e.Message) + (_skipUnknownEventParts + ? " " + ResourceUtilities.GetResourceString("Binlog_ReaderSkippingRecord") + : string.Empty); HandleError(error, _skipUnknownEventParts, ReaderErrorType.UnknownFormatOfEventData, e); } @@ -233,7 +237,10 @@ e is FormatException formatException || if (result == null && !hasError) { string error = - $"BuildEvent record number {recordNumber} (serialized size: {serializedEventLength}) is of unsupported type: {recordKind}.{(_skipUnknownEvents ? " Skipping it." : string.Empty)}"; + ResourceUtilities.FormatResourceStringStripCodeAndKeyword("Binlog_ReaderUnknownType", + recordNumber, serializedEventLength, recordKind) + (_skipUnknownEvents + ? " " + ResourceUtilities.GetResourceString("Binlog_ReaderSkippingRecord") + : string.Empty); HandleError(error, _skipUnknownEvents, ReaderErrorType.UnkownEventType); } @@ -241,7 +248,8 @@ e is FormatException formatException || if (_readStream.BytesCountAllowedToReadRemaining > 0) { string error = - $"BuildEvent record number {recordNumber} was expected to read exactly {serializedEventLength} bytes from the stream, but read {serializedEventLength - _readStream.BytesCountAllowedToReadRemaining} instead."; + ResourceUtilities.FormatResourceStringStripCodeAndKeyword("Binlog_ReaderUnderRead", + recordNumber, serializedEventLength, serializedEventLength - _readStream.BytesCountAllowedToReadRemaining); HandleError(error, _skipUnknownEventParts, ReaderErrorType.UnknownEventData); } diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFile.cs b/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFile.cs index d88b4191ef5..067857c9355 100644 --- a/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFile.cs +++ b/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFile.cs @@ -5,6 +5,7 @@ using System.IO; using System.IO.Compression; using System.Text; +using Microsoft.Build.Shared; namespace Microsoft.Build.Logging { @@ -43,7 +44,7 @@ public StreamReader GetContentReader() { if (_stringAcquired) { - throw new InvalidOperationException("Content already acquired as string via GetContent or initialized as string only."); + throw new InvalidOperationException(ResourceUtilities.GetResourceString("Binlog_ArchiveFile_AcquiredAsString")); } _streamAcquired = true; @@ -60,7 +61,7 @@ public string GetContent() { if (_streamAcquired) { - throw new InvalidOperationException("Content already acquired as StreamReader via GetContentReader."); + throw new InvalidOperationException(ResourceUtilities.GetResourceString("Binlog_ArchiveFile_AcquiredAsStream")); } if (!_stringAcquired) diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFileEventArgs.cs b/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFileEventArgs.cs index fa1da7e68fc..684a4e92384 100644 --- a/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFileEventArgs.cs +++ b/src/Build/Logging/BinaryLogger/Postprocessing/ArchiveFileEventArgs.cs @@ -3,6 +3,7 @@ using System; using System.IO; +using Microsoft.Build.Shared; namespace Microsoft.Build.Logging; @@ -27,7 +28,7 @@ public ArchiveFile ObtainArchiveFile() if (!_resultSet) { throw new InvalidOperationException( - "ArchiveFile was obtained, but the final edited version was not set."); + ResourceUtilities.GetResourceString("Binlog_ArchiveFile_NotSetAfterAcquire")); } _resultSet = false; diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/SubStream.cs b/src/Build/Logging/BinaryLogger/Postprocessing/SubStream.cs index b35e4e7e97b..3f81adb3558 100644 --- a/src/Build/Logging/BinaryLogger/Postprocessing/SubStream.cs +++ b/src/Build/Logging/BinaryLogger/Postprocessing/SubStream.cs @@ -7,6 +7,7 @@ using System.Linq; using System.Text; using System.Threading.Tasks; +using Microsoft.Build.Shared; namespace Microsoft.Build.Logging { @@ -27,7 +28,7 @@ public SubStream(Stream stream, long length) if (!stream.CanRead) { - throw new InvalidOperationException("Stream must be readable."); + throw new InvalidOperationException(ResourceUtilities.GetResourceString("Binlog_StreamUtils_MustBeReadable")); } } diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/TransparentReadStream.cs b/src/Build/Logging/BinaryLogger/Postprocessing/TransparentReadStream.cs index 931a2294197..f8818f9353f 100644 --- a/src/Build/Logging/BinaryLogger/Postprocessing/TransparentReadStream.cs +++ b/src/Build/Logging/BinaryLogger/Postprocessing/TransparentReadStream.cs @@ -9,6 +9,7 @@ using System.Runtime.InteropServices.ComTypes; using System.Text; using System.Threading.Tasks; +using Microsoft.Build.Shared; namespace Microsoft.Build.Logging { @@ -29,7 +30,7 @@ public static Stream EnsureSeekableStream(Stream stream) if(!stream.CanRead) { - throw new InvalidOperationException("Stream must be readable."); + throw new InvalidOperationException(ResourceUtilities.GetResourceString("Binlog_StreamUtils_MustBeReadable")); } return new TransparentReadStream(stream); @@ -44,7 +45,7 @@ public static TransparentReadStream EnsureTransparentReadStream(Stream stream) if (!stream.CanRead) { - throw new InvalidOperationException("Stream must be readable."); + throw new InvalidOperationException(ResourceUtilities.GetResourceString("Binlog_StreamUtils_MustBeReadable")); } return new TransparentReadStream(stream); @@ -85,7 +86,8 @@ public override int Read(byte[] buffer, int offset, int count) if (_position + count > _maxAllowedPosition) { throw new StreamChunkOverReadException( - $"Attempt to read {count} bytes, when only {_maxAllowedPosition - _position} are allowed to be read."); + ResourceUtilities.FormatResourceStringStripCodeAndKeyword("Binlog_StreamUtils_OverRead", count, + _maxAllowedPosition - _position)); } int cnt = _stream.Read(buffer, offset, count); @@ -97,7 +99,7 @@ public override long Seek(long offset, SeekOrigin origin) { if(origin != SeekOrigin.Current) { - throw new InvalidOperationException("Only seeking from SeekOrigin.Current is supported."); + throw new InvalidOperationException(ResourceUtilities.GetResourceString("Binlog_StreamUtils_SeekNonOrigin")); } this.SkipBytes((int)offset, true); @@ -107,12 +109,12 @@ public override long Seek(long offset, SeekOrigin origin) public override void SetLength(long value) { - throw new InvalidOperationException("Expanding stream is not supported."); + throw new InvalidOperationException(ResourceUtilities.GetResourceString("Binlog_StreamUtils_ExpandUnsupported")); } public override void Write(byte[] buffer, int offset, int count) { - throw new InvalidOperationException("Writing is not supported."); + throw new InvalidOperationException(ResourceUtilities.GetResourceString("Binlog_StreamUtils_WriteUnsupported")); } public override void Close() => _stream.Close(); diff --git a/src/Build/Logging/BinaryLogger/ProjectImportsCollector.cs b/src/Build/Logging/BinaryLogger/ProjectImportsCollector.cs index a1f37876ca9..b609f54bd25 100644 --- a/src/Build/Logging/BinaryLogger/ProjectImportsCollector.cs +++ b/src/Build/Logging/BinaryLogger/ProjectImportsCollector.cs @@ -257,7 +257,7 @@ public void ProcessResult(Action consumeStream, Action onError) if (fileStream.Length > int.MaxValue) { - onError("Imported files archive exceeded 2GB limit and it's not embedded."); + onError(ResourceUtilities.GetResourceString("Binlog_ImportFileSizeError")); } else { diff --git a/src/Build/Resources/Strings.resx b/src/Build/Resources/Strings.resx index 6b931215fe9..e9cff203b6e 100644 --- a/src/Build/Resources/Strings.resx +++ b/src/Build/Resources/Strings.resx @@ -2004,6 +2004,75 @@ Utilization: {0} Average Utilization: {1:###.0} Reporting file accesses is only currently supported using the x64 flavor of MSBuild. + + Structured events and raw events cannot be replayed at the same time. + + + No subscribers for any events. + + + Imported files archive exceeded 2GB limit and it's not embedded. + + + Forward compatible reading is not supported for file format version {0} (needs >= 18). + + LOCALIZATION: {0} is an integer number denoting version. + + + + Raw data sub-stream for record {0} was not fully read. + + LOCALIZATION: {0} is an integer number denoting order. + + + + Skipping the record. + + + BuildEvent record number {0} (serialized size: {1}) attempted to perform disallowed reads (error: {2}). + + LOCALIZATION: {0} is an integer number denoting order. {1} is an integer denoting size. {2} is an exception message string. + + + + BuildEvent record number {0} (serialized size: {1}) is of unsupported type: {2}. + + LOCALIZATION: {0} is an integer number denoting order. {1} is an integer denoting size. {2} is a string or integer value indicating type. + + + + BuildEvent record number {0} was expected to read exactly {1} bytes from the stream, but read {2} instead. + + LOCALIZATION: {0} is an integer number denoting order. {1} is an integer denoting size. {2} is an integer value indicating number of bytes. + + + + Content already acquired as StreamReader via GetContentReader. + + + Content already acquired as string via GetContent or initialized as string only. + + + ArchiveFile was obtained, but the final edited version was not set. + + + Stream must be readable. + + + "Attempt to read {0} bytes, when only {1} are allowed to be read. + + LOCALIZATION: {0} and {1} are integer numbers denoting number of bytes. + + + + Only seeking from SeekOrigin.Current is supported. + + + Expanding stream is not supported. + + + Writing is not supported. + + + CP0001 + T:Microsoft.Build.Logging.IBuildEventStringsReader + lib/net472/Microsoft.Build.dll + lib/net472/Microsoft.Build.dll + true + + + CP0001 + T:Microsoft.Build.Logging.IBuildEventStringsReader + lib/net8.0/Microsoft.Build.dll + lib/net8.0/Microsoft.Build.dll + true + + + CP0001 + T:Microsoft.Build.Logging.IBuildEventStringsReader + ref/net472/Microsoft.Build.dll + ref/net472/Microsoft.Build.dll + true + + + CP0001 + T:Microsoft.Build.Logging.IBuildEventStringsReader + ref/net8.0/Microsoft.Build.dll + ref/net8.0/Microsoft.Build.dll + true + + + CP0006 + E:Microsoft.Build.Logging.IBuildEventArgsReaderNotifications.ArchiveFileEncountered + lib/net472/Microsoft.Build.dll + lib/net472/Microsoft.Build.dll + true + + + CP0006 + E:Microsoft.Build.Logging.IBuildEventArgsReaderNotifications.OnRecoverableReadError + lib/net472/Microsoft.Build.dll + lib/net472/Microsoft.Build.dll + true + + + CP0006 + E:Microsoft.Build.Logging.IBuildEventArgsReaderNotifications.StringEncountered + lib/net472/Microsoft.Build.dll + lib/net472/Microsoft.Build.dll + true + + + CP0006 + E:Microsoft.Build.Logging.IBuildEventArgsReaderNotifications.StringReadDone + lib/net472/Microsoft.Build.dll + lib/net472/Microsoft.Build.dll + true + + + CP0006 + E:Microsoft.Build.Logging.IBuildEventArgsReaderNotifications.ArchiveFileEncountered + lib/net8.0/Microsoft.Build.dll + lib/net8.0/Microsoft.Build.dll + true + + + CP0006 + E:Microsoft.Build.Logging.IBuildEventArgsReaderNotifications.OnRecoverableReadError + lib/net8.0/Microsoft.Build.dll + lib/net8.0/Microsoft.Build.dll + true + + + CP0006 + E:Microsoft.Build.Logging.IBuildEventArgsReaderNotifications.StringEncountered + lib/net8.0/Microsoft.Build.dll + lib/net8.0/Microsoft.Build.dll + true + + + CP0006 + E:Microsoft.Build.Logging.IBuildEventArgsReaderNotifications.StringReadDone + lib/net8.0/Microsoft.Build.dll + lib/net8.0/Microsoft.Build.dll + true + + + CP0006 + E:Microsoft.Build.Logging.IBuildEventArgsReaderNotifications.ArchiveFileEncountered + ref/net8.0/Microsoft.Build.dll + ref/net8.0/Microsoft.Build.dll + true + + + CP0006 + E:Microsoft.Build.Logging.IBuildEventArgsReaderNotifications.OnRecoverableReadError + ref/net8.0/Microsoft.Build.dll + ref/net8.0/Microsoft.Build.dll + true + + + CP0006 + E:Microsoft.Build.Logging.IBuildEventArgsReaderNotifications.StringEncountered + ref/net8.0/Microsoft.Build.dll + ref/net8.0/Microsoft.Build.dll + true + + + CP0006 + E:Microsoft.Build.Logging.IBuildEventArgsReaderNotifications.StringReadDone + ref/net8.0/Microsoft.Build.dll + ref/net8.0/Microsoft.Build.dll + true + + + CP0008 + T:Microsoft.Build.Logging.BuildEventArgsReader + lib/net472/Microsoft.Build.dll + lib/net472/Microsoft.Build.dll + true + + + CP0008 + T:Microsoft.Build.Logging.IBuildEventArgsReaderNotifications + lib/net472/Microsoft.Build.dll + lib/net472/Microsoft.Build.dll + true + + + CP0008 + T:Microsoft.Build.Logging.BuildEventArgsReader + lib/net8.0/Microsoft.Build.dll + lib/net8.0/Microsoft.Build.dll + true + + + CP0008 + T:Microsoft.Build.Logging.IBuildEventArgsReaderNotifications + lib/net8.0/Microsoft.Build.dll + lib/net8.0/Microsoft.Build.dll + true + + + CP0008 + T:Microsoft.Build.Logging.BuildEventArgsReader + ref/net472/Microsoft.Build.dll + ref/net472/Microsoft.Build.dll + true + + + CP0008 + T:Microsoft.Build.Logging.IBuildEventArgsReaderNotifications + ref/net472/Microsoft.Build.dll + ref/net472/Microsoft.Build.dll + true + + + CP0008 + T:Microsoft.Build.Logging.BuildEventArgsReader + ref/net8.0/Microsoft.Build.dll + ref/net8.0/Microsoft.Build.dll + true + + + CP0008 + T:Microsoft.Build.Logging.IBuildEventArgsReaderNotifications + ref/net8.0/Microsoft.Build.dll + ref/net8.0/Microsoft.Build.dll + true + + \ No newline at end of file From f5c654bd82352692fa6ebf15fa937bb627a95fbf Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Thu, 23 Nov 2023 14:47:04 +0100 Subject: [PATCH 40/49] Reflect PR commnets --- src/Build/Logging/BinaryLogger/ProjectImportsCollector.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Build/Logging/BinaryLogger/ProjectImportsCollector.cs b/src/Build/Logging/BinaryLogger/ProjectImportsCollector.cs index 92563b63b4f..536d7f9ed48 100644 --- a/src/Build/Logging/BinaryLogger/ProjectImportsCollector.cs +++ b/src/Build/Logging/BinaryLogger/ProjectImportsCollector.cs @@ -136,14 +136,17 @@ private void AddFileHelper( } } - void TryAddFile() + bool TryAddFile() { try { addFileWorker(filePath); + return true; } catch { } + + return false; } } From 0c19bfd3fae419259e5251ce480f6ac22b156571 Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Thu, 23 Nov 2023 20:19:42 +0100 Subject: [PATCH 41/49] Bugfixes --- .../Logging/BinaryLogger/BinaryLogReplayEventSource.cs | 7 +++++-- src/Build/Logging/BinaryLogger/BuildEventArgsWriter.cs | 3 +-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/Build/Logging/BinaryLogger/BinaryLogReplayEventSource.cs b/src/Build/Logging/BinaryLogger/BinaryLogReplayEventSource.cs index cee55c5231d..bd9ec7d2888 100644 --- a/src/Build/Logging/BinaryLogger/BinaryLogReplayEventSource.cs +++ b/src/Build/Logging/BinaryLogger/BinaryLogReplayEventSource.cs @@ -147,7 +147,10 @@ public static BuildEventArgsReader OpenBuildEventsReader( bool allowForwardCompatibility = true) { int fileFormatVersion = binaryReader.ReadInt32(); - int minimumReaderVersion = binaryReader.ReadInt32(); + // Is this the new log format that contains the minimum reader version? + int minimumReaderVersion = fileFormatVersion >= BinaryLogger.ForwardCompatibilityMinimalVersion + ? binaryReader.ReadInt32() + : fileFormatVersion; // the log file is written using a newer version of file format // that we don't know how to read @@ -321,7 +324,7 @@ event Action? IBuildEventArgsReaderNotifications.StringEncountered } private Action? _rawLogRecordReceived; - /// + /// event Action? IRawLogEventsSource.RawLogRecordReceived { add => _rawLogRecordReceived += value; diff --git a/src/Build/Logging/BinaryLogger/BuildEventArgsWriter.cs b/src/Build/Logging/BinaryLogger/BuildEventArgsWriter.cs index b7fae9572c7..c3514bd62f5 100644 --- a/src/Build/Logging/BinaryLogger/BuildEventArgsWriter.cs +++ b/src/Build/Logging/BinaryLogger/BuildEventArgsWriter.cs @@ -1139,8 +1139,7 @@ private void Write(byte[] bytes) private void Write(Stream stream) { - Stream destinationStream = binaryWriter.BaseStream; - stream.CopyTo(destinationStream); + stream.CopyTo(binaryWriter.BaseStream); } private void Write(byte b) From bb7696e457c9efa07d487c5a89d1bc0b7b5f7179 Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Fri, 8 Dec 2023 17:59:35 +0100 Subject: [PATCH 42/49] Fix log wording --- src/Build/Resources/Strings.resx | 2 +- src/Build/Resources/xlf/Strings.cs.xlf | 4 ++-- src/Build/Resources/xlf/Strings.de.xlf | 4 ++-- src/Build/Resources/xlf/Strings.es.xlf | 4 ++-- src/Build/Resources/xlf/Strings.fr.xlf | 4 ++-- src/Build/Resources/xlf/Strings.it.xlf | 4 ++-- src/Build/Resources/xlf/Strings.ja.xlf | 4 ++-- src/Build/Resources/xlf/Strings.ko.xlf | 4 ++-- src/Build/Resources/xlf/Strings.pl.xlf | 4 ++-- src/Build/Resources/xlf/Strings.pt-BR.xlf | 4 ++-- src/Build/Resources/xlf/Strings.ru.xlf | 4 ++-- src/Build/Resources/xlf/Strings.tr.xlf | 4 ++-- src/Build/Resources/xlf/Strings.zh-Hans.xlf | 4 ++-- src/Build/Resources/xlf/Strings.zh-Hant.xlf | 4 ++-- 14 files changed, 27 insertions(+), 27 deletions(-) diff --git a/src/Build/Resources/Strings.resx b/src/Build/Resources/Strings.resx index e9402207f89..d25e6834067 100644 --- a/src/Build/Resources/Strings.resx +++ b/src/Build/Resources/Strings.resx @@ -1352,7 +1352,7 @@ {StrBegin="MSB4068: "} - MSB4235: The log file format version is {0} with minimum required read version {1}, whereas this version of MSBuild only supports versions up to {2}. + MSB4235: The log file format version is {0} with minimum required reader version {1}, whereas this version of MSBuild only supports versions up to {2}. {StrBegin="MSB4235: "} diff --git a/src/Build/Resources/xlf/Strings.cs.xlf b/src/Build/Resources/xlf/Strings.cs.xlf index 61d1b2f1a17..719da46234f 100644 --- a/src/Build/Resources/xlf/Strings.cs.xlf +++ b/src/Build/Resources/xlf/Strings.cs.xlf @@ -2585,8 +2585,8 @@ Využití: Průměrné využití {0}: {1:###.0} {StrBegin="MSB4238: "} - MSB4235: The log file format version is {0} with minimum required read version {1}, whereas this version of MSBuild only supports versions up to {2}. - MSB4235: The log file format version is {0} with minimum required read version {1}, whereas this version of MSBuild only supports versions up to {2}. + MSB4235: The log file format version is {0} with minimum required reader version {1}, whereas this version of MSBuild only supports versions up to {2}. + MSB4235: The log file format version is {0} with minimum required reader version {1}, whereas this version of MSBuild only supports versions up to {2}. {StrBegin="MSB4235: "} diff --git a/src/Build/Resources/xlf/Strings.de.xlf b/src/Build/Resources/xlf/Strings.de.xlf index 418dbf43f2c..9bc52d91042 100644 --- a/src/Build/Resources/xlf/Strings.de.xlf +++ b/src/Build/Resources/xlf/Strings.de.xlf @@ -2585,8 +2585,8 @@ Auslastung: {0} Durchschnittliche Auslastung: {1:###.0} {StrBegin="MSB4238: "} - MSB4235: The log file format version is {0} with minimum required read version {1}, whereas this version of MSBuild only supports versions up to {2}. - MSB4235: The log file format version is {0} with minimum required read version {1}, whereas this version of MSBuild only supports versions up to {2}. + MSB4235: The log file format version is {0} with minimum required reader version {1}, whereas this version of MSBuild only supports versions up to {2}. + MSB4235: The log file format version is {0} with minimum required reader version {1}, whereas this version of MSBuild only supports versions up to {2}. {StrBegin="MSB4235: "} diff --git a/src/Build/Resources/xlf/Strings.es.xlf b/src/Build/Resources/xlf/Strings.es.xlf index c31a5a73463..406a8c0111b 100644 --- a/src/Build/Resources/xlf/Strings.es.xlf +++ b/src/Build/Resources/xlf/Strings.es.xlf @@ -2585,8 +2585,8 @@ Utilización: Utilización media de {0}: {1:###.0} {StrBegin="MSB4238: "} - MSB4235: The log file format version is {0} with minimum required read version {1}, whereas this version of MSBuild only supports versions up to {2}. - MSB4235: The log file format version is {0} with minimum required read version {1}, whereas this version of MSBuild only supports versions up to {2}. + MSB4235: The log file format version is {0} with minimum required reader version {1}, whereas this version of MSBuild only supports versions up to {2}. + MSB4235: The log file format version is {0} with minimum required reader version {1}, whereas this version of MSBuild only supports versions up to {2}. {StrBegin="MSB4235: "} diff --git a/src/Build/Resources/xlf/Strings.fr.xlf b/src/Build/Resources/xlf/Strings.fr.xlf index fb4722b2e2f..ce3265990e0 100644 --- a/src/Build/Resources/xlf/Strings.fr.xlf +++ b/src/Build/Resources/xlf/Strings.fr.xlf @@ -2585,8 +2585,8 @@ Utilisation : {0} Utilisation moyenne : {1:###.0} {StrBegin="MSB4238: "} - MSB4235: The log file format version is {0} with minimum required read version {1}, whereas this version of MSBuild only supports versions up to {2}. - MSB4235: The log file format version is {0} with minimum required read version {1}, whereas this version of MSBuild only supports versions up to {2}. + MSB4235: The log file format version is {0} with minimum required reader version {1}, whereas this version of MSBuild only supports versions up to {2}. + MSB4235: The log file format version is {0} with minimum required reader version {1}, whereas this version of MSBuild only supports versions up to {2}. {StrBegin="MSB4235: "} diff --git a/src/Build/Resources/xlf/Strings.it.xlf b/src/Build/Resources/xlf/Strings.it.xlf index 41a12207488..a5e504e2899 100644 --- a/src/Build/Resources/xlf/Strings.it.xlf +++ b/src/Build/Resources/xlf/Strings.it.xlf @@ -2585,8 +2585,8 @@ Utilizzo: {0} Utilizzo medio: {1:###.0} {StrBegin="MSB4238: "} - MSB4235: The log file format version is {0} with minimum required read version {1}, whereas this version of MSBuild only supports versions up to {2}. - MSB4235: The log file format version is {0} with minimum required read version {1}, whereas this version of MSBuild only supports versions up to {2}. + MSB4235: The log file format version is {0} with minimum required reader version {1}, whereas this version of MSBuild only supports versions up to {2}. + MSB4235: The log file format version is {0} with minimum required reader version {1}, whereas this version of MSBuild only supports versions up to {2}. {StrBegin="MSB4235: "} diff --git a/src/Build/Resources/xlf/Strings.ja.xlf b/src/Build/Resources/xlf/Strings.ja.xlf index de107fe1a3a..f7512b9f8f9 100644 --- a/src/Build/Resources/xlf/Strings.ja.xlf +++ b/src/Build/Resources/xlf/Strings.ja.xlf @@ -2585,8 +2585,8 @@ Utilization: {0} Average Utilization: {1:###.0} {StrBegin="MSB4238: "} - MSB4235: The log file format version is {0} with minimum required read version {1}, whereas this version of MSBuild only supports versions up to {2}. - MSB4235: The log file format version is {0} with minimum required read version {1}, whereas this version of MSBuild only supports versions up to {2}. + MSB4235: The log file format version is {0} with minimum required reader version {1}, whereas this version of MSBuild only supports versions up to {2}. + MSB4235: The log file format version is {0} with minimum required reader version {1}, whereas this version of MSBuild only supports versions up to {2}. {StrBegin="MSB4235: "} diff --git a/src/Build/Resources/xlf/Strings.ko.xlf b/src/Build/Resources/xlf/Strings.ko.xlf index ea569b69acf..2611f000f2b 100644 --- a/src/Build/Resources/xlf/Strings.ko.xlf +++ b/src/Build/Resources/xlf/Strings.ko.xlf @@ -2585,8 +2585,8 @@ Utilization: {0} Average Utilization: {1:###.0} {StrBegin="MSB4238: "} - MSB4235: The log file format version is {0} with minimum required read version {1}, whereas this version of MSBuild only supports versions up to {2}. - MSB4235: The log file format version is {0} with minimum required read version {1}, whereas this version of MSBuild only supports versions up to {2}. + MSB4235: The log file format version is {0} with minimum required reader version {1}, whereas this version of MSBuild only supports versions up to {2}. + MSB4235: The log file format version is {0} with minimum required reader version {1}, whereas this version of MSBuild only supports versions up to {2}. {StrBegin="MSB4235: "} diff --git a/src/Build/Resources/xlf/Strings.pl.xlf b/src/Build/Resources/xlf/Strings.pl.xlf index c322c4a1736..a30ef170aab 100644 --- a/src/Build/Resources/xlf/Strings.pl.xlf +++ b/src/Build/Resources/xlf/Strings.pl.xlf @@ -2585,8 +2585,8 @@ Wykorzystanie: Średnie wykorzystanie {0}: {1:###.0} {StrBegin="MSB4238: "} - MSB4235: The log file format version is {0} with minimum required read version {1}, whereas this version of MSBuild only supports versions up to {2}. - MSB4235: The log file format version is {0} with minimum required read version {1}, whereas this version of MSBuild only supports versions up to {2}. + MSB4235: The log file format version is {0} with minimum required reader version {1}, whereas this version of MSBuild only supports versions up to {2}. + MSB4235: The log file format version is {0} with minimum required reader version {1}, whereas this version of MSBuild only supports versions up to {2}. {StrBegin="MSB4235: "} diff --git a/src/Build/Resources/xlf/Strings.pt-BR.xlf b/src/Build/Resources/xlf/Strings.pt-BR.xlf index ec128a9a7a3..ab07fc50c27 100644 --- a/src/Build/Resources/xlf/Strings.pt-BR.xlf +++ b/src/Build/Resources/xlf/Strings.pt-BR.xlf @@ -2585,8 +2585,8 @@ Utilização: {0} Utilização Média: {1:###.0} {StrBegin="MSB4238: "} - MSB4235: The log file format version is {0} with minimum required read version {1}, whereas this version of MSBuild only supports versions up to {2}. - MSB4235: The log file format version is {0} with minimum required read version {1}, whereas this version of MSBuild only supports versions up to {2}. + MSB4235: The log file format version is {0} with minimum required reader version {1}, whereas this version of MSBuild only supports versions up to {2}. + MSB4235: The log file format version is {0} with minimum required reader version {1}, whereas this version of MSBuild only supports versions up to {2}. {StrBegin="MSB4235: "} diff --git a/src/Build/Resources/xlf/Strings.ru.xlf b/src/Build/Resources/xlf/Strings.ru.xlf index 35abe93ce08..694907c581d 100644 --- a/src/Build/Resources/xlf/Strings.ru.xlf +++ b/src/Build/Resources/xlf/Strings.ru.xlf @@ -2585,8 +2585,8 @@ Utilization: {0} Average Utilization: {1:###.0} {StrBegin="MSB4238: "} - MSB4235: The log file format version is {0} with minimum required read version {1}, whereas this version of MSBuild only supports versions up to {2}. - MSB4235: The log file format version is {0} with minimum required read version {1}, whereas this version of MSBuild only supports versions up to {2}. + MSB4235: The log file format version is {0} with minimum required reader version {1}, whereas this version of MSBuild only supports versions up to {2}. + MSB4235: The log file format version is {0} with minimum required reader version {1}, whereas this version of MSBuild only supports versions up to {2}. {StrBegin="MSB4235: "} diff --git a/src/Build/Resources/xlf/Strings.tr.xlf b/src/Build/Resources/xlf/Strings.tr.xlf index fe1b2c29ec5..631652f8609 100644 --- a/src/Build/Resources/xlf/Strings.tr.xlf +++ b/src/Build/Resources/xlf/Strings.tr.xlf @@ -2585,8 +2585,8 @@ Kullanım: {0} Ortalama Kullanım: {1:###.0} {StrBegin="MSB4238: "} - MSB4235: The log file format version is {0} with minimum required read version {1}, whereas this version of MSBuild only supports versions up to {2}. - MSB4235: The log file format version is {0} with minimum required read version {1}, whereas this version of MSBuild only supports versions up to {2}. + MSB4235: The log file format version is {0} with minimum required reader version {1}, whereas this version of MSBuild only supports versions up to {2}. + MSB4235: The log file format version is {0} with minimum required reader version {1}, whereas this version of MSBuild only supports versions up to {2}. {StrBegin="MSB4235: "} diff --git a/src/Build/Resources/xlf/Strings.zh-Hans.xlf b/src/Build/Resources/xlf/Strings.zh-Hans.xlf index 780626cfa07..a65a05957d5 100644 --- a/src/Build/Resources/xlf/Strings.zh-Hans.xlf +++ b/src/Build/Resources/xlf/Strings.zh-Hans.xlf @@ -2585,8 +2585,8 @@ Utilization: {0} Average Utilization: {1:###.0} {StrBegin="MSB4238: "} - MSB4235: The log file format version is {0} with minimum required read version {1}, whereas this version of MSBuild only supports versions up to {2}. - MSB4235: The log file format version is {0} with minimum required read version {1}, whereas this version of MSBuild only supports versions up to {2}. + MSB4235: The log file format version is {0} with minimum required reader version {1}, whereas this version of MSBuild only supports versions up to {2}. + MSB4235: The log file format version is {0} with minimum required reader version {1}, whereas this version of MSBuild only supports versions up to {2}. {StrBegin="MSB4235: "} diff --git a/src/Build/Resources/xlf/Strings.zh-Hant.xlf b/src/Build/Resources/xlf/Strings.zh-Hant.xlf index 95bf3a11eed..13e4e15b3de 100644 --- a/src/Build/Resources/xlf/Strings.zh-Hant.xlf +++ b/src/Build/Resources/xlf/Strings.zh-Hant.xlf @@ -2585,8 +2585,8 @@ Utilization: {0} Average Utilization: {1:###.0} {StrBegin="MSB4238: "} - MSB4235: The log file format version is {0} with minimum required read version {1}, whereas this version of MSBuild only supports versions up to {2}. - MSB4235: The log file format version is {0} with minimum required read version {1}, whereas this version of MSBuild only supports versions up to {2}. + MSB4235: The log file format version is {0} with minimum required reader version {1}, whereas this version of MSBuild only supports versions up to {2}. + MSB4235: The log file format version is {0} with minimum required reader version {1}, whereas this version of MSBuild only supports versions up to {2}. {StrBegin="MSB4235: "} From 61e933de5228a0946dab6c8359618ab68ed044d5 Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Fri, 8 Dec 2023 18:32:57 +0100 Subject: [PATCH 43/49] Cleaning up the api (with single batch of breaking changes in the feature) --- documentation/wiki/Binary-Log.md | 8 +- .../BuildEventArgsSerialization_Tests.cs | 8 +- src/Build/CompatibilitySuppressions.xml | 169 +++++++++++++----- .../BinaryLogReplayEventSource.cs | 29 +-- .../BinaryLogger/BuildEventArgsReader.cs | 12 +- .../BinaryLogReaderErrorEventArgs.cs | 2 +- .../IBuildEventArgsReaderNotifications.cs | 19 +- src/Build/Resources/Strings.resx | 2 +- src/Build/Resources/xlf/Strings.cs.xlf | 4 +- src/Build/Resources/xlf/Strings.de.xlf | 4 +- src/Build/Resources/xlf/Strings.es.xlf | 4 +- src/Build/Resources/xlf/Strings.fr.xlf | 4 +- src/Build/Resources/xlf/Strings.it.xlf | 4 +- src/Build/Resources/xlf/Strings.ja.xlf | 4 +- src/Build/Resources/xlf/Strings.ko.xlf | 4 +- src/Build/Resources/xlf/Strings.pl.xlf | 4 +- src/Build/Resources/xlf/Strings.pt-BR.xlf | 4 +- src/Build/Resources/xlf/Strings.ru.xlf | 4 +- src/Build/Resources/xlf/Strings.tr.xlf | 4 +- src/Build/Resources/xlf/Strings.zh-Hans.xlf | 4 +- src/Build/Resources/xlf/Strings.zh-Hant.xlf | 4 +- 21 files changed, 177 insertions(+), 124 deletions(-) diff --git a/documentation/wiki/Binary-Log.md b/documentation/wiki/Binary-Log.md index 1252255ea99..d9bb101159f 100644 --- a/documentation/wiki/Binary-Log.md +++ b/documentation/wiki/Binary-Log.md @@ -132,11 +132,11 @@ In compatibility mode (default for `BinaryLogReplayEventSource`. Only supported The unknown events and event parts are regarded as recoverable errors, since the reader is able to continue reading subsequent records in the binlog. However the specific user logic should have the last call in deciding whether errors are really recoverable (e.g. is presence of unrecognized or unparseable event ok? It might be fine when searching only for specific events - e.g. errors but not acceptable when trying to provide definitive overview of the built). -To allow the calling code to decide - based on the type of error, type of events getting the error, or the number of errors - the `OnRecoverableReadError` event is exposed (from both `BinaryLogReplayEventSource` and `BuildEventArgsReader`). +To allow the calling code to decide - based on the type of error, type of events getting the error, or the number of errors - the `RecoverableReadError` event is exposed (from both `BinaryLogReplayEventSource` and `BuildEventArgsReader`). ```csharp /// -/// An event args for event. +/// An event args for event. /// public sealed class BinaryLogReaderErrorEventArgs : EventArgs { @@ -164,7 +164,7 @@ public sealed class BinaryLogReaderErrorEventArgs : EventArgs /// In case of this is raised before returning the structured representation of a build event /// that has some extra unknown data in the binlog. In case of other error types this event is raised and the offending build event is skipped and not returned. /// -event Action? OnRecoverableReadError; +event Action? RecoverableReadError; ``` Our sample usage of the [Reading API](#reading-api) can be enhanced with recoverable errors handling e.g. as such: @@ -172,7 +172,7 @@ Our sample usage of the [Reading API](#reading-api) can be enhanced with recover ```csharp // Those can be raised only during forward compatibility reading mode. -logReader.OnRecoverableReadError += errorEventArgs => +logReader.RecoverableReadError += errorEventArgs => { // ... diff --git a/src/Build.UnitTests/BuildEventArgsSerialization_Tests.cs b/src/Build.UnitTests/BuildEventArgsSerialization_Tests.cs index 4147411d105..4b879ecf122 100644 --- a/src/Build.UnitTests/BuildEventArgsSerialization_Tests.cs +++ b/src/Build.UnitTests/BuildEventArgsSerialization_Tests.cs @@ -939,7 +939,7 @@ public void ForwardCompatibleRead_HandleAppendOnlyChanges() }; List readerErrors = new(); - buildEventArgsReader.OnRecoverableReadError += readerErrors.Add; + buildEventArgsReader.RecoverableReadError += readerErrors.Add; var deserializedError = (BuildErrorEventArgs)buildEventArgsReader.Read(); @@ -999,7 +999,7 @@ public void ForwardCompatibleRead_HandleUnknownEvent() SkipUnknownEvents = true }; - buildEventArgsReader.OnRecoverableReadError += readerErrors.Add; + buildEventArgsReader.RecoverableReadError += readerErrors.Add; var deserializedEvent = buildEventArgsReader.Read(); @@ -1050,7 +1050,7 @@ public void ForwardCompatibleRead_HandleMismatchedFormatOfEvent() }; List readerErrors = new(); - buildEventArgsReader.OnRecoverableReadError += readerErrors.Add; + buildEventArgsReader.RecoverableReadError += readerErrors.Add; var deserializedEvent = buildEventArgsReader.Read(); @@ -1106,7 +1106,7 @@ public void ForwardCompatibleRead_HandleRemovalOfDataFromEventDefinition() }; List readerErrors = new(); - buildEventArgsReader.OnRecoverableReadError += readerErrors.Add; + buildEventArgsReader.RecoverableReadError += readerErrors.Add; var deserializedEvent = buildEventArgsReader.Read(); diff --git a/src/Build/CompatibilitySuppressions.xml b/src/Build/CompatibilitySuppressions.xml index 3772d5ac9f7..302d082368a 100644 --- a/src/Build/CompatibilitySuppressions.xml +++ b/src/Build/CompatibilitySuppressions.xml @@ -1,7 +1,34 @@  - + + CP0001 + T:Microsoft.Build.Logging.IBuildEventStringsReader + lib/net472/Microsoft.Build.dll + lib/net472/Microsoft.Build.dll + true + + + CP0001 + T:Microsoft.Build.Logging.IBuildEventStringsReader + lib/net8.0/Microsoft.Build.dll + lib/net8.0/Microsoft.Build.dll + true + + + CP0001 + T:Microsoft.Build.Logging.IBuildEventStringsReader + ref/net472/Microsoft.Build.dll + ref/net472/Microsoft.Build.dll + true + + + CP0001 + T:Microsoft.Build.Logging.IBuildEventStringsReader + ref/net8.0/Microsoft.Build.dll + ref/net8.0/Microsoft.Build.dll + true + CP0002 M:Microsoft.Build.Experimental.ProjectCache.ProjectCachePluginBase.HandleFileAccess(Microsoft.Build.Experimental.ProjectCache.FileAccessContext,Microsoft.Build.Framework.FileAccess.FileAccessData) @@ -16,6 +43,34 @@ lib/net472/Microsoft.Build.dll true + + CP0002 + M:Microsoft.Build.Logging.BinaryLogReplayEventSource.add_NotificationsSourceCreated(System.Action{Microsoft.Build.Logging.IBuildEventArgsReaderNotifications}) + lib/net472/Microsoft.Build.dll + lib/net472/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.BinaryLogReplayEventSource.remove_NotificationsSourceCreated(System.Action{Microsoft.Build.Logging.IBuildEventArgsReaderNotifications}) + lib/net472/Microsoft.Build.dll + lib/net472/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.BuildEventArgsReader.add_StringEncountered(System.Action) + lib/net472/Microsoft.Build.dll + lib/net472/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.BuildEventArgsReader.remove_StringEncountered(System.Action) + lib/net472/Microsoft.Build.dll + lib/net472/Microsoft.Build.dll + true + CP0002 M:Microsoft.Build.Experimental.ProjectCache.ProjectCachePluginBase.HandleFileAccess(Microsoft.Build.Experimental.ProjectCache.FileAccessContext,Microsoft.Build.Framework.FileAccess.FileAccessData) @@ -30,6 +85,34 @@ lib/net8.0/Microsoft.Build.dll true + + CP0002 + M:Microsoft.Build.Logging.BinaryLogReplayEventSource.add_NotificationsSourceCreated(System.Action{Microsoft.Build.Logging.IBuildEventArgsReaderNotifications}) + lib/net8.0/Microsoft.Build.dll + lib/net8.0/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.BinaryLogReplayEventSource.remove_NotificationsSourceCreated(System.Action{Microsoft.Build.Logging.IBuildEventArgsReaderNotifications}) + lib/net8.0/Microsoft.Build.dll + lib/net8.0/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.BuildEventArgsReader.add_StringEncountered(System.Action) + lib/net8.0/Microsoft.Build.dll + lib/net8.0/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.BuildEventArgsReader.remove_StringEncountered(System.Action) + lib/net8.0/Microsoft.Build.dll + lib/net8.0/Microsoft.Build.dll + true + CP0002 M:Microsoft.Build.Experimental.ProjectCache.ProjectCachePluginBase.HandleFileAccess(Microsoft.Build.Experimental.ProjectCache.FileAccessContext,Microsoft.Build.Framework.FileAccess.FileAccessData) @@ -44,6 +127,34 @@ ref/net472/Microsoft.Build.dll true + + CP0002 + M:Microsoft.Build.Logging.BinaryLogReplayEventSource.add_NotificationsSourceCreated(System.Action{Microsoft.Build.Logging.IBuildEventArgsReaderNotifications}) + ref/net472/Microsoft.Build.dll + ref/net472/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.BinaryLogReplayEventSource.remove_NotificationsSourceCreated(System.Action{Microsoft.Build.Logging.IBuildEventArgsReaderNotifications}) + ref/net472/Microsoft.Build.dll + ref/net472/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.BuildEventArgsReader.add_StringEncountered(System.Action) + ref/net472/Microsoft.Build.dll + ref/net472/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.BuildEventArgsReader.remove_StringEncountered(System.Action) + ref/net472/Microsoft.Build.dll + ref/net472/Microsoft.Build.dll + true + CP0002 M:Microsoft.Build.Experimental.ProjectCache.ProjectCachePluginBase.HandleFileAccess(Microsoft.Build.Experimental.ProjectCache.FileAccessContext,Microsoft.Build.Framework.FileAccess.FileAccessData) @@ -58,31 +169,30 @@ ref/net8.0/Microsoft.Build.dll true - - CP0001 - T:Microsoft.Build.Logging.IBuildEventStringsReader - lib/net472/Microsoft.Build.dll - lib/net472/Microsoft.Build.dll + CP0002 + M:Microsoft.Build.Logging.BinaryLogReplayEventSource.add_NotificationsSourceCreated(System.Action{Microsoft.Build.Logging.IBuildEventArgsReaderNotifications}) + ref/net8.0/Microsoft.Build.dll + ref/net8.0/Microsoft.Build.dll true - CP0001 - T:Microsoft.Build.Logging.IBuildEventStringsReader - lib/net8.0/Microsoft.Build.dll - lib/net8.0/Microsoft.Build.dll + CP0002 + M:Microsoft.Build.Logging.BinaryLogReplayEventSource.remove_NotificationsSourceCreated(System.Action{Microsoft.Build.Logging.IBuildEventArgsReaderNotifications}) + ref/net8.0/Microsoft.Build.dll + ref/net8.0/Microsoft.Build.dll true - CP0001 - T:Microsoft.Build.Logging.IBuildEventStringsReader - ref/net472/Microsoft.Build.dll - ref/net472/Microsoft.Build.dll + CP0002 + M:Microsoft.Build.Logging.BuildEventArgsReader.add_StringEncountered(System.Action) + ref/net8.0/Microsoft.Build.dll + ref/net8.0/Microsoft.Build.dll true - CP0001 - T:Microsoft.Build.Logging.IBuildEventStringsReader + CP0002 + M:Microsoft.Build.Logging.BuildEventArgsReader.remove_StringEncountered(System.Action) ref/net8.0/Microsoft.Build.dll ref/net8.0/Microsoft.Build.dll true @@ -96,14 +206,7 @@ CP0006 - E:Microsoft.Build.Logging.IBuildEventArgsReaderNotifications.OnRecoverableReadError - lib/net472/Microsoft.Build.dll - lib/net472/Microsoft.Build.dll - true - - - CP0006 - E:Microsoft.Build.Logging.IBuildEventArgsReaderNotifications.StringEncountered + E:Microsoft.Build.Logging.IBuildEventArgsReaderNotifications.RecoverableReadError lib/net472/Microsoft.Build.dll lib/net472/Microsoft.Build.dll true @@ -124,14 +227,7 @@ CP0006 - E:Microsoft.Build.Logging.IBuildEventArgsReaderNotifications.OnRecoverableReadError - lib/net8.0/Microsoft.Build.dll - lib/net8.0/Microsoft.Build.dll - true - - - CP0006 - E:Microsoft.Build.Logging.IBuildEventArgsReaderNotifications.StringEncountered + E:Microsoft.Build.Logging.IBuildEventArgsReaderNotifications.RecoverableReadError lib/net8.0/Microsoft.Build.dll lib/net8.0/Microsoft.Build.dll true @@ -152,14 +248,7 @@ CP0006 - E:Microsoft.Build.Logging.IBuildEventArgsReaderNotifications.OnRecoverableReadError - ref/net8.0/Microsoft.Build.dll - ref/net8.0/Microsoft.Build.dll - true - - - CP0006 - E:Microsoft.Build.Logging.IBuildEventArgsReaderNotifications.StringEncountered + E:Microsoft.Build.Logging.IBuildEventArgsReaderNotifications.RecoverableReadError ref/net8.0/Microsoft.Build.dll ref/net8.0/Microsoft.Build.dll true diff --git a/src/Build/Logging/BinaryLogger/BinaryLogReplayEventSource.cs b/src/Build/Logging/BinaryLogger/BinaryLogReplayEventSource.cs index bd9ec7d2888..0ea5f88df34 100644 --- a/src/Build/Logging/BinaryLogger/BinaryLogReplayEventSource.cs +++ b/src/Build/Logging/BinaryLogger/BinaryLogReplayEventSource.cs @@ -63,15 +63,8 @@ static BinaryLogReplayEventSource() /// public bool AllowForwardCompatibility { private get; init; } - /// - public event Action? OnRecoverableReadError; - - /// - /// WARNING: This event is under low support and low maintenance - please use events directly exposed by instead. - /// - /// Raised once is created during replaying - /// - public event Action? NotificationsSourceCreated; + /// + public event Action? RecoverableReadError; /// /// Read the provided binary log file and raise corresponding events for each BuildEventArgs @@ -205,7 +198,6 @@ public void Replay(BinaryReader binaryReader, CancellationToken cancellationToke public void Replay(BinaryReader binaryReader, bool closeInput, CancellationToken cancellationToken) { using var reader = OpenBuildEventsReader(binaryReader, closeInput, AllowForwardCompatibility); - NotificationsSourceCreated?.Invoke(reader); Replay(reader, cancellationToken); } @@ -231,7 +223,6 @@ public void Replay(BuildEventArgsReader reader, CancellationToken cancellationTo reader.EmbeddedContentRead += _embeddedContentRead; reader.ArchiveFileEncountered += _archiveFileEncountered; reader.StringReadDone += _stringReadDone; - reader.StringEncountered += _stringEncountered; if (HasStructuredEventsSubscribers || !supportsForwardCompatibility) { @@ -244,7 +235,7 @@ public void Replay(BuildEventArgsReader reader, CancellationToken cancellationTo // Forward compatible reading makes sense only for structured events reading. reader.SkipUnknownEvents = supportsForwardCompatibility && AllowForwardCompatibility; reader.SkipUnknownEventParts = supportsForwardCompatibility && AllowForwardCompatibility; - reader.OnRecoverableReadError += OnRecoverableReadError; + reader.RecoverableReadError += RecoverableReadError; while (!cancellationToken.IsCancellationRequested && reader.Read() is { } instance) { @@ -256,8 +247,7 @@ public void Replay(BuildEventArgsReader reader, CancellationToken cancellationTo if (this._rawLogRecordReceived == null && this._embeddedContentRead == null && this._stringReadDone == null && - this._archiveFileEncountered == null && - this._stringEncountered == null) + this._archiveFileEncountered == null) { throw new NotSupportedException( ResourceUtilities.GetResourceString("Binlog_Source_MissingSubscribeError")); @@ -274,8 +264,7 @@ public void Replay(BuildEventArgsReader reader, CancellationToken cancellationTo reader.EmbeddedContentRead -= _embeddedContentRead; reader.ArchiveFileEncountered -= _archiveFileEncountered; reader.StringReadDone -= _stringReadDone; - reader.StringEncountered -= _stringEncountered; - reader.OnRecoverableReadError -= OnRecoverableReadError; + reader.RecoverableReadError -= RecoverableReadError; } /// @@ -315,14 +304,6 @@ event Action? IBuildEventArgsReaderNotifications.ArchiveFi remove => _archiveFileEncountered -= value; } - private Action? _stringEncountered; - /// - event Action? IBuildEventArgsReaderNotifications.StringEncountered - { - add => _stringEncountered += value; - remove => _stringEncountered -= value; - } - private Action? _rawLogRecordReceived; /// event Action? IRawLogEventsSource.RawLogRecordReceived diff --git a/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs b/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs index 99b8ff0fb44..04a459b7571 100644 --- a/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs +++ b/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs @@ -125,10 +125,10 @@ private void EnsureForwardCompatibleReadingSupported() } /// - /// Receives recoverable errors during reading. See for documentation on arguments. + /// Receives recoverable errors during reading. See for documentation on arguments. /// Applicable mainly when or is set to true."/> /// - public event Action? OnRecoverableReadError; + public event Action? RecoverableReadError; public void Dispose() { @@ -142,9 +142,6 @@ public void Dispose() /// public event Action? StringReadDone; - /// - public event Action? StringEncountered; - internal int FileFormatVersion => _fileFormatVersion; /// @@ -197,7 +194,7 @@ internal RawRecord ReadRaw() [MethodImpl(MethodImplOptions.AggressiveInlining)] private void CheckErrorsSubscribed() { - if ((_skipUnknownEvents || _skipUnknownEventParts) && OnRecoverableReadError == null) + if ((_skipUnknownEvents || _skipUnknownEventParts) && RecoverableReadError == null) { throw new InvalidOperationException( ResourceUtilities.GetResourceString("Binlog_MissingRecoverableErrorSubscribeError")); @@ -284,7 +281,7 @@ void HandleError(FormatErrorMessage msgFactory, bool noThrow, ReaderErrorType re { if (noThrow) { - OnRecoverableReadError?.Invoke(new BinaryLogReaderErrorEventArgs(readerErrorType, recordKind, msgFactory)); + RecoverableReadError?.Invoke(new BinaryLogReaderErrorEventArgs(readerErrorType, recordKind, msgFactory)); SkipBytes(_readStream.BytesCountAllowedToReadRemaining); } else @@ -1542,7 +1539,6 @@ private ITaskItem ReadTaskItem() private string ReadString() { - this.StringEncountered?.Invoke(); string text = _binaryReader.ReadString(); if (this.StringReadDone != null) { diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/BinaryLogReaderErrorEventArgs.cs b/src/Build/Logging/BinaryLogger/Postprocessing/BinaryLogReaderErrorEventArgs.cs index f90f7f28636..adad3e7d363 100644 --- a/src/Build/Logging/BinaryLogger/Postprocessing/BinaryLogReaderErrorEventArgs.cs +++ b/src/Build/Logging/BinaryLogger/Postprocessing/BinaryLogReaderErrorEventArgs.cs @@ -13,7 +13,7 @@ namespace Microsoft.Build.Logging internal delegate string FormatErrorMessage(); /// - /// An event args for event. + /// An event args for event. /// public sealed class BinaryLogReaderErrorEventArgs : EventArgs { diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/IBuildEventArgsReaderNotifications.cs b/src/Build/Logging/BinaryLogger/Postprocessing/IBuildEventArgsReaderNotifications.cs index 54adc7af29a..aee71562898 100644 --- a/src/Build/Logging/BinaryLogger/Postprocessing/IBuildEventArgsReaderNotifications.cs +++ b/src/Build/Logging/BinaryLogger/Postprocessing/IBuildEventArgsReaderNotifications.cs @@ -15,20 +15,7 @@ public interface IBuildEventArgsReaderNotifications /// Subscriber may adjust the string by setting property. /// The passed event arg can be reused and should not be stored. /// - public event Action? StringReadDone; - - /// - /// An event that allows the caller to be notified when a string is encountered in the binary log. - /// BinaryReader passed in ctor is at the beginning of the string at this point. - /// - /// - /// [!CAUTION] - /// > Use instead of this method. - /// ]]> - /// - public event Action? StringEncountered; + event Action? StringReadDone; /// /// An event that allows the caller to be notified when an embedded file is encountered in the binary log. @@ -50,7 +37,7 @@ public interface IBuildEventArgsReaderNotifications /// } /// /// - public event Action? ArchiveFileEncountered; + event Action? ArchiveFileEncountered; /// /// Receives recoverable errors during reading. @@ -58,6 +45,6 @@ public interface IBuildEventArgsReaderNotifications /// In case of this is raised before returning the structured representation of a build event /// that has some extra unknown data in the binlog. In case of other error types this event is raised and the offending build event is skipped and not returned. /// - event Action? OnRecoverableReadError; + event Action? RecoverableReadError; } } diff --git a/src/Build/Resources/Strings.resx b/src/Build/Resources/Strings.resx index d25e6834067..52477838f97 100644 --- a/src/Build/Resources/Strings.resx +++ b/src/Build/Resources/Strings.resx @@ -2017,7 +2017,7 @@ Utilization: {0} Average Utilization: {1:###.0} No subscribers for any events. - Subscription to OnRecoverableReadError is mandatory during forward compatible reading. + Subscription to RecoverableReadError is mandatory during forward compatible reading. Imported files archive exceeded 2GB limit and it's not embedded. diff --git a/src/Build/Resources/xlf/Strings.cs.xlf b/src/Build/Resources/xlf/Strings.cs.xlf index 719da46234f..4eb4f1b3959 100644 --- a/src/Build/Resources/xlf/Strings.cs.xlf +++ b/src/Build/Resources/xlf/Strings.cs.xlf @@ -54,8 +54,8 @@ - Subscription to OnRecoverableReadError is mandatory during forward compatible reading. - Subscription to OnRecoverableReadError is mandatory during forward compatible reading. + Subscription to RecoverableReadError is mandatory during forward compatible reading. + Subscription to RecoverableReadError is mandatory during forward compatible reading. diff --git a/src/Build/Resources/xlf/Strings.de.xlf b/src/Build/Resources/xlf/Strings.de.xlf index 9bc52d91042..be14de48f54 100644 --- a/src/Build/Resources/xlf/Strings.de.xlf +++ b/src/Build/Resources/xlf/Strings.de.xlf @@ -54,8 +54,8 @@ - Subscription to OnRecoverableReadError is mandatory during forward compatible reading. - Subscription to OnRecoverableReadError is mandatory during forward compatible reading. + Subscription to RecoverableReadError is mandatory during forward compatible reading. + Subscription to RecoverableReadError is mandatory during forward compatible reading. diff --git a/src/Build/Resources/xlf/Strings.es.xlf b/src/Build/Resources/xlf/Strings.es.xlf index 406a8c0111b..098079384e5 100644 --- a/src/Build/Resources/xlf/Strings.es.xlf +++ b/src/Build/Resources/xlf/Strings.es.xlf @@ -54,8 +54,8 @@ - Subscription to OnRecoverableReadError is mandatory during forward compatible reading. - Subscription to OnRecoverableReadError is mandatory during forward compatible reading. + Subscription to RecoverableReadError is mandatory during forward compatible reading. + Subscription to RecoverableReadError is mandatory during forward compatible reading. diff --git a/src/Build/Resources/xlf/Strings.fr.xlf b/src/Build/Resources/xlf/Strings.fr.xlf index ce3265990e0..502103dff6c 100644 --- a/src/Build/Resources/xlf/Strings.fr.xlf +++ b/src/Build/Resources/xlf/Strings.fr.xlf @@ -54,8 +54,8 @@ - Subscription to OnRecoverableReadError is mandatory during forward compatible reading. - Subscription to OnRecoverableReadError is mandatory during forward compatible reading. + Subscription to RecoverableReadError is mandatory during forward compatible reading. + Subscription to RecoverableReadError is mandatory during forward compatible reading. diff --git a/src/Build/Resources/xlf/Strings.it.xlf b/src/Build/Resources/xlf/Strings.it.xlf index a5e504e2899..fba6fc8ea6e 100644 --- a/src/Build/Resources/xlf/Strings.it.xlf +++ b/src/Build/Resources/xlf/Strings.it.xlf @@ -54,8 +54,8 @@ - Subscription to OnRecoverableReadError is mandatory during forward compatible reading. - Subscription to OnRecoverableReadError is mandatory during forward compatible reading. + Subscription to RecoverableReadError is mandatory during forward compatible reading. + Subscription to RecoverableReadError is mandatory during forward compatible reading. diff --git a/src/Build/Resources/xlf/Strings.ja.xlf b/src/Build/Resources/xlf/Strings.ja.xlf index f7512b9f8f9..e87a7fa646d 100644 --- a/src/Build/Resources/xlf/Strings.ja.xlf +++ b/src/Build/Resources/xlf/Strings.ja.xlf @@ -54,8 +54,8 @@ - Subscription to OnRecoverableReadError is mandatory during forward compatible reading. - Subscription to OnRecoverableReadError is mandatory during forward compatible reading. + Subscription to RecoverableReadError is mandatory during forward compatible reading. + Subscription to RecoverableReadError is mandatory during forward compatible reading. diff --git a/src/Build/Resources/xlf/Strings.ko.xlf b/src/Build/Resources/xlf/Strings.ko.xlf index 2611f000f2b..541a5d4e662 100644 --- a/src/Build/Resources/xlf/Strings.ko.xlf +++ b/src/Build/Resources/xlf/Strings.ko.xlf @@ -54,8 +54,8 @@ - Subscription to OnRecoverableReadError is mandatory during forward compatible reading. - Subscription to OnRecoverableReadError is mandatory during forward compatible reading. + Subscription to RecoverableReadError is mandatory during forward compatible reading. + Subscription to RecoverableReadError is mandatory during forward compatible reading. diff --git a/src/Build/Resources/xlf/Strings.pl.xlf b/src/Build/Resources/xlf/Strings.pl.xlf index a30ef170aab..18a64eb5d12 100644 --- a/src/Build/Resources/xlf/Strings.pl.xlf +++ b/src/Build/Resources/xlf/Strings.pl.xlf @@ -54,8 +54,8 @@ - Subscription to OnRecoverableReadError is mandatory during forward compatible reading. - Subscription to OnRecoverableReadError is mandatory during forward compatible reading. + Subscription to RecoverableReadError is mandatory during forward compatible reading. + Subscription to RecoverableReadError is mandatory during forward compatible reading. diff --git a/src/Build/Resources/xlf/Strings.pt-BR.xlf b/src/Build/Resources/xlf/Strings.pt-BR.xlf index ab07fc50c27..35a450c1578 100644 --- a/src/Build/Resources/xlf/Strings.pt-BR.xlf +++ b/src/Build/Resources/xlf/Strings.pt-BR.xlf @@ -54,8 +54,8 @@ - Subscription to OnRecoverableReadError is mandatory during forward compatible reading. - Subscription to OnRecoverableReadError is mandatory during forward compatible reading. + Subscription to RecoverableReadError is mandatory during forward compatible reading. + Subscription to RecoverableReadError is mandatory during forward compatible reading. diff --git a/src/Build/Resources/xlf/Strings.ru.xlf b/src/Build/Resources/xlf/Strings.ru.xlf index 694907c581d..2c43ea7fa72 100644 --- a/src/Build/Resources/xlf/Strings.ru.xlf +++ b/src/Build/Resources/xlf/Strings.ru.xlf @@ -54,8 +54,8 @@ - Subscription to OnRecoverableReadError is mandatory during forward compatible reading. - Subscription to OnRecoverableReadError is mandatory during forward compatible reading. + Subscription to RecoverableReadError is mandatory during forward compatible reading. + Subscription to RecoverableReadError is mandatory during forward compatible reading. diff --git a/src/Build/Resources/xlf/Strings.tr.xlf b/src/Build/Resources/xlf/Strings.tr.xlf index 631652f8609..0102cea8785 100644 --- a/src/Build/Resources/xlf/Strings.tr.xlf +++ b/src/Build/Resources/xlf/Strings.tr.xlf @@ -54,8 +54,8 @@ - Subscription to OnRecoverableReadError is mandatory during forward compatible reading. - Subscription to OnRecoverableReadError is mandatory during forward compatible reading. + Subscription to RecoverableReadError is mandatory during forward compatible reading. + Subscription to RecoverableReadError is mandatory during forward compatible reading. diff --git a/src/Build/Resources/xlf/Strings.zh-Hans.xlf b/src/Build/Resources/xlf/Strings.zh-Hans.xlf index a65a05957d5..f55ff3cc279 100644 --- a/src/Build/Resources/xlf/Strings.zh-Hans.xlf +++ b/src/Build/Resources/xlf/Strings.zh-Hans.xlf @@ -54,8 +54,8 @@ - Subscription to OnRecoverableReadError is mandatory during forward compatible reading. - Subscription to OnRecoverableReadError is mandatory during forward compatible reading. + Subscription to RecoverableReadError is mandatory during forward compatible reading. + Subscription to RecoverableReadError is mandatory during forward compatible reading. diff --git a/src/Build/Resources/xlf/Strings.zh-Hant.xlf b/src/Build/Resources/xlf/Strings.zh-Hant.xlf index 13e4e15b3de..51e9939ee21 100644 --- a/src/Build/Resources/xlf/Strings.zh-Hant.xlf +++ b/src/Build/Resources/xlf/Strings.zh-Hant.xlf @@ -54,8 +54,8 @@ - Subscription to OnRecoverableReadError is mandatory during forward compatible reading. - Subscription to OnRecoverableReadError is mandatory during forward compatible reading. + Subscription to RecoverableReadError is mandatory during forward compatible reading. + Subscription to RecoverableReadError is mandatory during forward compatible reading. From eb7933a5c6a6d300ea7d265ee6d289c29c528a27 Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Mon, 11 Dec 2023 14:26:40 +0100 Subject: [PATCH 44/49] Preserve version on raw events replay --- .../BinaryLogger/BinaryLogReplayEventSource.cs | 18 ++++++++++++++++++ src/Build/Logging/BinaryLogger/BinaryLogger.cs | 11 ++++++++--- .../BinaryLogger/BuildEventArgsReader.cs | 1 + src/Build/Resources/Strings.resx | 3 +++ src/Build/Resources/xlf/Strings.cs.xlf | 5 +++++ src/Build/Resources/xlf/Strings.de.xlf | 5 +++++ src/Build/Resources/xlf/Strings.es.xlf | 5 +++++ src/Build/Resources/xlf/Strings.fr.xlf | 5 +++++ src/Build/Resources/xlf/Strings.it.xlf | 5 +++++ src/Build/Resources/xlf/Strings.ja.xlf | 5 +++++ src/Build/Resources/xlf/Strings.ko.xlf | 5 +++++ src/Build/Resources/xlf/Strings.pl.xlf | 5 +++++ src/Build/Resources/xlf/Strings.pt-BR.xlf | 5 +++++ src/Build/Resources/xlf/Strings.ru.xlf | 5 +++++ src/Build/Resources/xlf/Strings.tr.xlf | 5 +++++ src/Build/Resources/xlf/Strings.zh-Hans.xlf | 5 +++++ src/Build/Resources/xlf/Strings.zh-Hant.xlf | 5 +++++ 17 files changed, 95 insertions(+), 3 deletions(-) diff --git a/src/Build/Logging/BinaryLogger/BinaryLogReplayEventSource.cs b/src/Build/Logging/BinaryLogger/BinaryLogReplayEventSource.cs index 0ea5f88df34..2bb1dcd51a5 100644 --- a/src/Build/Logging/BinaryLogger/BinaryLogReplayEventSource.cs +++ b/src/Build/Logging/BinaryLogger/BinaryLogReplayEventSource.cs @@ -30,6 +30,16 @@ internal interface IRawLogEventsSource void DeferredInitialize( Action onRawReadingPossible, Action onStructuredReadingOnly); + + /// + /// File format version of the binary log file. + /// + int FileFormatVersion { get; } + + /// + /// The minimum reader version for the binary log file. + /// + int MinimumReaderVersion { get; } } /// @@ -157,6 +167,7 @@ public static BuildEventArgsReader OpenBuildEventsReader( return new BuildEventArgsReader(binaryReader, fileFormatVersion) { CloseInput = closeInput, + MinimumReaderVersion = minimumReaderVersion }; } @@ -208,6 +219,8 @@ public void Replay(BinaryReader binaryReader, bool closeInput, CancellationToken /// A indicating the replay should stop as soon as possible. public void Replay(BuildEventArgsReader reader, CancellationToken cancellationToken) { + _fileFormatVersion = reader.FileFormatVersion; + _minimumReaderVersion = reader.MinimumReaderVersion; bool supportsForwardCompatibility = reader.FileFormatVersion >= BinaryLogger.ForwardCompatibilityMinimalVersion; // Allow any possible deferred subscriptions to be registered @@ -276,6 +289,11 @@ void IRawLogEventsSource.DeferredInitialize( this._onStructuredReadingOnly += onStructuredReadingOnly; } + public int FileFormatVersion => _fileFormatVersion ?? throw new InvalidOperationException(ResourceUtilities.GetResourceString("Binlog_Source_VersionUninitialized")); + public int MinimumReaderVersion => _minimumReaderVersion ?? throw new InvalidOperationException(ResourceUtilities.GetResourceString("Binlog_Source_VersionUninitialized")); + + private int? _fileFormatVersion; + private int? _minimumReaderVersion; private Action? _onRawReadingPossible; private Action? _onStructuredReadingOnly; private Action? _embeddedContentRead; diff --git a/src/Build/Logging/BinaryLogger/BinaryLogger.cs b/src/Build/Logging/BinaryLogger/BinaryLogger.cs index bb0a03a019c..9f4cd60f50a 100644 --- a/src/Build/Logging/BinaryLogger/BinaryLogger.cs +++ b/src/Build/Logging/BinaryLogger/BinaryLogger.cs @@ -206,9 +206,6 @@ public void Initialize(IEventSource eventSource) eventArgsWriter.EmbedFile += EventArgsWriter_EmbedFile; } - binaryWriter.Write(FileFormatVersion); - binaryWriter.Write(MinimumReaderVersion); - if (replayEventsSource != null) { if (CollectProjectImports == ProjectImportsCollectionMode.Embed) @@ -228,8 +225,11 @@ public void Initialize(IEventSource eventSource) replayEventsSource.DeferredInitialize( // For raw events we cannot write the initial info - as we cannot write // at the same time as raw events are being written - this would break the deduplicated strings store. + // But we need to write the version info - but since we read/write raw - let's not change the version info. () => { + binaryWriter.Write(replayEventsSource.FileFormatVersion); + binaryWriter.Write(replayEventsSource.MinimumReaderVersion); replayEventsSource.RawLogRecordReceived += RawEvents_LogDataSliceReceived; // Replay separated strings here as well (and do not deduplicate! It would skew string indexes) replayEventsSource.StringReadDone += strArg => eventArgsWriter.WriteStringRecord(strArg.StringToBeUsed); @@ -245,6 +245,11 @@ public void Initialize(IEventSource eventSource) void SubscribeToStructuredEvents() { + // Write the version info - the latest version is written only for structured events replaying + // as raw events do not change structure - hence the version is the same as the one they were written with. + binaryWriter.Write(FileFormatVersion); + binaryWriter.Write(MinimumReaderVersion); + if (!omitInitialInfo) { LogInitialInfo(); diff --git a/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs b/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs index 04a459b7571..9e5bee6d8c0 100644 --- a/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs +++ b/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs @@ -143,6 +143,7 @@ public void Dispose() public event Action? StringReadDone; internal int FileFormatVersion => _fileFormatVersion; + internal int MinimumReaderVersion { get; set; } = BinaryLogger.ForwardCompatibilityMinimalVersion; /// internal event Action? EmbeddedContentRead; diff --git a/src/Build/Resources/Strings.resx b/src/Build/Resources/Strings.resx index 52477838f97..aab80835e99 100644 --- a/src/Build/Resources/Strings.resx +++ b/src/Build/Resources/Strings.resx @@ -2016,6 +2016,9 @@ Utilization: {0} Average Utilization: {1:###.0} No subscribers for any events. + + Version info not yet initialized. Replay must be called first. + Subscription to RecoverableReadError is mandatory during forward compatible reading. diff --git a/src/Build/Resources/xlf/Strings.cs.xlf b/src/Build/Resources/xlf/Strings.cs.xlf index 4eb4f1b3959..a9c52fed296 100644 --- a/src/Build/Resources/xlf/Strings.cs.xlf +++ b/src/Build/Resources/xlf/Strings.cs.xlf @@ -94,6 +94,11 @@ Structured events and raw events cannot be replayed at the same time. + + Version info not yet initialized. Replay must be called first. + Version info not yet initialized. Replay must be called first. + + Stream must be readable. Stream must be readable. diff --git a/src/Build/Resources/xlf/Strings.de.xlf b/src/Build/Resources/xlf/Strings.de.xlf index be14de48f54..1eb79228ae4 100644 --- a/src/Build/Resources/xlf/Strings.de.xlf +++ b/src/Build/Resources/xlf/Strings.de.xlf @@ -94,6 +94,11 @@ Structured events and raw events cannot be replayed at the same time. + + Version info not yet initialized. Replay must be called first. + Version info not yet initialized. Replay must be called first. + + Stream must be readable. Stream must be readable. diff --git a/src/Build/Resources/xlf/Strings.es.xlf b/src/Build/Resources/xlf/Strings.es.xlf index 098079384e5..18f768dbcab 100644 --- a/src/Build/Resources/xlf/Strings.es.xlf +++ b/src/Build/Resources/xlf/Strings.es.xlf @@ -94,6 +94,11 @@ Structured events and raw events cannot be replayed at the same time. + + Version info not yet initialized. Replay must be called first. + Version info not yet initialized. Replay must be called first. + + Stream must be readable. Stream must be readable. diff --git a/src/Build/Resources/xlf/Strings.fr.xlf b/src/Build/Resources/xlf/Strings.fr.xlf index 502103dff6c..3bad0fad7e3 100644 --- a/src/Build/Resources/xlf/Strings.fr.xlf +++ b/src/Build/Resources/xlf/Strings.fr.xlf @@ -94,6 +94,11 @@ Structured events and raw events cannot be replayed at the same time. + + Version info not yet initialized. Replay must be called first. + Version info not yet initialized. Replay must be called first. + + Stream must be readable. Stream must be readable. diff --git a/src/Build/Resources/xlf/Strings.it.xlf b/src/Build/Resources/xlf/Strings.it.xlf index fba6fc8ea6e..5cbd3aa833f 100644 --- a/src/Build/Resources/xlf/Strings.it.xlf +++ b/src/Build/Resources/xlf/Strings.it.xlf @@ -94,6 +94,11 @@ Structured events and raw events cannot be replayed at the same time. + + Version info not yet initialized. Replay must be called first. + Version info not yet initialized. Replay must be called first. + + Stream must be readable. Stream must be readable. diff --git a/src/Build/Resources/xlf/Strings.ja.xlf b/src/Build/Resources/xlf/Strings.ja.xlf index e87a7fa646d..458e98f4025 100644 --- a/src/Build/Resources/xlf/Strings.ja.xlf +++ b/src/Build/Resources/xlf/Strings.ja.xlf @@ -94,6 +94,11 @@ Structured events and raw events cannot be replayed at the same time. + + Version info not yet initialized. Replay must be called first. + Version info not yet initialized. Replay must be called first. + + Stream must be readable. Stream must be readable. diff --git a/src/Build/Resources/xlf/Strings.ko.xlf b/src/Build/Resources/xlf/Strings.ko.xlf index 541a5d4e662..3d59626edc2 100644 --- a/src/Build/Resources/xlf/Strings.ko.xlf +++ b/src/Build/Resources/xlf/Strings.ko.xlf @@ -94,6 +94,11 @@ Structured events and raw events cannot be replayed at the same time. + + Version info not yet initialized. Replay must be called first. + Version info not yet initialized. Replay must be called first. + + Stream must be readable. Stream must be readable. diff --git a/src/Build/Resources/xlf/Strings.pl.xlf b/src/Build/Resources/xlf/Strings.pl.xlf index 18a64eb5d12..beca14a7343 100644 --- a/src/Build/Resources/xlf/Strings.pl.xlf +++ b/src/Build/Resources/xlf/Strings.pl.xlf @@ -94,6 +94,11 @@ Structured events and raw events cannot be replayed at the same time. + + Version info not yet initialized. Replay must be called first. + Version info not yet initialized. Replay must be called first. + + Stream must be readable. Stream must be readable. diff --git a/src/Build/Resources/xlf/Strings.pt-BR.xlf b/src/Build/Resources/xlf/Strings.pt-BR.xlf index 35a450c1578..7180455a252 100644 --- a/src/Build/Resources/xlf/Strings.pt-BR.xlf +++ b/src/Build/Resources/xlf/Strings.pt-BR.xlf @@ -94,6 +94,11 @@ Structured events and raw events cannot be replayed at the same time. + + Version info not yet initialized. Replay must be called first. + Version info not yet initialized. Replay must be called first. + + Stream must be readable. Stream must be readable. diff --git a/src/Build/Resources/xlf/Strings.ru.xlf b/src/Build/Resources/xlf/Strings.ru.xlf index 2c43ea7fa72..7220bfe702f 100644 --- a/src/Build/Resources/xlf/Strings.ru.xlf +++ b/src/Build/Resources/xlf/Strings.ru.xlf @@ -94,6 +94,11 @@ Structured events and raw events cannot be replayed at the same time. + + Version info not yet initialized. Replay must be called first. + Version info not yet initialized. Replay must be called first. + + Stream must be readable. Stream must be readable. diff --git a/src/Build/Resources/xlf/Strings.tr.xlf b/src/Build/Resources/xlf/Strings.tr.xlf index 0102cea8785..636c733aaf9 100644 --- a/src/Build/Resources/xlf/Strings.tr.xlf +++ b/src/Build/Resources/xlf/Strings.tr.xlf @@ -94,6 +94,11 @@ Structured events and raw events cannot be replayed at the same time. + + Version info not yet initialized. Replay must be called first. + Version info not yet initialized. Replay must be called first. + + Stream must be readable. Stream must be readable. diff --git a/src/Build/Resources/xlf/Strings.zh-Hans.xlf b/src/Build/Resources/xlf/Strings.zh-Hans.xlf index f55ff3cc279..0695918b4d4 100644 --- a/src/Build/Resources/xlf/Strings.zh-Hans.xlf +++ b/src/Build/Resources/xlf/Strings.zh-Hans.xlf @@ -94,6 +94,11 @@ Structured events and raw events cannot be replayed at the same time. + + Version info not yet initialized. Replay must be called first. + Version info not yet initialized. Replay must be called first. + + Stream must be readable. Stream must be readable. diff --git a/src/Build/Resources/xlf/Strings.zh-Hant.xlf b/src/Build/Resources/xlf/Strings.zh-Hant.xlf index 51e9939ee21..151c31bb5bb 100644 --- a/src/Build/Resources/xlf/Strings.zh-Hant.xlf +++ b/src/Build/Resources/xlf/Strings.zh-Hant.xlf @@ -94,6 +94,11 @@ Structured events and raw events cannot be replayed at the same time. + + Version info not yet initialized. Replay must be called first. + Version info not yet initialized. Replay must be called first. + + Stream must be readable. Stream must be readable. From 03012249468704e414c422b3cf16b992632252f3 Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Thu, 21 Dec 2023 13:12:57 +0100 Subject: [PATCH 45/49] Reflecting PR comments --- .../BuildEventArgsSerialization_Tests.cs | 2 +- .../BinaryLogReplayEventSource.cs | 57 +++++++++++-------- .../Logging/BinaryLogger/BinaryLogger.cs | 21 +++---- .../BinaryLogger/BuildEventArgsReader.cs | 4 +- .../Postprocessing/IEmbeddedContentSource.cs | 17 ------ .../Postprocessing/ReaderErrorType.cs | 2 +- 6 files changed, 48 insertions(+), 55 deletions(-) delete mode 100644 src/Build/Logging/BinaryLogger/Postprocessing/IEmbeddedContentSource.cs diff --git a/src/Build.UnitTests/BuildEventArgsSerialization_Tests.cs b/src/Build.UnitTests/BuildEventArgsSerialization_Tests.cs index 4b879ecf122..99474115048 100644 --- a/src/Build.UnitTests/BuildEventArgsSerialization_Tests.cs +++ b/src/Build.UnitTests/BuildEventArgsSerialization_Tests.cs @@ -1004,7 +1004,7 @@ public void ForwardCompatibleRead_HandleUnknownEvent() var deserializedEvent = buildEventArgsReader.Read(); readerErrors.Count.ShouldBe(1); - readerErrors[0].ErrorType.ShouldBe(ReaderErrorType.UnkownEventType); + readerErrors[0].ErrorType.ShouldBe(ReaderErrorType.UnknownEventType); readerErrors[0].RecordKind.ShouldBe(unknownType); deserializedEvent.Should().BeEquivalentTo(finished); diff --git a/src/Build/Logging/BinaryLogger/BinaryLogReplayEventSource.cs b/src/Build/Logging/BinaryLogger/BinaryLogReplayEventSource.cs index 2bb1dcd51a5..ec70649d965 100644 --- a/src/Build/Logging/BinaryLogger/BinaryLogReplayEventSource.cs +++ b/src/Build/Logging/BinaryLogger/BinaryLogReplayEventSource.cs @@ -11,7 +11,12 @@ namespace Microsoft.Build.Logging { - internal interface IRawLogEventsSource + /// + /// Interface for replaying a binary log file (*.binlog) + /// + internal interface IBinaryLogReplaySource : + IEventSource, + IBuildEventArgsReaderNotifications { /// /// Event raised when non-textual log record is read. @@ -40,17 +45,14 @@ void DeferredInitialize( /// The minimum reader version for the binary log file. /// int MinimumReaderVersion { get; } - } - /// - /// Interface for replaying a binary log file (*.binlog) - /// - internal interface IBinaryLogReplaySource : - IEventSource, - IRawLogEventsSource, - IBuildEventArgsReaderNotifications, - IEmbeddedContentSource - { } + /// + /// Raised when the log reader encounters a project import archive (embedded content) in the stream. + /// The subscriber must read the exactly given length of binary data from the stream - otherwise exception is raised. + /// If no subscriber is attached, the data is skipped. + /// + event Action EmbeddedContentRead; + } /// /// Provides a method to read a binary log file (*.binlog) and replay all stored BuildEventArgs @@ -60,6 +62,12 @@ internal interface IBinaryLogReplaySource : public sealed class BinaryLogReplayEventSource : EventArgsDispatcher, IBinaryLogReplaySource { + private int? _fileFormatVersion; + private int? _minimumReaderVersion; + + public int FileFormatVersion => _fileFormatVersion ?? throw new InvalidOperationException(ResourceUtilities.GetResourceString("Binlog_Source_VersionUninitialized")); + public int MinimumReaderVersion => _minimumReaderVersion ?? throw new InvalidOperationException(ResourceUtilities.GetResourceString("Binlog_Source_VersionUninitialized")); + /// Touches the static constructor /// to ensure it initializes /// and @@ -280,8 +288,16 @@ public void Replay(BuildEventArgsReader reader, CancellationToken cancellationTo reader.RecoverableReadError -= RecoverableReadError; } - /// - void IRawLogEventsSource.DeferredInitialize( + // Following members are explicit implementations of the IBinaryLogReplaySource interface + // to avoid exposing them publicly. + // We want an interface so that BinaryLogger can fine tune its initialization logic + // in case the given event source is the replay source. On the other hand we don't want + // to expose these members publicly because they are not intended to be used by the consumers. + + private Action? _onRawReadingPossible; + private Action? _onStructuredReadingOnly; + /// + void IBinaryLogReplaySource.DeferredInitialize( Action onRawReadingPossible, Action onStructuredReadingOnly) { @@ -289,16 +305,9 @@ void IRawLogEventsSource.DeferredInitialize( this._onStructuredReadingOnly += onStructuredReadingOnly; } - public int FileFormatVersion => _fileFormatVersion ?? throw new InvalidOperationException(ResourceUtilities.GetResourceString("Binlog_Source_VersionUninitialized")); - public int MinimumReaderVersion => _minimumReaderVersion ?? throw new InvalidOperationException(ResourceUtilities.GetResourceString("Binlog_Source_VersionUninitialized")); - - private int? _fileFormatVersion; - private int? _minimumReaderVersion; - private Action? _onRawReadingPossible; - private Action? _onStructuredReadingOnly; private Action? _embeddedContentRead; - /// - event Action? IEmbeddedContentSource.EmbeddedContentRead + /// + event Action? IBinaryLogReplaySource.EmbeddedContentRead { // Explicitly implemented event has to declare explicit add/remove accessors // https://stackoverflow.com/a/2268472/2308106 @@ -323,8 +332,8 @@ event Action? IBuildEventArgsReaderNotifications.ArchiveFi } private Action? _rawLogRecordReceived; - /// - event Action? IRawLogEventsSource.RawLogRecordReceived + /// + event Action? IBinaryLogReplaySource.RawLogRecordReceived { add => _rawLogRecordReceived += value; remove => _rawLogRecordReceived -= value; diff --git a/src/Build/Logging/BinaryLogger/BinaryLogger.cs b/src/Build/Logging/BinaryLogger/BinaryLogger.cs index 9f4cd60f50a..a9e6f75705f 100644 --- a/src/Build/Logging/BinaryLogger/BinaryLogger.cs +++ b/src/Build/Logging/BinaryLogger/BinaryLogger.cs @@ -75,6 +75,7 @@ public sealed class BinaryLogger : ILogger // The current version of the binary log representation. // Changes with each update of the binary log format. internal const int FileFormatVersion = 18; + // The minimum version of the binary log reader that can read log of above version. // This should be changed only when the binary log format is changed in a way that would prevent it from being // read by older readers. (changing of the individual BuildEventArgs or adding new is fine - as reader can @@ -147,7 +148,7 @@ public void Initialize(IEventSource eventSource) bool logPropertiesAndItemsAfterEvaluation = Traits.Instance.EscapeHatches.LogPropertiesAndItemsAfterEvaluation ?? true; ProcessParameters(out bool omitInitialInfo); - var replayEventsSource = eventSource as IBinaryLogReplaySource; + var replayEventSource = eventSource as IBinaryLogReplaySource; try { @@ -169,7 +170,7 @@ public void Initialize(IEventSource eventSource) stream = new FileStream(FilePath, FileMode.Create); - if (CollectProjectImports != ProjectImportsCollectionMode.None && replayEventsSource == null) + if (CollectProjectImports != ProjectImportsCollectionMode.None && replayEventSource == null) { projectImportsCollector = new ProjectImportsCollector(FilePath, CollectProjectImports == ProjectImportsCollectionMode.ZipFile); } @@ -206,33 +207,33 @@ public void Initialize(IEventSource eventSource) eventArgsWriter.EmbedFile += EventArgsWriter_EmbedFile; } - if (replayEventsSource != null) + if (replayEventSource != null) { if (CollectProjectImports == ProjectImportsCollectionMode.Embed) { - replayEventsSource.EmbeddedContentRead += args => + replayEventSource.EmbeddedContentRead += args => eventArgsWriter.WriteBlob(args.ContentKind, args.ContentStream); } else if (CollectProjectImports == ProjectImportsCollectionMode.ZipFile) { - replayEventsSource.EmbeddedContentRead += args => + replayEventSource.EmbeddedContentRead += args => ProjectImportsCollector.FlushBlobToFile(FilePath, args.ContentStream); } // If raw events are provided - let's try to use the advantage. // But other subscribers can later on subscribe to structured events - // for this reason we do only subscribe delayed. - replayEventsSource.DeferredInitialize( + replayEventSource.DeferredInitialize( // For raw events we cannot write the initial info - as we cannot write // at the same time as raw events are being written - this would break the deduplicated strings store. // But we need to write the version info - but since we read/write raw - let's not change the version info. () => { - binaryWriter.Write(replayEventsSource.FileFormatVersion); - binaryWriter.Write(replayEventsSource.MinimumReaderVersion); - replayEventsSource.RawLogRecordReceived += RawEvents_LogDataSliceReceived; + binaryWriter.Write(replayEventSource.FileFormatVersion); + binaryWriter.Write(replayEventSource.MinimumReaderVersion); + replayEventSource.RawLogRecordReceived += RawEvents_LogDataSliceReceived; // Replay separated strings here as well (and do not deduplicate! It would skew string indexes) - replayEventsSource.StringReadDone += strArg => eventArgsWriter.WriteStringRecord(strArg.StringToBeUsed); + replayEventSource.StringReadDone += strArg => eventArgsWriter.WriteStringRecord(strArg.StringToBeUsed); }, SubscribeToStructuredEvents); } diff --git a/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs b/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs index 9e5bee6d8c0..d33f3769fe5 100644 --- a/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs +++ b/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs @@ -145,7 +145,7 @@ public void Dispose() internal int FileFormatVersion => _fileFormatVersion; internal int MinimumReaderVersion { get; set; } = BinaryLogger.ForwardCompatibilityMinimalVersion; - /// + /// internal event Action? EmbeddedContentRead; /// @@ -261,7 +261,7 @@ string ErrorFactory() => ? " " + ResourceUtilities.GetResourceString("Binlog_ReaderSkippingRecord") : string.Empty); - HandleError(ErrorFactory, _skipUnknownEvents, ReaderErrorType.UnkownEventType, recordKind); + HandleError(ErrorFactory, _skipUnknownEvents, ReaderErrorType.UnknownEventType, recordKind); } if (_readStream.BytesCountAllowedToReadRemaining > 0) diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/IEmbeddedContentSource.cs b/src/Build/Logging/BinaryLogger/Postprocessing/IEmbeddedContentSource.cs deleted file mode 100644 index 02c31a80f8a..00000000000 --- a/src/Build/Logging/BinaryLogger/Postprocessing/IEmbeddedContentSource.cs +++ /dev/null @@ -1,17 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; - -namespace Microsoft.Build.Logging -{ - internal interface IEmbeddedContentSource - { - /// - /// Raised when the log reader encounters a project import archive (embedded content) in the stream. - /// The subscriber must read the exactly given length of binary data from the stream - otherwise exception is raised. - /// If no subscriber is attached, the data is skipped. - /// - event Action EmbeddedContentRead; - } -} diff --git a/src/Build/Logging/BinaryLogger/Postprocessing/ReaderErrorType.cs b/src/Build/Logging/BinaryLogger/Postprocessing/ReaderErrorType.cs index 80bc87c29b1..5bce4a50935 100644 --- a/src/Build/Logging/BinaryLogger/Postprocessing/ReaderErrorType.cs +++ b/src/Build/Logging/BinaryLogger/Postprocessing/ReaderErrorType.cs @@ -11,7 +11,7 @@ public enum ReaderErrorType /// /// The encountered event is completely unknown to the reader. It cannot interpret any part of it. /// - UnkownEventType, + UnknownEventType, /// /// The encountered event is known to the reader and reader is able to read the event as it knows it. From fce1a779e87aa2592b2457d3c7b296992592ec4c Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Thu, 21 Dec 2023 13:16:28 +0100 Subject: [PATCH 46/49] Reintroduce suppressions for the binlog forward compatible reading API --- src/Build/CompatibilitySuppressions.xml | 263 ++++++++++++++++++++++++ 1 file changed, 263 insertions(+) create mode 100644 src/Build/CompatibilitySuppressions.xml diff --git a/src/Build/CompatibilitySuppressions.xml b/src/Build/CompatibilitySuppressions.xml new file mode 100644 index 00000000000..8497df99fb3 --- /dev/null +++ b/src/Build/CompatibilitySuppressions.xml @@ -0,0 +1,263 @@ + + + + + CP0001 + T:Microsoft.Build.Logging.IBuildEventStringsReader + lib/net472/Microsoft.Build.dll + lib/net472/Microsoft.Build.dll + true + + + CP0001 + T:Microsoft.Build.Logging.IBuildEventStringsReader + lib/net8.0/Microsoft.Build.dll + lib/net8.0/Microsoft.Build.dll + true + + + CP0001 + T:Microsoft.Build.Logging.IBuildEventStringsReader + ref/net472/Microsoft.Build.dll + ref/net472/Microsoft.Build.dll + true + + + CP0001 + T:Microsoft.Build.Logging.IBuildEventStringsReader + ref/net8.0/Microsoft.Build.dll + ref/net8.0/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.BinaryLogReplayEventSource.add_NotificationsSourceCreated(System.Action{Microsoft.Build.Logging.IBuildEventArgsReaderNotifications}) + lib/net472/Microsoft.Build.dll + lib/net472/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.BinaryLogReplayEventSource.remove_NotificationsSourceCreated(System.Action{Microsoft.Build.Logging.IBuildEventArgsReaderNotifications}) + lib/net472/Microsoft.Build.dll + lib/net472/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.BuildEventArgsReader.add_StringEncountered(System.Action) + lib/net472/Microsoft.Build.dll + lib/net472/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.BuildEventArgsReader.remove_StringEncountered(System.Action) + lib/net472/Microsoft.Build.dll + lib/net472/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.BinaryLogReplayEventSource.add_NotificationsSourceCreated(System.Action{Microsoft.Build.Logging.IBuildEventArgsReaderNotifications}) + lib/net8.0/Microsoft.Build.dll + lib/net8.0/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.BinaryLogReplayEventSource.remove_NotificationsSourceCreated(System.Action{Microsoft.Build.Logging.IBuildEventArgsReaderNotifications}) + lib/net8.0/Microsoft.Build.dll + lib/net8.0/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.BuildEventArgsReader.add_StringEncountered(System.Action) + lib/net8.0/Microsoft.Build.dll + lib/net8.0/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.BuildEventArgsReader.remove_StringEncountered(System.Action) + lib/net8.0/Microsoft.Build.dll + lib/net8.0/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.BinaryLogReplayEventSource.add_NotificationsSourceCreated(System.Action{Microsoft.Build.Logging.IBuildEventArgsReaderNotifications}) + ref/net472/Microsoft.Build.dll + ref/net472/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.BinaryLogReplayEventSource.remove_NotificationsSourceCreated(System.Action{Microsoft.Build.Logging.IBuildEventArgsReaderNotifications}) + ref/net472/Microsoft.Build.dll + ref/net472/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.BuildEventArgsReader.add_StringEncountered(System.Action) + ref/net472/Microsoft.Build.dll + ref/net472/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.BuildEventArgsReader.remove_StringEncountered(System.Action) + ref/net472/Microsoft.Build.dll + ref/net472/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.BinaryLogReplayEventSource.add_NotificationsSourceCreated(System.Action{Microsoft.Build.Logging.IBuildEventArgsReaderNotifications}) + ref/net8.0/Microsoft.Build.dll + ref/net8.0/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.BinaryLogReplayEventSource.remove_NotificationsSourceCreated(System.Action{Microsoft.Build.Logging.IBuildEventArgsReaderNotifications}) + ref/net8.0/Microsoft.Build.dll + ref/net8.0/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.BuildEventArgsReader.add_StringEncountered(System.Action) + ref/net8.0/Microsoft.Build.dll + ref/net8.0/Microsoft.Build.dll + true + + + CP0002 + M:Microsoft.Build.Logging.BuildEventArgsReader.remove_StringEncountered(System.Action) + ref/net8.0/Microsoft.Build.dll + ref/net8.0/Microsoft.Build.dll + true + + + CP0006 + E:Microsoft.Build.Logging.IBuildEventArgsReaderNotifications.ArchiveFileEncountered + lib/net472/Microsoft.Build.dll + lib/net472/Microsoft.Build.dll + true + + + CP0006 + E:Microsoft.Build.Logging.IBuildEventArgsReaderNotifications.RecoverableReadError + lib/net472/Microsoft.Build.dll + lib/net472/Microsoft.Build.dll + true + + + CP0006 + E:Microsoft.Build.Logging.IBuildEventArgsReaderNotifications.StringReadDone + lib/net472/Microsoft.Build.dll + lib/net472/Microsoft.Build.dll + true + + + CP0006 + E:Microsoft.Build.Logging.IBuildEventArgsReaderNotifications.ArchiveFileEncountered + lib/net8.0/Microsoft.Build.dll + lib/net8.0/Microsoft.Build.dll + true + + + CP0006 + E:Microsoft.Build.Logging.IBuildEventArgsReaderNotifications.RecoverableReadError + lib/net8.0/Microsoft.Build.dll + lib/net8.0/Microsoft.Build.dll + true + + + CP0006 + E:Microsoft.Build.Logging.IBuildEventArgsReaderNotifications.StringReadDone + lib/net8.0/Microsoft.Build.dll + lib/net8.0/Microsoft.Build.dll + true + + + CP0006 + E:Microsoft.Build.Logging.IBuildEventArgsReaderNotifications.ArchiveFileEncountered + ref/net8.0/Microsoft.Build.dll + ref/net8.0/Microsoft.Build.dll + true + + + CP0006 + E:Microsoft.Build.Logging.IBuildEventArgsReaderNotifications.RecoverableReadError + ref/net8.0/Microsoft.Build.dll + ref/net8.0/Microsoft.Build.dll + true + + + CP0006 + E:Microsoft.Build.Logging.IBuildEventArgsReaderNotifications.StringReadDone + ref/net8.0/Microsoft.Build.dll + ref/net8.0/Microsoft.Build.dll + true + + + CP0008 + T:Microsoft.Build.Logging.BuildEventArgsReader + lib/net472/Microsoft.Build.dll + lib/net472/Microsoft.Build.dll + true + + + CP0008 + T:Microsoft.Build.Logging.IBuildEventArgsReaderNotifications + lib/net472/Microsoft.Build.dll + lib/net472/Microsoft.Build.dll + true + + + CP0008 + T:Microsoft.Build.Logging.BuildEventArgsReader + lib/net8.0/Microsoft.Build.dll + lib/net8.0/Microsoft.Build.dll + true + + + CP0008 + T:Microsoft.Build.Logging.IBuildEventArgsReaderNotifications + lib/net8.0/Microsoft.Build.dll + lib/net8.0/Microsoft.Build.dll + true + + + CP0008 + T:Microsoft.Build.Logging.BuildEventArgsReader + ref/net472/Microsoft.Build.dll + ref/net472/Microsoft.Build.dll + true + + + CP0008 + T:Microsoft.Build.Logging.IBuildEventArgsReaderNotifications + ref/net472/Microsoft.Build.dll + ref/net472/Microsoft.Build.dll + true + + + CP0008 + T:Microsoft.Build.Logging.BuildEventArgsReader + ref/net8.0/Microsoft.Build.dll + ref/net8.0/Microsoft.Build.dll + true + + + CP0008 + T:Microsoft.Build.Logging.IBuildEventArgsReaderNotifications + ref/net8.0/Microsoft.Build.dll + ref/net8.0/Microsoft.Build.dll + true + + \ No newline at end of file From b97a0806302945a2413b1f57c58127baab56d76d Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Wed, 27 Dec 2023 21:25:13 +0100 Subject: [PATCH 47/49] Fix raw replay and redact perf --- src/Build/Logging/BinaryLogger/BuildEventArgsWriter.cs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/Build/Logging/BinaryLogger/BuildEventArgsWriter.cs b/src/Build/Logging/BinaryLogger/BuildEventArgsWriter.cs index c3514bd62f5..8a8311e1fd6 100644 --- a/src/Build/Logging/BinaryLogger/BuildEventArgsWriter.cs +++ b/src/Build/Logging/BinaryLogger/BuildEventArgsWriter.cs @@ -260,7 +260,7 @@ public void WriteBlob(BinaryLogRecordKind kind, Stream stream) Write(kind); Write((int)stream.Length); - Write(stream); + WriteToOriginalStream(stream); } /// @@ -1137,9 +1137,12 @@ private void Write(byte[] bytes) binaryWriter.Write(bytes); } - private void Write(Stream stream) + private void WriteToOriginalStream(Stream stream) { - stream.CopyTo(binaryWriter.BaseStream); + // WARNING: avoid calling binaryWriter.BaseStream here + // as it will flush the underlying stream - since that is a + // BufferedStream it will make buffering nearly useless + stream.CopyTo(originalStream); } private void Write(byte b) From ad4bf54536376a555bf2dbd842a5a23efdaf5f27 Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Tue, 9 Jan 2024 19:04:05 +0100 Subject: [PATCH 48/49] Forward compatible reading is off by default --- src/Build/Logging/BinaryLogger/BinaryLogReplayEventSource.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Build/Logging/BinaryLogger/BinaryLogReplayEventSource.cs b/src/Build/Logging/BinaryLogger/BinaryLogReplayEventSource.cs index ec70649d965..abf9e4e80c9 100644 --- a/src/Build/Logging/BinaryLogger/BinaryLogReplayEventSource.cs +++ b/src/Build/Logging/BinaryLogger/BinaryLogReplayEventSource.cs @@ -155,7 +155,7 @@ public static BinaryReader OpenReader(Stream sourceFileStream) public static BuildEventArgsReader OpenBuildEventsReader( BinaryReader binaryReader, bool closeInput, - bool allowForwardCompatibility = true) + bool allowForwardCompatibility = false) { int fileFormatVersion = binaryReader.ReadInt32(); // Is this the new log format that contains the minimum reader version? From 38f57916d463906d36be4302e4950b22156ea44c Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Thu, 11 Jan 2024 12:19:29 +0100 Subject: [PATCH 49/49] Bugfix the reading of old version of logs --- .../BinaryLogger/BuildEventArgsReader.cs | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs b/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs index cdbaa6fcc1b..2c49c17c8a7 100644 --- a/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs +++ b/src/Build/Logging/BinaryLogger/BuildEventArgsReader.cs @@ -653,12 +653,16 @@ private BuildEventArgs ReadProjectEvaluationFinishedEventArgs() if (_fileFormatVersion >= 12) { - if (_fileFormatVersion < BinaryLogger.ForwardCompatibilityMinimalVersion) + IEnumerable? globalProperties = null; + // In newer versions, we store the global properties always, as it handles + // null and empty within WriteProperties already. + // This saves a single boolean, but mainly doesn't hide the difference between null and empty + // during write->read roundtrip. + if (_fileFormatVersion >= BinaryLogger.ForwardCompatibilityMinimalVersion || + ReadBoolean()) { - // Throw away, but need to advance past it - ReadBoolean(); + globalProperties = ReadStringDictionary(); } - IEnumerable? globalProperties = ReadStringDictionary(); var propertyList = ReadPropertyList(); var itemList = ReadProjectItems(); @@ -709,12 +713,12 @@ private BuildEventArgs ReadProjectStartedEventArgs() if (_fileFormatVersion > 6) { - if (_fileFormatVersion < BinaryLogger.ForwardCompatibilityMinimalVersion) + // See ReadProjectEvaluationFinishedEventArgs for details on why we always store global properties in newer version. + if (_fileFormatVersion >= BinaryLogger.ForwardCompatibilityMinimalVersion || + ReadBoolean()) { - // Throw away, but need to advance past it - ReadBoolean(); + globalProperties = ReadStringDictionary(); } - globalProperties = ReadStringDictionary(); } var propertyList = ReadPropertyList();