diff --git a/docs/design/mono/webcil.md b/docs/design/mono/webcil.md new file mode 100644 index 0000000000000..3bcb7d6365353 --- /dev/null +++ b/docs/design/mono/webcil.md @@ -0,0 +1,111 @@ +# WebCIL assembly format + +## Version + +This is version 0.0 of the Webcil format. + +## Motivation + +When deploying the .NET runtime to the browser using WebAssembly, we have received some reports from +customers that certain users are unable to use their apps because firewalls and anti-virus software +may prevent browsers from downloading or caching assemblies with a .DLL extension and PE contents. + +This document defines a new container format for ECMA-335 assemblies +that uses the `.webcil` extension and uses a new WebCIL container +format. + + +## Specification + +As our starting point we take section II.25.1 "Structure of the +runtime file format" from ECMA-335 6th Edition. + +| | +|--------| +| PE Headers | +| CLI Header | +| CLI Data | +| Native Image Sections | +| | + + + +A Webcil file follows a similar structure + + +| | +|--------| +| Webcil Headers | +| CLI Header | +| CLI Data | +| | + +## Webcil Headers + +The Webcil headers consist of a Webcil header followed by a sequence of section headers. +(All multi-byte integers are in little endian format). + +### Webcil Header + +``` c +struct WebcilHeader { + uint8_t id[4]; // 'W' 'b' 'I' 'L' + // 4 bytes + uint16_t version_major; // 0 + uint16_t version_minor; // 0 + // 8 bytes + uint16_t coff_sections; + uint16_t reserved0; // 0 + // 12 bytes + + uint32_t pe_cli_header_rva; + uint32_t pe_cli_header_size; + // 20 bytes + + uint32_t pe_debug_rva; + uint32_t pe_debug_size; + // 28 bytes +}; +``` + +The Webcil header starts with the magic characters 'W' 'b' 'I' 'L' followed by the version in major +minor format (must be 0 and 0). Then a count of the section headers and two reserved bytes. + +The next pairs of integers are a subset of the PE Header data directory specifying the RVA and size +of the CLI header, as well as the directory entry for the PE debug directory. + + +### Section header table + +Immediately following the Webcil header is a sequence (whose length is given by `coff_sections` +above) of section headers giving their virtual address and virtual size, as well as the offset in +the Webcil file and the size in the file. This is a subset of the PE section header that includes +enough information to correctly interpret the RVAs from the webcil header and from the .NET +metadata. Other information (such as the section names) are not included. + +``` c +struct SectionHeader { + uint32_t st_virtual_size; + uint32_t st_virtual_address; + uint32_t st_raw_data_size; + uint32_t st_raw_data_ptr; +}; +``` + +### Sections + +Immediately following the section table are the sections. These are copied verbatim from the PE file. + +## Rationale + +The intention is to include only the information necessary for the runtime to locate the metadata +root, and to resolve the RVA references in the metadata (for locating data declarations and method IL). + +A goal is for the files not to be executable by .NET Framework. + +Unlike PE files, mixing native and managed code is not a goal. + +Lossless conversion from Webcil back to PE is not intended to be supported. The format is being +documented in order to support diagnostic tooling and utilities such as decompilers, disassemblers, +file identification utilities, dependency analyzers, etc. + diff --git a/src/libraries/Microsoft.NET.WebAssembly.Webcil/Directory.Build.props b/src/libraries/Microsoft.NET.WebAssembly.Webcil/Directory.Build.props new file mode 100644 index 0000000000000..6bda1e7da2544 --- /dev/null +++ b/src/libraries/Microsoft.NET.WebAssembly.Webcil/Directory.Build.props @@ -0,0 +1,11 @@ + + + + true + + false + + false + + diff --git a/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Common/IsExternalInit.cs b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Common/IsExternalInit.cs new file mode 100644 index 0000000000000..d7be691310347 --- /dev/null +++ b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Common/IsExternalInit.cs @@ -0,0 +1,7 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace System.Runtime.CompilerServices +{ + internal sealed class IsExternalInit { } +} diff --git a/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Microsoft.NET.WebAssembly.Webcil.csproj b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Microsoft.NET.WebAssembly.Webcil.csproj new file mode 100644 index 0000000000000..6c66737e5535d --- /dev/null +++ b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Microsoft.NET.WebAssembly.Webcil.csproj @@ -0,0 +1,24 @@ + + + $(NetCoreAppToolCurrent);$(NetFrameworkToolCurrent) + Abstractions for modifying .NET webcil binary images + true + true + true + false + + + + + + + + + + + + + + + + diff --git a/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/Internal/Constants.cs b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/Internal/Constants.cs new file mode 100644 index 0000000000000..2d486645d23b6 --- /dev/null +++ b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/Internal/Constants.cs @@ -0,0 +1,10 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.NET.WebAssembly.Webcil.Internal; + +internal static unsafe class Constants +{ + public const int WC_VERSION_MAJOR = 0; + public const int WC_VERSION_MINOR = 0; +} diff --git a/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebciHeader.cs b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebciHeader.cs new file mode 100644 index 0000000000000..33dcc85791fff --- /dev/null +++ b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebciHeader.cs @@ -0,0 +1,33 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Runtime.InteropServices; + +namespace Microsoft.NET.WebAssembly.Webcil; + +/// +/// The header of a WebCIL file. +/// +/// +/// +/// The header is a subset of the PE, COFF and CLI headers that are needed by the mono runtime to load managed assemblies. +/// +[StructLayout(LayoutKind.Sequential, Pack = 1)] +public unsafe struct WebcilHeader +{ + public fixed byte id[4]; // 'W' 'b' 'I' 'L' + // 4 bytes + public ushort version_major; // 0 + public ushort version_minor; // 0 + // 8 bytes + + public ushort coff_sections; + public ushort reserved0; // 0 + // 12 bytes + public uint pe_cli_header_rva; + public uint pe_cli_header_size; + // 20 bytes + public uint pe_debug_rva; + public uint pe_debug_size; + // 28 bytes +} diff --git a/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilConverter.cs b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilConverter.cs new file mode 100644 index 0000000000000..421b62439e190 --- /dev/null +++ b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilConverter.cs @@ -0,0 +1,342 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; +using System.Collections.Immutable; +using System.Reflection.PortableExecutable; +using System.Runtime.InteropServices; + +namespace Microsoft.NET.WebAssembly.Webcil; + +/// +/// Reads a .NET assembly in a normal PE COFF file and writes it out as a Webcil file +/// +public class WebcilConverter +{ + + // Interesting stuff we've learned about the input PE file + public record PEFileInfo( + // The sections in the PE file + ImmutableArray SectionHeaders, + // The location of the debug directory entries + DirectoryEntry DebugTableDirectory, + // The file offset of the sections, following the section directory + FilePosition SectionStart, + // The debug directory entries + ImmutableArray DebugDirectoryEntries + ); + + // Intersting stuff we know about the webcil file we're writing + public record WCFileInfo( + // The header of the webcil file + WebcilHeader Header, + // The section directory of the webcil file + ImmutableArray SectionHeaders, + // The file offset of the sections, following the section directory + FilePosition SectionStart + ); + + private readonly string _inputPath; + private readonly string _outputPath; + + private string InputPath => _inputPath; + + private WebcilConverter(string inputPath, string outputPath) + { + _inputPath = inputPath; + _outputPath = outputPath; + } + + public static WebcilConverter FromPortableExecutable(string inputPath, string outputPath) + => new WebcilConverter(inputPath, outputPath); + + public void ConvertToWebcil() + { + using var inputStream = File.Open(_inputPath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); + PEFileInfo peInfo; + WCFileInfo wcInfo; + using (var peReader = new PEReader(inputStream, PEStreamOptions.LeaveOpen)) + { + GatherInfo(peReader, out wcInfo, out peInfo); + } + + using var outputStream = File.Open(_outputPath, FileMode.Create, FileAccess.Write); + WriteHeader(outputStream, wcInfo.Header); + WriteSectionHeaders(outputStream, wcInfo.SectionHeaders); + CopySections(outputStream, inputStream, peInfo.SectionHeaders); + if (wcInfo.Header.pe_debug_size != 0 && wcInfo.Header.pe_debug_rva != 0) + { + var wcDebugDirectoryEntries = FixupDebugDirectoryEntries(peInfo, wcInfo); + OverwriteDebugDirectoryEntries(outputStream, wcInfo, wcDebugDirectoryEntries); + } + } + + public record struct FilePosition(int Position) + { + public static implicit operator FilePosition(int position) => new(position); + + public static FilePosition operator +(FilePosition left, int right) => new(left.Position + right); + } + + private static unsafe int SizeOfHeader() + { + return sizeof(WebcilHeader); + } + + public unsafe void GatherInfo(PEReader peReader, out WCFileInfo wcInfo, out PEFileInfo peInfo) + { + var headers = peReader.PEHeaders; + var peHeader = headers.PEHeader!; + var coffHeader = headers.CoffHeader!; + var sections = headers.SectionHeaders; + WebcilHeader header; + header.id[0] = (byte)'W'; + header.id[1] = (byte)'b'; + header.id[2] = (byte)'I'; + header.id[3] = (byte)'L'; + header.version_major = Internal.Constants.WC_VERSION_MAJOR; + header.version_minor = Internal.Constants.WC_VERSION_MINOR; + header.coff_sections = (ushort)coffHeader.NumberOfSections; + header.reserved0 = 0; + header.pe_cli_header_rva = (uint)peHeader.CorHeaderTableDirectory.RelativeVirtualAddress; + header.pe_cli_header_size = (uint)peHeader.CorHeaderTableDirectory.Size; + header.pe_debug_rva = (uint)peHeader.DebugTableDirectory.RelativeVirtualAddress; + header.pe_debug_size = (uint)peHeader.DebugTableDirectory.Size; + + // current logical position in the output file + FilePosition pos = SizeOfHeader(); + // position of the current section in the output file + // initially it's after all the section headers + FilePosition curSectionPos = pos + sizeof(WebcilSectionHeader) * coffHeader.NumberOfSections; + // The first WC section is immediately after the section directory + FilePosition firstWCSection = curSectionPos; + + FilePosition firstPESection = 0; + + ImmutableArray.Builder headerBuilder = ImmutableArray.CreateBuilder(coffHeader.NumberOfSections); + foreach (var sectionHeader in sections) + { + // The first section is the one with the lowest file offset + if (firstPESection.Position == 0) + { + firstPESection = sectionHeader.PointerToRawData; + } + else + { + firstPESection = Math.Min(firstPESection.Position, sectionHeader.PointerToRawData); + } + + var newHeader = new WebcilSectionHeader + ( + virtualSize: sectionHeader.VirtualSize, + virtualAddress: sectionHeader.VirtualAddress, + sizeOfRawData: sectionHeader.SizeOfRawData, + pointerToRawData: curSectionPos.Position + ); + + pos += sizeof(WebcilSectionHeader); + curSectionPos += sectionHeader.SizeOfRawData; + headerBuilder.Add(newHeader); + } + + ImmutableArray debugDirectoryEntries = peReader.ReadDebugDirectory(); + + peInfo = new PEFileInfo(SectionHeaders: sections, + DebugTableDirectory: peHeader.DebugTableDirectory, + SectionStart: firstPESection, + DebugDirectoryEntries: debugDirectoryEntries); + + wcInfo = new WCFileInfo(Header: header, + SectionHeaders: headerBuilder.MoveToImmutable(), + SectionStart: firstWCSection); + } + + private static void WriteHeader(Stream s, WebcilHeader header) + { + WriteStructure(s, header); + } + + private static void WriteSectionHeaders(Stream s, ImmutableArray sectionsHeaders) + { + // FIXME: fixup endianness + if (!BitConverter.IsLittleEndian) + throw new NotImplementedException(); + foreach (var sectionHeader in sectionsHeaders) + { + WriteSectionHeader(s, sectionHeader); + } + } + + private static void WriteSectionHeader(Stream s, WebcilSectionHeader sectionHeader) + { + WriteStructure(s, sectionHeader); + } + +#if NETCOREAPP2_1_OR_GREATER + private static void WriteStructure(Stream s, T structure) + where T : unmanaged + { + // FIXME: fixup endianness + if (!BitConverter.IsLittleEndian) + throw new NotImplementedException(); + unsafe + { + byte* p = (byte*)&structure; + s.Write(new ReadOnlySpan(p, sizeof(T))); + } + } +#else + private static void WriteStructure(Stream s, T structure) + where T : unmanaged + { + // FIXME: fixup endianness + if (!BitConverter.IsLittleEndian) + throw new NotImplementedException(); + int size = Marshal.SizeOf(); + byte[] buffer = new byte[size]; + IntPtr ptr = IntPtr.Zero; + try + { + ptr = Marshal.AllocHGlobal(size); + Marshal.StructureToPtr(structure, ptr, false); + Marshal.Copy(ptr, buffer, 0, size); + } + finally + { + Marshal.FreeHGlobal(ptr); + } + s.Write(buffer, 0, size); + } +#endif + + private static void CopySections(Stream outStream, Stream inputStream, ImmutableArray peSections) + { + // endianness: ok, we're just copying from one stream to another + foreach (var peHeader in peSections) + { + var buffer = new byte[peHeader.SizeOfRawData]; + inputStream.Seek(peHeader.PointerToRawData, SeekOrigin.Begin); + ReadExactly(inputStream, buffer); + outStream.Write(buffer, 0, buffer.Length); + } + } + +#if NETCOREAPP2_1_OR_GREATER + private static void ReadExactly(Stream s, Span buffer) + { + s.ReadExactly(buffer); + } +#else + private static void ReadExactly(Stream s, byte[] buffer) + { + int offset = 0; + while (offset < buffer.Length) + { + int read = s.Read(buffer, offset, buffer.Length - offset); + if (read == 0) + throw new EndOfStreamException(); + offset += read; + } + } +#endif + + private static FilePosition GetPositionOfRelativeVirtualAddress(ImmutableArray wcSections, uint relativeVirtualAddress) + { + foreach (var section in wcSections) + { + if (relativeVirtualAddress >= section.VirtualAddress && relativeVirtualAddress < section.VirtualAddress + section.VirtualSize) + { + FilePosition pos = section.PointerToRawData + ((int)relativeVirtualAddress - section.VirtualAddress); + return pos; + } + } + + throw new InvalidOperationException("relative virtual address not in any section"); + } + + // Given a physical file offset, return the section and the offset within the section. + private (WebcilSectionHeader section, int offset) GetSectionFromFileOffset(ImmutableArray peSections, FilePosition fileOffset) + { + foreach (var section in peSections) + { + if (fileOffset.Position >= section.PointerToRawData && fileOffset.Position < section.PointerToRawData + section.SizeOfRawData) + { + return (section, fileOffset.Position - section.PointerToRawData); + } + } + + throw new InvalidOperationException($"file offset not in any section (Webcil) for {InputPath}"); + } + + private void GetSectionFromFileOffset(ImmutableArray sections, FilePosition fileOffset) + { + foreach (var section in sections) + { + if (fileOffset.Position >= section.PointerToRawData && fileOffset.Position < section.PointerToRawData + section.SizeOfRawData) + { + return; + } + } + + throw new InvalidOperationException($"file offset {fileOffset.Position} not in any section (PE) for {InputPath}"); + } + + // Make a new set of debug directory entries that + // have their data pointers adjusted to be relative to the start of the webcil file. + // This is necessary because the debug directory entires in the PE file are relative to the start of the PE file, + // and a PE header is bigger than a webcil header. + private ImmutableArray FixupDebugDirectoryEntries(PEFileInfo peInfo, WCFileInfo wcInfo) + { + int dataPointerAdjustment = peInfo.SectionStart.Position - wcInfo.SectionStart.Position; + ImmutableArray entries = peInfo.DebugDirectoryEntries; + ImmutableArray.Builder newEntries = ImmutableArray.CreateBuilder(entries.Length); + foreach (var entry in entries) + { + DebugDirectoryEntry newEntry; + if (entry.Type == DebugDirectoryEntryType.Reproducible || entry.DataPointer == 0 || entry.DataSize == 0) + { + // this entry doesn't have an associated data pointer, so just copy it + newEntry = entry; + } + else + { + // the "DataPointer" field is a file offset in the PE file, adjust the entry wit the corresponding offset in the Webcil file + var newDataPointer = entry.DataPointer - dataPointerAdjustment; + newEntry = new DebugDirectoryEntry(entry.Stamp, entry.MajorVersion, entry.MinorVersion, entry.Type, entry.DataSize, entry.DataRelativeVirtualAddress, newDataPointer); + GetSectionFromFileOffset(peInfo.SectionHeaders, entry.DataPointer); + // validate that the new entry is in some section + GetSectionFromFileOffset(wcInfo.SectionHeaders, newDataPointer); + } + newEntries.Add(newEntry); + } + return newEntries.MoveToImmutable(); + } + + private static void OverwriteDebugDirectoryEntries(Stream s, WCFileInfo wcInfo, ImmutableArray entries) + { + FilePosition debugDirectoryPos = GetPositionOfRelativeVirtualAddress(wcInfo.SectionHeaders, wcInfo.Header.pe_debug_rva); + using var writer = new BinaryWriter(s, System.Text.Encoding.UTF8, leaveOpen: true); + writer.Seek(debugDirectoryPos.Position, SeekOrigin.Begin); + foreach (var entry in entries) + { + WriteDebugDirectoryEntry(writer, entry); + } + // TODO check that we overwrite with the same size as the original + + // restore the stream position + writer.Seek(0, SeekOrigin.End); + } + + private static void WriteDebugDirectoryEntry(BinaryWriter writer, DebugDirectoryEntry entry) + { + writer.Write((uint)0); // Characteristics + writer.Write(entry.Stamp); + writer.Write(entry.MajorVersion); + writer.Write(entry.MinorVersion); + writer.Write((uint)entry.Type); + writer.Write(entry.DataSize); + writer.Write(entry.DataRelativeVirtualAddress); + writer.Write(entry.DataPointer); + } +} diff --git a/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilReader.cs b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilReader.cs new file mode 100644 index 0000000000000..6782ebf4a5aae --- /dev/null +++ b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilReader.cs @@ -0,0 +1,342 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Immutable; +using System.IO; +using System.Reflection; +using System.Runtime.InteropServices; + +using System.Reflection.Metadata; +using System.Reflection.PortableExecutable; + +namespace Microsoft.NET.WebAssembly.Webcil; + + +public sealed class WebcilReader : IDisposable +{ + // WISH: + // This should be implemented in terms of System.Reflection.Internal.MemoryBlockProvider like the PEReader, + // but the memory block classes are internal to S.R.M. + + private readonly Stream _stream; + private WebcilHeader _header; + private DirectoryEntry _corHeaderMetadataDirectory; + private MetadataReaderProvider? _metadataReaderProvider; + private ImmutableArray? _sections; + + private string? InputPath { get; } + + public WebcilReader(Stream stream) + { + this._stream = stream; + if (!stream.CanRead || !stream.CanSeek) + { + throw new ArgumentException("Stream must be readable and seekable", nameof(stream)); + } + if (!ReadHeader()) + { + throw new BadImageFormatException("Stream does not contain a valid Webcil file", nameof(stream)); + } + if (!ReadCorHeader()) + { + throw new BadImageFormatException("Stream does not contain a valid COR header in the Webcil file", nameof(stream)); + } + } + + public WebcilReader (Stream stream, string inputPath) : this(stream) + { + InputPath = inputPath; + } + + private unsafe bool ReadHeader() + { + WebcilHeader header; + var buffer = new byte[Marshal.SizeOf()]; + if (_stream.Read(buffer, 0, buffer.Length) != buffer.Length) + { + return false; + } + if (!BitConverter.IsLittleEndian) + { + throw new NotImplementedException("TODO: implement big endian support"); + } + fixed (byte* p = buffer) + { + header = *(WebcilHeader*)p; + } + if (header.id[0] != 'W' || header.id[1] != 'b' + || header.id[2] != 'I' || header.id[3] != 'L' + || header.version_major != Internal.Constants.WC_VERSION_MAJOR + || header.version_minor != Internal.Constants.WC_VERSION_MINOR) + { + return false; + } + _header = header; + return true; + } + + private unsafe bool ReadCorHeader() + { + // we can't construct CorHeader because it's constructor is internal + // but we don't care, really, we only want the metadata directory entry + var pos = TranslateRVA(_header.pe_cli_header_rva); + if (_stream.Seek(pos, SeekOrigin.Begin) != pos) + { + return false; + } + using var reader = new BinaryReader(_stream, System.Text.Encoding.UTF8, leaveOpen: true); + reader.ReadInt32(); // byte count + reader.ReadUInt16(); // major version + reader.ReadUInt16(); // minor version + _corHeaderMetadataDirectory = new DirectoryEntry(reader.ReadInt32(), reader.ReadInt32()); + return true; + } + + public MetadataReaderProvider GetMetadataReaderProvider() + { + // FIXME threading + if (_metadataReaderProvider == null) + { + long pos = TranslateRVA((uint)_corHeaderMetadataDirectory.RelativeVirtualAddress); + if (_stream.Seek(pos, SeekOrigin.Begin) != pos) + { + throw new BadImageFormatException("Could not seek to metadata in ", InputPath); + } + _metadataReaderProvider = MetadataReaderProvider.FromMetadataStream(_stream, MetadataStreamOptions.LeaveOpen); + } + return _metadataReaderProvider; + } + + public MetadataReader GetMetadataReader() => GetMetadataReaderProvider().GetMetadataReader(); + + public ImmutableArray ReadDebugDirectory() + { + var debugRVA = _header.pe_debug_rva; + if (debugRVA == 0) + { + return ImmutableArray.Empty; + } + var debugSize = _header.pe_debug_size; + if (debugSize == 0) + { + return ImmutableArray.Empty; + } + var debugOffset = TranslateRVA(debugRVA); + _stream.Seek(debugOffset, SeekOrigin.Begin); + var buffer = new byte[debugSize]; + if (_stream.Read(buffer, 0, buffer.Length) != buffer.Length) + { + throw new BadImageFormatException("Could not read debug directory", InputPath); + } + unsafe + { + fixed (byte* p = buffer) + { + return ReadDebugDirectoryEntries(new BlobReader(p, buffer.Length)); + } + } + } + + // FIXME: copied from DebugDirectoryEntry.Size + internal const int DebugDirectoryEntrySize = + sizeof(uint) + // Characteristics + sizeof(uint) + // TimeDataStamp + sizeof(uint) + // Version + sizeof(uint) + // Type + sizeof(uint) + // SizeOfData + sizeof(uint) + // AddressOfRawData + sizeof(uint); // PointerToRawData + + + // FIXME: copy-pasted from PEReader + private static ImmutableArray ReadDebugDirectoryEntries(BlobReader reader) + { + int entryCount = reader.Length / DebugDirectoryEntrySize; + var builder = ImmutableArray.CreateBuilder(entryCount); + for (int i = 0; i < entryCount; i++) + { + // Reserved, must be zero. + int characteristics = reader.ReadInt32(); + if (characteristics != 0) + { + throw new BadImageFormatException(); + } + + uint stamp = reader.ReadUInt32(); + ushort majorVersion = reader.ReadUInt16(); + ushort minorVersion = reader.ReadUInt16(); + + var type = (DebugDirectoryEntryType)reader.ReadInt32(); + + int dataSize = reader.ReadInt32(); + int dataRva = reader.ReadInt32(); + int dataPointer = reader.ReadInt32(); + + builder.Add(new DebugDirectoryEntry(stamp, majorVersion, minorVersion, type, dataSize, dataRva, dataPointer)); + } + + return builder.MoveToImmutable(); + } + + public CodeViewDebugDirectoryData ReadCodeViewDebugDirectoryData(DebugDirectoryEntry entry) + { + var pos = entry.DataPointer; + var buffer = new byte[entry.DataSize]; + if (_stream.Seek(pos, SeekOrigin.Begin) != pos) + { + throw new BadImageFormatException("Could not seek to CodeView debug directory data", nameof(_stream)); + } + if (_stream.Read(buffer, 0, buffer.Length) != buffer.Length) + { + throw new BadImageFormatException("Could not read CodeView debug directory data", nameof(_stream)); + } + unsafe + { + fixed (byte* p = buffer) + { + return DecodeCodeViewDebugDirectoryData(new BlobReader(p, buffer.Length)); + } + } + } + + private static CodeViewDebugDirectoryData DecodeCodeViewDebugDirectoryData(BlobReader reader) + { + // FIXME: copy-pasted from PEReader.DecodeCodeViewDebugDirectoryData + + if (reader.ReadByte() != (byte)'R' || + reader.ReadByte() != (byte)'S' || + reader.ReadByte() != (byte)'D' || + reader.ReadByte() != (byte)'S') + { + throw new BadImageFormatException("Unexpected CodeView data signature"); + } + + Guid guid = reader.ReadGuid(); + int age = reader.ReadInt32(); + string path = ReadUtf8NullTerminated(reader)!; + + return MakeCodeViewDebugDirectoryData(guid, age, path); + } + + private static string? ReadUtf8NullTerminated(BlobReader reader) + { + var mi = typeof(BlobReader).GetMethod("ReadUtf8NullTerminated", BindingFlags.NonPublic | BindingFlags.Instance); + if (mi == null) + { + throw new InvalidOperationException("Could not find BlobReader.ReadUtf8NullTerminated"); + } + return (string?)mi.Invoke(reader, null); + } + + private static CodeViewDebugDirectoryData MakeCodeViewDebugDirectoryData(Guid guid, int age, string path) + { + var types = new Type[] { typeof(Guid), typeof(int), typeof(string) }; + var mi = typeof(CodeViewDebugDirectoryData).GetConstructor(BindingFlags.NonPublic | BindingFlags.Instance, null, types, null); + if (mi == null) + { + throw new InvalidOperationException("Could not find CodeViewDebugDirectoryData constructor"); + } + return (CodeViewDebugDirectoryData)mi.Invoke(new object[] { guid, age, path }); + } + + public MetadataReaderProvider ReadEmbeddedPortablePdbDebugDirectoryData(DebugDirectoryEntry entry) + { + var pos = entry.DataPointer; + var buffer = new byte[entry.DataSize]; + if (_stream.Seek(pos, SeekOrigin.Begin) != pos) + { + throw new BadImageFormatException("Could not seek to Embedded Portable PDB debug directory data", nameof(_stream)); + } + if (_stream.Read(buffer, 0, buffer.Length) != buffer.Length) + { + throw new BadImageFormatException("Could not read Embedded Portable PDB debug directory data", nameof(_stream)); + } + unsafe + { + fixed (byte* p = buffer) + { + return DecodeEmbeddedPortablePdbDirectoryData(new BlobReader(p, buffer.Length)); + } + } + } + + private const uint PortablePdbVersions_DebugDirectoryEmbeddedSignature = 0x4244504d; + private static MetadataReaderProvider DecodeEmbeddedPortablePdbDirectoryData(BlobReader reader) + { + // FIXME: inspired by PEReader.DecodeEmbeddedPortablePdbDebugDirectoryData + // but not using its internal utility classes. + + if (reader.ReadUInt32() != PortablePdbVersions_DebugDirectoryEmbeddedSignature) + { + throw new BadImageFormatException("Unexpected embedded portable PDB data signature"); + } + + int decompressedSize = reader.ReadInt32(); + + byte[] decompressedBuffer; + + byte[] compressedBuffer = reader.ReadBytes(reader.RemainingBytes); + + using (var compressedStream = new MemoryStream(compressedBuffer, writable: false)) + using (var deflateStream = new System.IO.Compression.DeflateStream(compressedStream, System.IO.Compression.CompressionMode.Decompress, leaveOpen: true)) + { +#if NETCOREAPP1_1_OR_GREATER + decompressedBuffer = GC.AllocateUninitializedArray(decompressedSize); +#else + decompressedBuffer = new byte[decompressedSize]; +#endif + using (var decompressedStream = new MemoryStream(decompressedBuffer, writable: true)) + { + deflateStream.CopyTo(decompressedStream); + } + } + + + return MetadataReaderProvider.FromPortablePdbStream(new MemoryStream(decompressedBuffer, writable: false)); + + } + + private long TranslateRVA(uint rva) + { + if (_sections == null) + { + _sections = ReadSections(); + } + foreach (var section in _sections.Value) + { + if (rva >= section.VirtualAddress && rva < section.VirtualAddress + section.VirtualSize) + { + return section.PointerToRawData + (rva - section.VirtualAddress); + } + } + throw new BadImageFormatException("RVA not found in any section", nameof(_stream)); + } + + private static long SectionDirectoryOffset => Marshal.SizeOf(); + + private unsafe ImmutableArray ReadSections() + { + var sections = ImmutableArray.CreateBuilder(_header.coff_sections); + var buffer = new byte[Marshal.SizeOf()]; + _stream.Seek(SectionDirectoryOffset, SeekOrigin.Begin); + for (int i = 0; i < _header.coff_sections; i++) + { + if (_stream.Read(buffer, 0, buffer.Length) != buffer.Length) + { + throw new BadImageFormatException("Stream does not contain a valid Webcil file", nameof(_stream)); + } + fixed (byte* p = buffer) + { + // FIXME endianness + sections.Add(*(WebcilSectionHeader*)p); + } + } + return sections.MoveToImmutable(); + } + + public void Dispose() + { + _stream.Dispose(); + } +} diff --git a/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilSectionHeader.cs b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilSectionHeader.cs new file mode 100644 index 0000000000000..8571fb8ac325c --- /dev/null +++ b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilSectionHeader.cs @@ -0,0 +1,26 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Runtime.InteropServices; + +namespace Microsoft.NET.WebAssembly.Webcil; + +/// +/// This is the Webcil analog of System.Reflection.PortableExecutable.SectionHeader, but with fewer fields +/// +[StructLayout(LayoutKind.Sequential, Pack = 1)] +public readonly struct WebcilSectionHeader +{ + public readonly int VirtualSize; + public readonly int VirtualAddress; + public readonly int SizeOfRawData; + public readonly int PointerToRawData; + + public WebcilSectionHeader(int virtualSize, int virtualAddress, int sizeOfRawData, int pointerToRawData) + { + VirtualSize = virtualSize; + VirtualAddress = virtualAddress; + SizeOfRawData = sizeOfRawData; + PointerToRawData = pointerToRawData; + } +} diff --git a/src/libraries/sendtohelix-wasm.targets b/src/libraries/sendtohelix-wasm.targets index e1afc40d2cb1f..0e9db2b6f861d 100644 --- a/src/libraries/sendtohelix-wasm.targets +++ b/src/libraries/sendtohelix-wasm.targets @@ -236,6 +236,7 @@ $(RepositoryEngineeringDir)testing\scenarios\BuildWasmAppsJobsList.txt Workloads- NoWorkload- + $(WorkItemPrefix)Webcil- @@ -260,7 +261,7 @@ $(_workItemTimeout) - + $(_BuildWasmAppsPayloadArchive) $(HelixCommand) $(_workItemTimeout) diff --git a/src/libraries/sendtohelix.proj b/src/libraries/sendtohelix.proj index 65d5ca651db72..71619fc40466a 100644 --- a/src/libraries/sendtohelix.proj +++ b/src/libraries/sendtohelix.proj @@ -78,12 +78,21 @@ - + <_TestUsingWorkloadsValues Include="true;false" /> - + <_TestUsingWebcilValues Include="true;false" /> + + + <_TestUsingCrossProductValuesTemp Include="@(_TestUsingWorkloadsValues)"> + %(_TestUsingWorkloadsValues.Identity) + + <_TestUsingCrossProductValues Include="@(_TestUsingCrossProductValuesTemp)"> + %(_TestUsingWebcilValues.Identity) + + <_BuildWasmAppsProjectsToBuild Include="$(PerScenarioProjectFile)"> - $(_PropertiesToPass);Scenario=BuildWasmApps;TestArchiveRuntimeFile=$(TestArchiveRuntimeFile);TestUsingWorkloads=%(_TestUsingWorkloadsValues.Identity) + $(_PropertiesToPass);Scenario=BuildWasmApps;TestArchiveRuntimeFile=$(TestArchiveRuntimeFile);TestUsingWorkloads=%(_TestUsingCrossProductValues.Workloads);TestUsingWebcil=%(_TestUsingCrossProductValues.Webcil) %(_BuildWasmAppsProjectsToBuild.AdditionalProperties);NeedsToBuildWasmAppsOnHelix=$(NeedsToBuildWasmAppsOnHelix) diff --git a/src/libraries/sendtohelixhelp.proj b/src/libraries/sendtohelixhelp.proj index 6f0b7704b575f..bfb6447e647b6 100644 --- a/src/libraries/sendtohelixhelp.proj +++ b/src/libraries/sendtohelixhelp.proj @@ -136,6 +136,7 @@ + @@ -287,7 +288,7 @@ + Text="Scenario: $(Scenario), TestUsingWorkloads: $(TestUsingWorkloads), TestUsingWebcil: $(TestUsingWebcil)" /> diff --git a/src/mono/cmake/config.h.in b/src/mono/cmake/config.h.in index ca8e02ac6fd88..d0b5d79ff2097 100644 --- a/src/mono/cmake/config.h.in +++ b/src/mono/cmake/config.h.in @@ -933,6 +933,9 @@ /* Enable System.WeakAttribute support */ #cmakedefine ENABLE_WEAK_ATTR 1 +/* Enable WebCIL image loader */ +#cmakedefine ENABLE_WEBCIL 1 + #if defined(ENABLE_LLVM) && defined(HOST_WIN32) && defined(TARGET_WIN32) && (!defined(TARGET_AMD64) || !defined(_MSC_VER)) #error LLVM for host=Windows and target=Windows is only supported on x64 MSVC build. #endif diff --git a/src/mono/cmake/options.cmake b/src/mono/cmake/options.cmake index f00430fc9500d..202a578db4fc4 100644 --- a/src/mono/cmake/options.cmake +++ b/src/mono/cmake/options.cmake @@ -57,6 +57,7 @@ option (ENABLE_SIGALTSTACK "Enable support for using sigaltstack for SIGSEGV and option (USE_MALLOC_FOR_MEMPOOLS "Use malloc for each single mempool allocation, so tools like Valgrind can run better") option (STATIC_COMPONENTS "Compile mono runtime components as static (not dynamic) libraries") option (DISABLE_WASM_USER_THREADS "Disable creation of user managed threads on WebAssembly, only allow runtime internal managed and native threads") +option (ENABLE_WEBCIL "Enable the WebCIL loader") set (MONO_GC "sgen" CACHE STRING "Garbage collector implementation (sgen or boehm). Default: sgen") set (GC_SUSPEND "default" CACHE STRING "GC suspend method (default, preemptive, coop, hybrid)") diff --git a/src/mono/mono.proj b/src/mono/mono.proj index b4289a0b76f8b..43a084f07a5a0 100644 --- a/src/mono/mono.proj +++ b/src/mono/mono.proj @@ -405,6 +405,7 @@ <_MonoCMakeArgs Include="-DDISABLE_ICALL_TABLES=1"/> <_MonoCMakeArgs Include="-DENABLE_ICALL_EXPORT=1"/> <_MonoCMakeArgs Include="-DENABLE_LAZY_GC_THREAD_CREATION=1"/> + <_MonoCMakeArgs Include="-DENABLE_WEBCIL=1"/> <_MonoCFLAGS Include="-fexceptions"/> <_MonoCFLAGS Condition="'$(MonoWasmThreads)' == 'true'" Include="-pthread"/> <_MonoCFLAGS Condition="'$(MonoWasmThreads)' == 'true'" Include="-D_GNU_SOURCE=1" /> diff --git a/src/mono/mono/metadata/CMakeLists.txt b/src/mono/mono/metadata/CMakeLists.txt index 67f7a66175b6e..797637a566cbc 100644 --- a/src/mono/mono/metadata/CMakeLists.txt +++ b/src/mono/mono/metadata/CMakeLists.txt @@ -78,6 +78,8 @@ set(metadata_common_sources icall-eventpipe.c image.c image-internals.h + webcil-loader.h + webcil-loader.c jit-info.h jit-info.c loader.c diff --git a/src/mono/mono/metadata/assembly.c b/src/mono/mono/metadata/assembly.c index c7be11b4c7af0..2538329ad7ac9 100644 --- a/src/mono/mono/metadata/assembly.c +++ b/src/mono/mono/metadata/assembly.c @@ -720,6 +720,7 @@ search_bundle_for_assembly (MonoAssemblyLoadContext *alc, MonoAssemblyName *anam if (!image && !g_str_has_suffix (aname->name, ".dll")) { char *name = g_strdup_printf ("%s.dll", aname->name); image = mono_assembly_open_from_bundle (alc, name, &status, aname->culture); + g_free (name); } if (image) { mono_assembly_request_prepare_load (&req, alc); @@ -1449,6 +1450,25 @@ absolute_dir (const gchar *filename) return res; } +static gboolean +bundled_assembly_match (const char *bundled_name, const char *name) +{ +#ifndef ENABLE_WEBCIL + return strcmp (bundled_name, name) == 0; +#else + if (strcmp (bundled_name, name) == 0) + return TRUE; + /* if they want a .dll and we have the matching .webcil, return it */ + if (g_str_has_suffix (bundled_name, ".webcil") && g_str_has_suffix (name, ".dll")) { + size_t bprefix = strlen (bundled_name) - 7; + size_t nprefix = strlen (name) - 4; + if (bprefix == nprefix && strncmp (bundled_name, name, bprefix) == 0) + return TRUE; + } + return FALSE; +#endif +} + static MonoImage * open_from_bundle_internal (MonoAssemblyLoadContext *alc, const char *filename, MonoImageOpenStatus *status, gboolean is_satellite) { @@ -1458,7 +1478,7 @@ open_from_bundle_internal (MonoAssemblyLoadContext *alc, const char *filename, M MonoImage *image = NULL; char *name = is_satellite ? g_strdup (filename) : g_path_get_basename (filename); for (int i = 0; !image && bundles [i]; ++i) { - if (strcmp (bundles [i]->name, name) == 0) { + if (bundled_assembly_match (bundles[i]->name, name)) { // Since bundled images don't exist on disk, don't give them a legit filename image = mono_image_open_from_data_internal (alc, (char*)bundles [i]->data, bundles [i]->size, FALSE, status, FALSE, name, NULL); break; @@ -1479,7 +1499,7 @@ open_from_satellite_bundle (MonoAssemblyLoadContext *alc, const char *filename, char *name = g_strdup (filename); for (int i = 0; !image && satellite_bundles [i]; ++i) { - if (strcmp (satellite_bundles [i]->name, name) == 0 && strcmp (satellite_bundles [i]->culture, culture) == 0) { + if (bundled_assembly_match (satellite_bundles[i]->name, name) && strcmp (satellite_bundles [i]->culture, culture) == 0) { char *bundle_name = g_strconcat (culture, "/", name, (const char *)NULL); image = mono_image_open_from_data_internal (alc, (char *)satellite_bundles [i]->data, satellite_bundles [i]->size, FALSE, status, FALSE, bundle_name, NULL); g_free (bundle_name); @@ -2708,6 +2728,14 @@ mono_assembly_load_corlib (void) corlib = mono_assembly_request_open (corlib_name, &req, &status); g_free (corlib_name); } +#ifdef ENABLE_WEBCIL + if (!corlib) { + /* Maybe its in a bundle */ + char *corlib_name = g_strdup_printf ("%s.webcil", MONO_ASSEMBLY_CORLIB_NAME); + corlib = mono_assembly_request_open (corlib_name, &req, &status); + g_free (corlib_name); + } +#endif g_assert (corlib); // exit the process if we weren't able to load corlib diff --git a/src/mono/mono/metadata/image.c b/src/mono/mono/metadata/image.c index 61fac42a5d4e9..2eef0230f2a71 100644 --- a/src/mono/mono/metadata/image.c +++ b/src/mono/mono/metadata/image.c @@ -46,6 +46,7 @@ #include #include #include +#include #include #include #ifdef HAVE_UNISTD_H @@ -259,6 +260,10 @@ mono_images_init (void) install_pe_loader (); +#ifdef ENABLE_WEBCIL + mono_webcil_loader_install (); +#endif + mutex_inited = TRUE; } @@ -923,26 +928,43 @@ do_load_header (MonoImage *image, MonoDotNetHeader *header, int offset) return offset; } -mono_bool -mono_has_pdb_checksum (char *raw_data, uint32_t raw_data_len) +static int32_t +try_load_pe_cli_header (char *raw_data, uint32_t raw_data_len, MonoDotNetHeader *cli_header) { - MonoDotNetHeader cli_header; MonoMSDOSHeader msdos; - guint8 *data; int offset = 0; memcpy (&msdos, raw_data + offset, sizeof (msdos)); if (!(msdos.msdos_sig [0] == 'M' && msdos.msdos_sig [1] == 'Z')) { - return FALSE; + return -1; } msdos.pe_offset = GUINT32_FROM_LE (msdos.pe_offset); offset = msdos.pe_offset; - int ret = do_load_header_internal (raw_data, raw_data_len, &cli_header, offset, FALSE); - if ( ret >= 0 ) { + int32_t ret = do_load_header_internal (raw_data, raw_data_len, cli_header, offset, FALSE); + return ret; +} + +mono_bool +mono_has_pdb_checksum (char *raw_data, uint32_t raw_data_len) +{ + guint8 *data; + MonoDotNetHeader cli_header; + gboolean is_pe = TRUE; + + int32_t ret = try_load_pe_cli_header (raw_data, raw_data_len, &cli_header); + +#ifdef ENABLE_WEBCIL + if (ret == -1) { + ret = mono_webcil_load_cli_header (raw_data, raw_data_len, 0, &cli_header); + is_pe = FALSE; + } +#endif + + if (ret > 0) { MonoPEDirEntry *debug_dir_entry = (MonoPEDirEntry *) &cli_header.datadir.pe_debug; ImageDebugDirectory debug_dir; if (!debug_dir_entry->size) @@ -951,28 +973,39 @@ mono_has_pdb_checksum (char *raw_data, uint32_t raw_data_len) const int top = cli_header.coff.coff_sections; guint32 addr = debug_dir_entry->rva; int i = 0; + gboolean found = FALSE; for (i = 0; i < top; i++){ - MonoSectionTable t; + MonoSectionTable t = {0,}; - if (ret + sizeof (MonoSectionTable) > raw_data_len) { - return FALSE; - } + if (G_LIKELY (is_pe)) { + if (ret + sizeof (MonoSectionTable) > raw_data_len) + return FALSE; - memcpy (&t, raw_data + ret, sizeof (MonoSectionTable)); - ret += sizeof (MonoSectionTable); + memcpy (&t, raw_data + ret, sizeof (MonoSectionTable)); + ret += sizeof (MonoSectionTable); #if G_BYTE_ORDER != G_LITTLE_ENDIAN - t.st_virtual_address = GUINT32_FROM_LE (t.st_virtual_address); - t.st_raw_data_size = GUINT32_FROM_LE (t.st_raw_data_size); - t.st_raw_data_ptr = GUINT32_FROM_LE (t.st_raw_data_ptr); + t.st_virtual_address = GUINT32_FROM_LE (t.st_virtual_address); + t.st_raw_data_size = GUINT32_FROM_LE (t.st_raw_data_size); + t.st_raw_data_ptr = GUINT32_FROM_LE (t.st_raw_data_ptr); #endif + } +#ifdef ENABLE_WEBCIL + else { + ret = mono_webcil_load_section_table (raw_data, raw_data_len, ret, &t); + if (ret == -1) + return FALSE; + } +#endif /* consistency checks here */ if ((addr >= t.st_virtual_address) && (addr < t.st_virtual_address + t.st_raw_data_size)){ addr = addr - t.st_virtual_address + t.st_raw_data_ptr; + found = TRUE; break; } } + g_assert (found); for (guint32 idx = 0; idx < debug_dir_entry->size / sizeof (ImageDebugDirectory); ++idx) { data = (guint8 *) ((ImageDebugDirectory *) (raw_data + addr) + idx); debug_dir.characteristics = read32(data); diff --git a/src/mono/mono/metadata/mono-debug.c b/src/mono/mono/metadata/mono-debug.c index 5c81475ccb244..958b33657d43a 100644 --- a/src/mono/mono/metadata/mono-debug.c +++ b/src/mono/mono/metadata/mono-debug.c @@ -1097,13 +1097,31 @@ mono_register_symfile_for_assembly (const char *assembly_name, const mono_byte * bundled_symfiles = bsymfile; } +static gboolean +bsymfile_match (BundledSymfile *bsymfile, const char *assembly_name) +{ + if (!strcmp (bsymfile->aname, assembly_name)) + return TRUE; +#ifdef ENABLE_WEBCIL + const char *p = strstr (assembly_name, ".webcil"); + /* if assembly_name ends with .webcil, check if aname matches, with a .dll extension instead */ + if (p && *(p + 7) == 0) { + size_t n = p - assembly_name; + if (!strncmp (bsymfile->aname, assembly_name, n) + && !strcmp (bsymfile->aname + n, ".dll")) + return TRUE; + } +#endif + return FALSE; +} + static MonoDebugHandle * open_symfile_from_bundle (MonoImage *image) { BundledSymfile *bsymfile; for (bsymfile = bundled_symfiles; bsymfile; bsymfile = bsymfile->next) { - if (strcmp (bsymfile->aname, image->module_name)) + if (!bsymfile_match (bsymfile, image->module_name)) continue; return mono_debug_open_image (image, bsymfile->raw_contents, bsymfile->size); @@ -1117,7 +1135,7 @@ mono_get_symfile_bytes_from_bundle (const char *assembly_name, int *size) { BundledSymfile *bsymfile; for (bsymfile = bundled_symfiles; bsymfile; bsymfile = bsymfile->next) { - if (strcmp (bsymfile->aname, assembly_name)) + if (!bsymfile_match (bsymfile, assembly_name)) continue; *size = bsymfile->size; return bsymfile->raw_contents; diff --git a/src/mono/mono/metadata/reflection.c b/src/mono/mono/metadata/reflection.c index 487a31f4ec317..ad4d9795b6877 100644 --- a/src/mono/mono/metadata/reflection.c +++ b/src/mono/mono/metadata/reflection.c @@ -1285,7 +1285,7 @@ method_body_object_construct (MonoClass *unused_class, MonoMethod *method, gpoin if ((method->flags & METHOD_ATTRIBUTE_PINVOKE_IMPL) || (method->flags & METHOD_ATTRIBUTE_ABSTRACT) || (method->iflags & METHOD_IMPL_ATTRIBUTE_INTERNAL_CALL) || - (image->raw_data && image->raw_data [1] != 'Z') || + (image->raw_data && (image->raw_data [1] != 'Z' && image->raw_data [1] != 'b')) || (method->iflags & METHOD_IMPL_ATTRIBUTE_RUNTIME)) return MONO_HANDLE_CAST (MonoReflectionMethodBody, NULL_HANDLE); diff --git a/src/mono/mono/metadata/webcil-loader.c b/src/mono/mono/metadata/webcil-loader.c new file mode 100644 index 0000000000000..1323c0f3ff137 --- /dev/null +++ b/src/mono/mono/metadata/webcil-loader.c @@ -0,0 +1,170 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// + +#include + +#include + +#include "mono/metadata/metadata-internals.h" +#include "mono/metadata/webcil-loader.h" + +/* keep in sync with webcil-writer */ +enum { + MONO_WEBCIL_VERSION_MAJOR = 0, + MONO_WEBCIL_VERSION_MINOR = 0, +}; + +typedef struct MonoWebCilHeader { + uint8_t id[4]; // 'W' 'b' 'I' 'L' + // 4 bytes + uint16_t version_major; // 0 + uint16_t version_minor; // 0 + // 8 bytes + uint16_t coff_sections; + uint16_t reserved0; // 0 + // 12 bytes + + uint32_t pe_cli_header_rva; + uint32_t pe_cli_header_size; + // 20 bytes + + uint32_t pe_debug_rva; + uint32_t pe_debug_size; + // 28 bytes +} MonoWebCilHeader; + +static gboolean +webcil_image_match (MonoImage *image) +{ + if (image->raw_data_len >= sizeof (MonoWebCilHeader)) { + return image->raw_data[0] == 'W' && image->raw_data[1] == 'b' && image->raw_data[2] == 'I' && image->raw_data[3] == 'L'; + } + return FALSE; +} + +/* + * Fills the MonoDotNetHeader with data from the given raw_data+offset + * by reading the webcil header. + * most of MonoDotNetHeader is unused and left uninitialized (assumed zero); + */ +static int32_t +do_load_header (const char *raw_data, uint32_t raw_data_len, int32_t offset, MonoDotNetHeader *header) +{ + MonoWebCilHeader wcheader; + if (offset + sizeof (MonoWebCilHeader) > raw_data_len) + return -1; + memcpy (&wcheader, raw_data + offset, sizeof (wcheader)); + + if (!(wcheader.id [0] == 'W' && wcheader.id [1] == 'b' && wcheader.id[2] == 'I' && wcheader.id[3] == 'L' && + GUINT16_FROM_LE (wcheader.version_major) == MONO_WEBCIL_VERSION_MAJOR && GUINT16_FROM_LE (wcheader.version_minor) == MONO_WEBCIL_VERSION_MINOR)) + return -1; + + memset (header, 0, sizeof(MonoDotNetHeader)); + header->coff.coff_sections = GUINT16_FROM_LE (wcheader.coff_sections); + header->datadir.pe_cli_header.rva = GUINT32_FROM_LE (wcheader.pe_cli_header_rva); + header->datadir.pe_cli_header.size = GUINT32_FROM_LE (wcheader.pe_cli_header_size); + header->datadir.pe_debug.rva = GUINT32_FROM_LE (wcheader.pe_debug_rva); + header->datadir.pe_debug.size = GUINT32_FROM_LE (wcheader.pe_debug_size); + + offset += sizeof (wcheader); + return offset; +} + +int32_t +mono_webcil_load_section_table (const char *raw_data, uint32_t raw_data_len, int32_t offset, MonoSectionTable *t) +{ + /* WebCIL section table entries are a subset of a PE section + * header. Initialize just the parts we have. + */ + uint32_t st [4]; + + if (G_UNLIKELY (offset < 0)) + return offset; + if ((uint32_t)offset > raw_data_len) + return -1; + memcpy (st, raw_data + offset, sizeof (st)); + t->st_virtual_size = GUINT32_FROM_LE (st [0]); + t->st_virtual_address = GUINT32_FROM_LE (st [1]); + t->st_raw_data_size = GUINT32_FROM_LE (st [2]); + t->st_raw_data_ptr = GUINT32_FROM_LE (st [3]); + offset += sizeof(st); + return offset; +} + + +static gboolean +webcil_image_load_pe_data (MonoImage *image) +{ + MonoCLIImageInfo *iinfo; + MonoDotNetHeader *header; + int32_t offset = 0; + int top; + + iinfo = image->image_info; + header = &iinfo->cli_header; + + offset = do_load_header (image->raw_data, image->raw_data_len, offset, header); + if (offset == -1) + goto invalid_image; + + top = iinfo->cli_header.coff.coff_sections; + + iinfo->cli_section_count = top; + iinfo->cli_section_tables = g_new0 (MonoSectionTable, top); + iinfo->cli_sections = g_new0 (void *, top); + + for (int i = 0; i < top; i++) { + MonoSectionTable *t = &iinfo->cli_section_tables [i]; + offset = mono_webcil_load_section_table (image->raw_data, image->raw_data_len, offset, t); + if (offset == -1) + goto invalid_image; + } + + return TRUE; + +invalid_image: + return FALSE; + +} + +static gboolean +webcil_image_load_cli_data (MonoImage *image) +{ + MonoCLIImageInfo *iinfo; + + iinfo = image->image_info; + + if (!mono_image_load_cli_header (image, iinfo)) + return FALSE; + + if (!mono_image_load_metadata (image, iinfo)) + return FALSE; + + return TRUE; +} + +static gboolean +webcil_image_load_tables (MonoImage *image) +{ + return TRUE; +} + +static const MonoImageLoader webcil_loader = { + webcil_image_match, + webcil_image_load_pe_data, + webcil_image_load_cli_data, + webcil_image_load_tables, +}; + +void +mono_webcil_loader_install (void) +{ + mono_install_image_loader (&webcil_loader); +} + +int32_t +mono_webcil_load_cli_header (const char *raw_data, uint32_t raw_data_len, int32_t offset, MonoDotNetHeader *header) +{ + return do_load_header (raw_data, raw_data_len, offset, header); +} diff --git a/src/mono/mono/metadata/webcil-loader.h b/src/mono/mono/metadata/webcil-loader.h new file mode 100644 index 0000000000000..c95c2c5690409 --- /dev/null +++ b/src/mono/mono/metadata/webcil-loader.h @@ -0,0 +1,17 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// + +#ifndef _MONO_METADATA_WEBCIL_LOADER_H +#define _MONO_METADATA_WEBCIL_LOADER_H + +void +mono_webcil_loader_install (void); + +int32_t +mono_webcil_load_cli_header (const char *raw_data, uint32_t raw_data_len, int32_t offset, MonoDotNetHeader *header); + +int32_t +mono_webcil_load_section_table (const char *raw_data, uint32_t raw_data_len, int32_t offset, MonoSectionTable *t); + +#endif /*_MONO_METADATA_WEBCIL_LOADER_H*/ diff --git a/src/mono/mono/mini/monovm.c b/src/mono/mono/mini/monovm.c index 144594276f218..63e25581ac7a6 100644 --- a/src/mono/mono/mini/monovm.c +++ b/src/mono/mono/mini/monovm.c @@ -131,6 +131,22 @@ mono_core_preload_hook (MonoAssemblyLoadContext *alc, MonoAssemblyName *aname, c if (result) break; } +#ifdef ENABLE_WEBCIL + else { + /* /path/foo.dll -> /path/foo.webcil */ + size_t n = strlen (fullpath) - 4; + char *fullpath2 = g_malloc (n + 8); + g_strlcpy (fullpath2, fullpath, n + 1); + g_strlcpy (fullpath2 + n, ".webcil", 8); + if (g_file_test (fullpath2, G_FILE_TEST_IS_REGULAR)) { + MonoImageOpenStatus status; + result = mono_assembly_request_open (fullpath2, &req, &status); + } + g_free (fullpath2); + if (result) + break; + } +#endif } } diff --git a/src/mono/nuget/Microsoft.NETCore.BrowserDebugHost.Transport/Microsoft.NETCore.BrowserDebugHost.Transport.pkgproj b/src/mono/nuget/Microsoft.NETCore.BrowserDebugHost.Transport/Microsoft.NETCore.BrowserDebugHost.Transport.pkgproj index 7a6b8326bcfb3..2119425f85f0c 100644 --- a/src/mono/nuget/Microsoft.NETCore.BrowserDebugHost.Transport/Microsoft.NETCore.BrowserDebugHost.Transport.pkgproj +++ b/src/mono/nuget/Microsoft.NETCore.BrowserDebugHost.Transport/Microsoft.NETCore.BrowserDebugHost.Transport.pkgproj @@ -14,6 +14,7 @@ <_browserDebugHostFiles Include="$(ArtifactsDir)bin\BrowserDebugHost\$(TargetArchitecture)\$(Configuration)\BrowserDebugHost.dll" /> <_browserDebugHostFiles Include="$(ArtifactsDir)bin\BrowserDebugHost\$(TargetArchitecture)\$(Configuration)\BrowserDebugHost.runtimeconfig.json" /> <_browserDebugHostFiles Include="$(ArtifactsDir)bin\BrowserDebugHost\$(TargetArchitecture)\$(Configuration)\BrowserDebugProxy.dll" /> + <_browserDebugHostFiles Include="$(ArtifactsDir)bin\BrowserDebugHost\$(TargetArchitecture)\$(Configuration)\Microsoft.NET.WebAssembly.Webcil.dll" /> <_browserDebugHostFiles Include="$(ArtifactsDir)bin\BrowserDebugHost\$(TargetArchitecture)\$(Configuration)\Microsoft.CodeAnalysis.CSharp.dll" /> <_browserDebugHostFiles Include="$(ArtifactsDir)bin\BrowserDebugHost\$(TargetArchitecture)\$(Configuration)\Microsoft.CodeAnalysis.dll" /> <_browserDebugHostFiles Include="$(ArtifactsDir)bin\BrowserDebugHost\$(TargetArchitecture)\$(Configuration)\Newtonsoft.Json.dll" /> diff --git a/src/mono/wasm/Wasm.Build.Tests/BuildEnvironment.cs b/src/mono/wasm/Wasm.Build.Tests/BuildEnvironment.cs index f40ddec9aa4a8..6d88eb47a2563 100644 --- a/src/mono/wasm/Wasm.Build.Tests/BuildEnvironment.cs +++ b/src/mono/wasm/Wasm.Build.Tests/BuildEnvironment.cs @@ -24,11 +24,14 @@ public class BuildEnvironment public string WorkloadPacksDir { get; init; } public string BuiltNuGetsPath { get; init; } + public bool UseWebcil { get; init; } + public static readonly string RelativeTestAssetsPath = @"..\testassets\"; public static readonly string TestAssetsPath = Path.Combine(AppContext.BaseDirectory, "testassets"); public static readonly string TestDataPath = Path.Combine(AppContext.BaseDirectory, "data"); public static readonly string TmpPath = Path.Combine(AppContext.BaseDirectory, "wbt"); + private static readonly Dictionary s_runtimePackVersions = new(); public BuildEnvironment() @@ -86,6 +89,8 @@ public BuildEnvironment() DirectoryBuildTargetsContents = s_directoryBuildTargetsForLocal; } + UseWebcil = EnvironmentVariables.UseWebcil; + if (EnvironmentVariables.BuiltNuGetsPath is null || !Directory.Exists(EnvironmentVariables.BuiltNuGetsPath)) throw new Exception($"Cannot find 'BUILT_NUGETS_PATH={EnvironmentVariables.BuiltNuGetsPath}'"); diff --git a/src/mono/wasm/Wasm.Build.Tests/BuildTestBase.cs b/src/mono/wasm/Wasm.Build.Tests/BuildTestBase.cs index 0b4777ae7ac31..9ebc210e803b5 100644 --- a/src/mono/wasm/Wasm.Build.Tests/BuildTestBase.cs +++ b/src/mono/wasm/Wasm.Build.Tests/BuildTestBase.cs @@ -50,6 +50,7 @@ public abstract class BuildTestBase : IClassFixture s_buildEnv.IsWorkload; public static bool IsNotUsingWorkloads => !s_buildEnv.IsWorkload; + public static bool UseWebcil => s_buildEnv.UseWebcil; public static string GetNuGetConfigPathFor(string targetFramework) => Path.Combine(BuildEnvironment.TestDataPath, "nuget8.config"); // for now - we are still using net7, but with // targetFramework == "net7.0" ? "nuget7.config" : "nuget8.config"); @@ -71,6 +72,8 @@ static BuildTestBase() Console.WriteLine (""); Console.WriteLine ($"=============================================================================================="); Console.WriteLine ($"=============== Running with {(s_buildEnv.IsWorkload ? "Workloads" : "No workloads")} ==============="); + if (UseWebcil) + Console.WriteLine($"=============== Using .webcil ==============="); Console.WriteLine ($"=============================================================================================="); Console.WriteLine (""); } @@ -336,6 +339,10 @@ protected static BuildArgs ExpandBuildArgs(BuildArgs buildArgs, string extraProp extraProperties += $"\n{RuntimeInformation.IsOSPlatform(OSPlatform.Windows)}\n"; } + if (UseWebcil) { + extraProperties += "true\n"; + } + string projectContents = projectTemplate .Replace("##EXTRA_PROPERTIES##", extraProperties) .Replace("##EXTRA_ITEMS##", extraItems) @@ -423,7 +430,8 @@ protected static BuildArgs ExpandBuildArgs(BuildArgs buildArgs, string extraProp options.HasV8Script, options.TargetFramework ?? DefaultTargetFramework, options.HasIcudt, - options.DotnetWasmFromRuntimePack ?? !buildArgs.AOT); + options.DotnetWasmFromRuntimePack ?? !buildArgs.AOT, + UseWebcil); } if (options.UseCache) @@ -487,6 +495,8 @@ public string CreateWasmTemplateProject(string id, string template = "wasmbrowse string projectfile = Path.Combine(_projectDir!, $"{id}.csproj"); if (runAnalyzers) AddItemsPropertiesToProject(projectfile, "true"); + if (UseWebcil) + AddItemsPropertiesToProject(projectfile, "true"); return projectfile; } @@ -499,7 +509,10 @@ public string CreateBlazorWasmTemplateProject(string id) .ExecuteWithCapturedOutput("new blazorwasm") .EnsureSuccessful(); - return Path.Combine(_projectDir!, $"{id}.csproj"); + string projectFile = Path.Combine(_projectDir!, $"{id}.csproj"); + if (UseWebcil) + AddItemsPropertiesToProject(projectFile, "true"); + return projectFile; } protected (CommandResult, string) BlazorBuild(BlazorBuildOptions options, params string[] extraArgs) @@ -550,6 +563,7 @@ public string CreateBlazorWasmTemplateProject(string id) label, // same as the command name $"-bl:{logPath}", $"-p:Configuration={config}", + UseWebcil ? "-p:WasmEnableWebcil=true" : string.Empty, "-p:BlazorEnableCompression=false", "-nr:false", setWasmDevel ? "-p:_WasmDevel=true" : string.Empty @@ -616,7 +630,8 @@ protected static void AssertBasicAppBundle(string bundleDir, bool hasV8Script, string targetFramework, bool hasIcudt = true, - bool dotnetWasmFromRuntimePack = true) + bool dotnetWasmFromRuntimePack = true, + bool useWebcil = true) { AssertFilesExist(bundleDir, new [] { @@ -632,7 +647,9 @@ protected static void AssertBasicAppBundle(string bundleDir, AssertFilesExist(bundleDir, new[] { "icudt.dat" }, expectToExist: hasIcudt); string managedDir = Path.Combine(bundleDir, "managed"); - AssertFilesExist(managedDir, new[] { $"{projectName}.dll" }); + string bundledMainAppAssembly = + useWebcil ? $"{projectName}.webcil" : $"{projectName}.dll"; + AssertFilesExist(managedDir, new[] { bundledMainAppAssembly }); bool is_debug = config == "Debug"; if (is_debug) @@ -1094,7 +1111,7 @@ public record BuildProjectOptions bool CreateProject = true, bool Publish = true, bool BuildOnlyAfterPublish = true, - bool HasV8Script = true, + bool HasV8Script = true, string? Verbosity = null, string? Label = null, string? TargetFramework = null, diff --git a/src/mono/wasm/Wasm.Build.Tests/EnvironmentVariables.cs b/src/mono/wasm/Wasm.Build.Tests/EnvironmentVariables.cs index 328006786d09c..91f6fadb51f5b 100644 --- a/src/mono/wasm/Wasm.Build.Tests/EnvironmentVariables.cs +++ b/src/mono/wasm/Wasm.Build.Tests/EnvironmentVariables.cs @@ -18,5 +18,6 @@ internal static class EnvironmentVariables internal static readonly string? BuiltNuGetsPath = Environment.GetEnvironmentVariable("BUILT_NUGETS_PATH"); internal static readonly string? BrowserPathForTests = Environment.GetEnvironmentVariable("BROWSER_PATH_FOR_TESTS"); internal static readonly bool ShowBuildOutput = Environment.GetEnvironmentVariable("SHOW_BUILD_OUTPUT") is not null; + internal static readonly bool UseWebcil = Environment.GetEnvironmentVariable("USE_WEBCIL_FOR_TESTS") is "true"; } } diff --git a/src/mono/wasm/Wasm.Build.Tests/PInvokeTableGeneratorTests.cs b/src/mono/wasm/Wasm.Build.Tests/PInvokeTableGeneratorTests.cs index 5473667f294c9..0168840ad99bf 100644 --- a/src/mono/wasm/Wasm.Build.Tests/PInvokeTableGeneratorTests.cs +++ b/src/mono/wasm/Wasm.Build.Tests/PInvokeTableGeneratorTests.cs @@ -466,15 +466,29 @@ public static void Main() string tasksDir = Path.Combine(s_buildEnv.WorkloadPacksDir, "Microsoft.NET.Runtime.WebAssembly.Sdk", s_buildEnv.GetRuntimePackVersion(DefaultTargetFramework), - "tasks"); - if (!Directory.Exists(tasksDir)) + "tasks", + BuildTestBase.DefaultTargetFramework); // not net472! + if (!Directory.Exists(tasksDir)) { + string? tasksDirParent = Path.GetDirectoryName (tasksDir); + if (!string.IsNullOrEmpty (tasksDirParent)) { + if (!Directory.Exists(tasksDirParent)) { + _testOutput.WriteLine($"Expected {tasksDirParent} to exist and contain TFM subdirectories"); + } + _testOutput.WriteLine($"runtime pack tasks dir {tasksDir} contains subdirectories:"); + foreach (string subdir in Directory.EnumerateDirectories(tasksDirParent)) { + _testOutput.WriteLine($" - {subdir}"); + } + } throw new DirectoryNotFoundException($"Could not find tasks directory {tasksDir}"); + } string? taskPath = Directory.EnumerateFiles(tasksDir, "WasmAppBuilder.dll", SearchOption.AllDirectories) .FirstOrDefault(); if (string.IsNullOrEmpty(taskPath)) throw new FileNotFoundException($"Could not find WasmAppBuilder.dll in {tasksDir}"); + _testOutput.WriteLine ("Using WasmAppBuilder.dll from {0}", taskPath); + projectCode = projectCode .Replace("###WasmPInvokeModule###", AddAssembly("System.Private.CoreLib") + AddAssembly("System.Runtime") + AddAssembly(libraryBuildArgs.ProjectName)) .Replace("###WasmAppBuilder###", taskPath); diff --git a/src/mono/wasm/Wasm.Build.Tests/Wasm.Build.Tests.csproj b/src/mono/wasm/Wasm.Build.Tests/Wasm.Build.Tests.csproj index cf7a20b562391..133140d696095 100644 --- a/src/mono/wasm/Wasm.Build.Tests/Wasm.Build.Tests.csproj +++ b/src/mono/wasm/Wasm.Build.Tests/Wasm.Build.Tests.csproj @@ -66,6 +66,9 @@ + + + diff --git a/src/mono/wasm/Wasm.Build.Tests/data/RunScriptTemplate.cmd b/src/mono/wasm/Wasm.Build.Tests/data/RunScriptTemplate.cmd index f38746bf13151..23e3dbf4ce470 100644 --- a/src/mono/wasm/Wasm.Build.Tests/data/RunScriptTemplate.cmd +++ b/src/mono/wasm/Wasm.Build.Tests/data/RunScriptTemplate.cmd @@ -104,6 +104,11 @@ if [%TEST_USING_WORKLOADS%] == [true] ( set _DIR_NAME=dotnet-none set SDK_HAS_WORKLOAD_INSTALLED=false ) +if [%TEST_USING_WEBCIL%] == [true] ( + set USE_WEBCIL_FOR_TESTS=true +) else ( + set USE_WEBCIL_FOR_TESTS=false +) if [%HELIX_CORRELATION_PAYLOAD%] NEQ [] ( robocopy /mt /np /nfl /NDL /nc /e %BASE_DIR%\%_DIR_NAME% %EXECUTION_DIR%\%_DIR_NAME% diff --git a/src/mono/wasm/Wasm.Build.Tests/data/RunScriptTemplate.sh b/src/mono/wasm/Wasm.Build.Tests/data/RunScriptTemplate.sh index 1f520aebd65f6..70187b9c5e58e 100644 --- a/src/mono/wasm/Wasm.Build.Tests/data/RunScriptTemplate.sh +++ b/src/mono/wasm/Wasm.Build.Tests/data/RunScriptTemplate.sh @@ -79,6 +79,12 @@ function set_env_vars() export SDK_HAS_WORKLOAD_INSTALLED=false fi + if [ "x$TEST_USING_WEBCIL" = "xtrue" ]; then + export USE_WEBCIL_FOR_TESTS=true + else + export USE_WEBCIL_FOR_TESTS=false + fi + local _SDK_DIR= if [[ -n "$HELIX_WORKITEM_UPLOAD_ROOT" ]]; then cp -r $BASE_DIR/$_DIR_NAME $EXECUTION_DIR diff --git a/src/mono/wasm/build/WasmApp.targets b/src/mono/wasm/build/WasmApp.targets index e248125876003..8934610bfa84a 100644 --- a/src/mono/wasm/build/WasmApp.targets +++ b/src/mono/wasm/build/WasmApp.targets @@ -67,6 +67,7 @@ - $(WasmAotProfilePath) - Path to an AOT profile file. - $(WasmEnableExceptionHandling) - Enable support for the WASM Exception Handling feature. - $(WasmEnableSIMD) - Enable support for the WASM SIMD feature. + - $(WasmEnableWebcil) - Enable conversion of assembly .dlls to .webcil Public items: - @(WasmExtraFilesToDeploy) - Files to copy to $(WasmAppDir). @@ -117,6 +118,9 @@ -1 + + + false @@ -367,6 +371,7 @@ DebugLevel="$(WasmDebugLevel)" IncludeThreadsWorker="$(_WasmAppIncludeThreadsWorker)" PThreadPoolSize="$(_WasmPThreadPoolSize)" + UseWebcil="$(WasmEnableWebcil)" > diff --git a/src/mono/wasm/debugger/BrowserDebugProxy/BrowserDebugProxy.csproj b/src/mono/wasm/debugger/BrowserDebugProxy/BrowserDebugProxy.csproj index c05515cdd1062..eb93b1a34e23f 100644 --- a/src/mono/wasm/debugger/BrowserDebugProxy/BrowserDebugProxy.csproj +++ b/src/mono/wasm/debugger/BrowserDebugProxy/BrowserDebugProxy.csproj @@ -14,4 +14,8 @@ + + + + diff --git a/src/mono/wasm/debugger/BrowserDebugProxy/DebugStore.cs b/src/mono/wasm/debugger/BrowserDebugProxy/DebugStore.cs index 1e0b19dd435ee..d053921f05415 100644 --- a/src/mono/wasm/debugger/BrowserDebugProxy/DebugStore.cs +++ b/src/mono/wasm/debugger/BrowserDebugProxy/DebugStore.cs @@ -21,6 +21,7 @@ using System.Reflection; using System.Diagnostics; using System.Text; +using Microsoft.NET.WebAssembly.Webcil; namespace Microsoft.WebAssembly.Diagnostics { @@ -854,7 +855,7 @@ internal sealed class AssemblyInfo private readonly List sources = new List(); internal string Url { get; } //The caller must keep the PEReader alive and undisposed throughout the lifetime of the metadata reader - internal PEReader peReader; + private readonly IDisposable peReaderOrWebcilReader; internal MetadataReader asmMetadataReader { get; } internal MetadataReader pdbMetadataReader { get; set; } internal List> enCMetadataReader = new List>(); @@ -867,36 +868,54 @@ internal sealed class AssemblyInfo private readonly Dictionary _documentIdToSourceFileTable = new Dictionary(); - public AssemblyInfo(ILogger logger) + public static AssemblyInfo FromBytes(MonoProxy monoProxy, SessionId sessionId, byte[] assembly, byte[] pdb, ILogger logger, CancellationToken token) { - debugId = -1; - this.id = Interlocked.Increment(ref next_id); - this.logger = logger; + // First try to read it as a PE file, otherwise try it as a WebCIL file + using var asmStream = new MemoryStream(assembly); + try + { + var peReader = new PEReader(asmStream); + if (!peReader.HasMetadata) + throw new BadImageFormatException(); + return FromPEReader(monoProxy, sessionId, peReader, pdb, logger, token); + } + catch (BadImageFormatException) + { + // This is a WebAssembly file + asmStream.Seek(0, SeekOrigin.Begin); + var webcilReader = new WebcilReader(asmStream); + return FromWebcilReader(monoProxy, sessionId, webcilReader, pdb, logger, token); + } + } + + public static AssemblyInfo WithoutDebugInfo(ILogger logger) + { + return new AssemblyInfo(logger); } - public unsafe AssemblyInfo(MonoProxy monoProxy, SessionId sessionId, byte[] assembly, byte[] pdb, ILogger logger, CancellationToken token) + private AssemblyInfo(ILogger logger) { debugId = -1; this.id = Interlocked.Increment(ref next_id); this.logger = logger; - using var asmStream = new MemoryStream(assembly); - peReader = new PEReader(asmStream); + } + + private static AssemblyInfo FromPEReader(MonoProxy monoProxy, SessionId sessionId, PEReader peReader, byte[] pdb, ILogger logger, CancellationToken token) + { var entries = peReader.ReadDebugDirectory(); + CodeViewDebugDirectoryData? codeViewData = null; if (entries.Length > 0) { var codeView = entries[0]; if (codeView.Type == DebugDirectoryEntryType.CodeView) { - CodeViewDebugDirectoryData codeViewData = peReader.ReadCodeViewDebugDirectoryData(codeView); - PdbAge = codeViewData.Age; - PdbGuid = codeViewData.Guid; - PdbName = codeViewData.Path; - CodeViewInformationAvailable = true; + codeViewData = peReader.ReadCodeViewDebugDirectoryData(codeView); } } - asmMetadataReader = PEReaderExtensions.GetMetadataReader(peReader); - var asmDef = asmMetadataReader.GetAssemblyDefinition(); - Name = asmDef.GetAssemblyName().Name + ".dll"; + var asmMetadataReader = PEReaderExtensions.GetMetadataReader(peReader); + string name = ReadAssemblyName(asmMetadataReader); + + MetadataReader pdbMetadataReader = null; if (pdb != null) { var pdbStream = new MemoryStream(pdb); @@ -907,7 +926,7 @@ public unsafe AssemblyInfo(MonoProxy monoProxy, SessionId sessionId, byte[] asse } catch (BadImageFormatException) { - monoProxy.SendLog(sessionId, $"Warning: Unable to read debug information of: {Name} (use DebugType=Portable/Embedded)", token); + monoProxy.SendLog(sessionId, $"Warning: Unable to read debug information of: {name} (use DebugType=Portable/Embedded)", token); } } else @@ -918,6 +937,74 @@ public unsafe AssemblyInfo(MonoProxy monoProxy, SessionId sessionId, byte[] asse pdbMetadataReader = peReader.ReadEmbeddedPortablePdbDebugDirectoryData(embeddedPdbEntry).GetMetadataReader(); } } + + var assemblyInfo = new AssemblyInfo(peReader, name, asmMetadataReader, codeViewData, pdbMetadataReader, logger); + return assemblyInfo; + } + + private static AssemblyInfo FromWebcilReader(MonoProxy monoProxy, SessionId sessionId, WebcilReader wcReader, byte[] pdb, ILogger logger, CancellationToken token) + { + var entries = wcReader.ReadDebugDirectory(); + CodeViewDebugDirectoryData? codeViewData = null; + if (entries.Length > 0) + { + var codeView = entries[0]; + if (codeView.Type == DebugDirectoryEntryType.CodeView) + { + codeViewData = wcReader.ReadCodeViewDebugDirectoryData(codeView); + } + } + var asmMetadataReader = wcReader.GetMetadataReader(); + string name = ReadAssemblyName(asmMetadataReader); + + MetadataReader pdbMetadataReader = null; + if (pdb != null) + { + var pdbStream = new MemoryStream(pdb); + try + { + // MetadataReaderProvider.FromPortablePdbStream takes ownership of the stream + pdbMetadataReader = MetadataReaderProvider.FromPortablePdbStream(pdbStream).GetMetadataReader(); + } + catch (BadImageFormatException) + { + monoProxy.SendLog(sessionId, $"Warning: Unable to read debug information of: {name} (use DebugType=Portable/Embedded)", token); + } + } + else + { + var embeddedPdbEntry = entries.FirstOrDefault(e => e.Type == DebugDirectoryEntryType.EmbeddedPortablePdb); + if (embeddedPdbEntry.DataSize != 0) + { + pdbMetadataReader = wcReader.ReadEmbeddedPortablePdbDebugDirectoryData(embeddedPdbEntry).GetMetadataReader(); + } + } + + var assemblyInfo = new AssemblyInfo(wcReader, name, asmMetadataReader, codeViewData, pdbMetadataReader, logger); + return assemblyInfo; + } + + private static string ReadAssemblyName(MetadataReader asmMetadataReader) + { + var asmDef = asmMetadataReader.GetAssemblyDefinition(); + return asmDef.GetAssemblyName().Name + ".dll"; + } + + private unsafe AssemblyInfo(IDisposable owningReader, string name, MetadataReader asmMetadataReader, CodeViewDebugDirectoryData? codeViewData, MetadataReader pdbMetadataReader, ILogger logger) + : this(logger) + { + peReaderOrWebcilReader = owningReader; + if (codeViewData != null) + { + PdbAge = codeViewData.Value.Age; + PdbGuid = codeViewData.Value.Guid; + PdbName = codeViewData.Value.Path; + CodeViewInformationAvailable = true; + } + this.asmMetadataReader = asmMetadataReader; + Name = name; + logger.LogTrace($"Info: loading AssemblyInfo with name {Name}"); + this.pdbMetadataReader = pdbMetadataReader; Populate(); } @@ -1410,7 +1497,7 @@ public IEnumerable Add(SessionId id, byte[] assembly_data, byte[] pd AssemblyInfo assembly; try { - assembly = new AssemblyInfo(monoProxy, id, assembly_data, pdb_data, logger, token); + assembly = AssemblyInfo.FromBytes(monoProxy, id, assembly_data, pdb_data, logger, token); } catch (Exception e) { @@ -1504,7 +1591,7 @@ public async IAsyncEnumerable Load(SessionId id, string[] loaded_fil logger.LogDebug($"Bytes from assembly {step.Url} is NULL"); continue; } - assembly = new AssemblyInfo(monoProxy, id, bytes[0], bytes[1], logger, token); + assembly = AssemblyInfo.FromBytes(monoProxy, id, bytes[0], bytes[1], logger, token); } catch (Exception e) { diff --git a/src/mono/wasm/debugger/BrowserDebugProxy/MonoSDBHelper.cs b/src/mono/wasm/debugger/BrowserDebugProxy/MonoSDBHelper.cs index 055c44880842e..29d6356acb09e 100644 --- a/src/mono/wasm/debugger/BrowserDebugProxy/MonoSDBHelper.cs +++ b/src/mono/wasm/debugger/BrowserDebugProxy/MonoSDBHelper.cs @@ -855,7 +855,7 @@ public async Task GetAssemblyInfo(int assemblyId, CancellationToke asm = store.GetAssemblyByName(assemblyName); if (asm == null) { - asm = new AssemblyInfo(logger); + asm = AssemblyInfo.WithoutDebugInfo(logger); logger.LogDebug($"Created assembly without debug information: {assemblyName}"); } } @@ -1205,7 +1205,13 @@ public async Task GetAssemblyName(int assembly_id, CancellationToken tok commandParamsWriter.Write(assembly_id); using var retDebuggerCmdReader = await SendDebuggerAgentCommand(CmdAssembly.GetLocation, commandParamsWriter, token); - return retDebuggerCmdReader.ReadString(); + string result = retDebuggerCmdReader.ReadString(); + if (result.EndsWith(".webcil")) { + /* don't leak .webcil names to the debugger - work in terms of the original .dlls */ + string baseName = result.Substring(0, result.Length - 7); + result = baseName + ".dll"; + } + return result; } public async Task GetFullAssemblyName(int assemblyId, CancellationToken token) diff --git a/src/mono/wasm/debugger/DebuggerTestSuite/DebuggerTestBase.cs b/src/mono/wasm/debugger/DebuggerTestSuite/DebuggerTestBase.cs index e2ca58fe62054..d5cac7e9a3d4f 100644 --- a/src/mono/wasm/debugger/DebuggerTestSuite/DebuggerTestBase.cs +++ b/src/mono/wasm/debugger/DebuggerTestSuite/DebuggerTestBase.cs @@ -275,7 +275,7 @@ public async Task WaitForScriptParsedEventsAsync(params string[] paths) // sets breakpoint by method name and line offset internal async Task CheckInspectLocalsAtBreakpointSite(string type, string method, int line_offset, string bp_function_name, string eval_expression, - Func locals_fn = null, Func wait_for_event_fn = null, bool use_cfo = false, string assembly = "debugger-test.dll", int col = 0) + Func locals_fn = null, Func wait_for_event_fn = null, bool use_cfo = false, string assembly = "debugger-test", int col = 0) { UseCallFunctionOnBeforeGetProperties = use_cfo; diff --git a/src/mono/wasm/debugger/DebuggerTestSuite/MonoJsTests.cs b/src/mono/wasm/debugger/DebuggerTestSuite/MonoJsTests.cs index 93d7221c51f7e..139daae4ae326 100644 --- a/src/mono/wasm/debugger/DebuggerTestSuite/MonoJsTests.cs +++ b/src/mono/wasm/debugger/DebuggerTestSuite/MonoJsTests.cs @@ -136,7 +136,7 @@ async Task AssemblyLoadedEventTest(string asm_name, string asm_path, string pdb_ : await Task.FromResult(ProtocolEventHandlerReturn.KeepHandler); }); - byte[] bytes = File.ReadAllBytes(asm_path); + byte[] bytes = File.Exists(asm_path) ? File.ReadAllBytes(asm_path) : File.ReadAllBytes(Path.ChangeExtension(asm_path, ".webcil")); // hack! string asm_base64 = Convert.ToBase64String(bytes); string pdb_base64 = String.Empty; diff --git a/src/mono/wasm/debugger/Wasm.Debugger.Tests/wasm.helix.targets b/src/mono/wasm/debugger/Wasm.Debugger.Tests/wasm.helix.targets index e874cc4239df4..b9a4726d94f5c 100644 --- a/src/mono/wasm/debugger/Wasm.Debugger.Tests/wasm.helix.targets +++ b/src/mono/wasm/debugger/Wasm.Debugger.Tests/wasm.helix.targets @@ -3,8 +3,8 @@ true $(DebuggerHost)- true - <_DebuggerTestsWorkItemTimeout Condition="'$(Scenario)' == 'WasmDebuggerTests'">00:20:00 - <_DebuggerTestsWorkItemTimeout Condition="'$(Scenario)' == 'WasmDebuggerTests' and '$(BrowserHost)' == 'windows'">00:30:00 + <_DebuggerTestsWorkItemTimeout Condition="'$(Scenario)' == 'WasmDebuggerTests'">00:30:00 + <_DebuggerTestsWorkItemTimeout Condition="'$(Scenario)' == 'WasmDebuggerTests' and '$(BrowserHost)' == 'windows'">00:40:00 $(HelixExtensionTargets);_AddWorkItemsForWasmDebuggerTests diff --git a/src/tasks/WasmAppBuilder/WasmAppBuilder.cs b/src/tasks/WasmAppBuilder/WasmAppBuilder.cs index 2fead7b9860bf..8dd76eeab7952 100644 --- a/src/tasks/WasmAppBuilder/WasmAppBuilder.cs +++ b/src/tasks/WasmAppBuilder/WasmAppBuilder.cs @@ -48,6 +48,7 @@ public class WasmAppBuilder : Task public string? MainHTMLPath { get; set; } public bool IncludeThreadsWorker {get; set; } public int PThreadPoolSize {get; set; } + public bool UseWebcil { get; set; } // // Extra json elements to add to mono-config.json @@ -187,9 +188,22 @@ private bool ExecuteInternal () var asmRootPath = Path.Combine(AppDir, config.AssemblyRootFolder); Directory.CreateDirectory(AppDir!); Directory.CreateDirectory(asmRootPath); + if (UseWebcil) + Log.LogMessage (MessageImportance.Normal, "Converting assemblies to Webcil"); foreach (var assembly in _assemblies) { - FileCopyChecked(assembly, Path.Combine(asmRootPath, Path.GetFileName(assembly)), "Assemblies"); + if (UseWebcil) + { + var tmpWebcil = Path.GetTempFileName(); + var webcilWriter = Microsoft.WebAssembly.Build.Tasks.WebcilConverter.FromPortableExecutable(inputPath: assembly, outputPath: tmpWebcil, logger: Log); + webcilWriter.ConvertToWebcil(); + var finalWebcil = Path.ChangeExtension(assembly, ".webcil"); + FileCopyChecked(tmpWebcil, Path.Combine(asmRootPath, Path.GetFileName(finalWebcil)), "Assemblies"); + } + else + { + FileCopyChecked(assembly, Path.Combine(asmRootPath, Path.GetFileName(assembly)), "Assemblies"); + } if (DebugLevel != 0) { var pdb = assembly; @@ -234,7 +248,10 @@ private bool ExecuteInternal () foreach (var assembly in _assemblies) { - config.Assets.Add(new AssemblyEntry(Path.GetFileName(assembly))); + string assemblyPath = assembly; + if (UseWebcil) + assemblyPath = Path.ChangeExtension(assemblyPath, ".webcil"); + config.Assets.Add(new AssemblyEntry(Path.GetFileName(assemblyPath))); if (DebugLevel != 0) { var pdb = assembly; pdb = Path.ChangeExtension(pdb, ".pdb"); @@ -261,8 +278,21 @@ private bool ExecuteInternal () string name = Path.GetFileName(fullPath); string directory = Path.Combine(AppDir, config.AssemblyRootFolder, culture); Directory.CreateDirectory(directory); - FileCopyChecked(fullPath, Path.Combine(directory, name), "SatelliteAssemblies"); - config.Assets.Add(new SatelliteAssemblyEntry(name, culture)); + if (UseWebcil) + { + var tmpWebcil = Path.GetTempFileName(); + var webcilWriter = Microsoft.WebAssembly.Build.Tasks.WebcilConverter.FromPortableExecutable(inputPath: fullPath, outputPath: tmpWebcil, logger: Log); + webcilWriter.ConvertToWebcil(); + var finalWebcil = Path.ChangeExtension(name, ".webcil"); + FileCopyChecked(tmpWebcil, Path.Combine(directory, finalWebcil), "SatelliteAssemblies"); + config.Assets.Add(new SatelliteAssemblyEntry(finalWebcil, culture)); + } + else + { + FileCopyChecked(fullPath, Path.Combine(directory, name), "SatelliteAssemblies"); + config.Assets.Add(new SatelliteAssemblyEntry(name, culture)); + } + } } diff --git a/src/tasks/WasmAppBuilder/WasmAppBuilder.csproj b/src/tasks/WasmAppBuilder/WasmAppBuilder.csproj index c18167fcd2149..3fd2f17c30a25 100644 --- a/src/tasks/WasmAppBuilder/WasmAppBuilder.csproj +++ b/src/tasks/WasmAppBuilder/WasmAppBuilder.csproj @@ -7,6 +7,7 @@ $(NoWarn),CS8604,CS8602 false true + true @@ -20,6 +21,13 @@ + + + + + + + @@ -29,6 +37,9 @@ TargetPath="tasks\$(TargetFrameworkForNETCoreTasks)" /> + + +/// Reads a .NET assembly in a normal PE COFF file and writes it out as a Webcil file +/// +public class WebcilConverter +{ + private readonly string _inputPath; + private readonly string _outputPath; + + private readonly NET.WebAssembly.Webcil.WebcilConverter _converter; + + private TaskLoggingHelper Log { get; } + private WebcilConverter(NET.WebAssembly.Webcil.WebcilConverter converter, string inputPath, string outputPath, TaskLoggingHelper logger) + { + _converter = converter; + _inputPath = inputPath; + _outputPath = outputPath; + Log = logger; + } + + public static WebcilConverter FromPortableExecutable(string inputPath, string outputPath, TaskLoggingHelper logger) + { + var converter = NET.WebAssembly.Webcil.WebcilConverter.FromPortableExecutable(inputPath, outputPath); + return new WebcilConverter(converter, inputPath, outputPath, logger); + } + + public void ConvertToWebcil() + { + Log.LogMessage(MessageImportance.Low, $"Converting to Webcil: input {_inputPath} output: {_outputPath}"); + _converter.ConvertToWebcil(); + } + +}