From c55c69645da9aa183ff9bafa596ada3a3446fe36 Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Wed, 7 Dec 2022 15:43:59 -0500 Subject: [PATCH 01/57] Implement WebCIL loader; on by default It will try to look for WebCIL formatted images in normal .dll files --- src/mono/cmake/config.h.in | 3 + src/mono/cmake/options.cmake | 1 + src/mono/mono/metadata/CMakeLists.txt | 2 + src/mono/mono/metadata/image.c | 5 + src/mono/mono/metadata/webcil-loader.c | 135 +++++++++++++++++++++++++ src/mono/mono/metadata/webcil-loader.h | 11 ++ 6 files changed, 157 insertions(+) create mode 100644 src/mono/mono/metadata/webcil-loader.c create mode 100644 src/mono/mono/metadata/webcil-loader.h diff --git a/src/mono/cmake/config.h.in b/src/mono/cmake/config.h.in index ca8e02ac6fd88..069b4ba5d9400 100644 --- a/src/mono/cmake/config.h.in +++ b/src/mono/cmake/config.h.in @@ -933,6 +933,9 @@ /* Enable System.WeakAttribute support */ #cmakedefine ENABLE_WEAK_ATTR 1 +/* Disable WebCIL image loader */ +#cmakedefine DISABLE_WEBCIL 1 + #if defined(ENABLE_LLVM) && defined(HOST_WIN32) && defined(TARGET_WIN32) && (!defined(TARGET_AMD64) || !defined(_MSC_VER)) #error LLVM for host=Windows and target=Windows is only supported on x64 MSVC build. #endif diff --git a/src/mono/cmake/options.cmake b/src/mono/cmake/options.cmake index f00430fc9500d..b7ca5648e2d4d 100644 --- a/src/mono/cmake/options.cmake +++ b/src/mono/cmake/options.cmake @@ -57,6 +57,7 @@ option (ENABLE_SIGALTSTACK "Enable support for using sigaltstack for SIGSEGV and option (USE_MALLOC_FOR_MEMPOOLS "Use malloc for each single mempool allocation, so tools like Valgrind can run better") option (STATIC_COMPONENTS "Compile mono runtime components as static (not dynamic) libraries") option (DISABLE_WASM_USER_THREADS "Disable creation of user managed threads on WebAssembly, only allow runtime internal managed and native threads") +option (DISABLE_WEBCIL "Disable the WebCIL loader") set (MONO_GC "sgen" CACHE STRING "Garbage collector implementation (sgen or boehm). Default: sgen") set (GC_SUSPEND "default" CACHE STRING "GC suspend method (default, preemptive, coop, hybrid)") diff --git a/src/mono/mono/metadata/CMakeLists.txt b/src/mono/mono/metadata/CMakeLists.txt index 67f7a66175b6e..797637a566cbc 100644 --- a/src/mono/mono/metadata/CMakeLists.txt +++ b/src/mono/mono/metadata/CMakeLists.txt @@ -78,6 +78,8 @@ set(metadata_common_sources icall-eventpipe.c image.c image-internals.h + webcil-loader.h + webcil-loader.c jit-info.h jit-info.c loader.c diff --git a/src/mono/mono/metadata/image.c b/src/mono/mono/metadata/image.c index 61fac42a5d4e9..8ea3cbe84464d 100644 --- a/src/mono/mono/metadata/image.c +++ b/src/mono/mono/metadata/image.c @@ -46,6 +46,7 @@ #include #include #include +#include #include #include #ifdef HAVE_UNISTD_H @@ -259,6 +260,10 @@ mono_images_init (void) install_pe_loader (); +#ifndef DISABLE_WEBCIL + mono_webcil_loader_install (); +#endif + mutex_inited = TRUE; } diff --git a/src/mono/mono/metadata/webcil-loader.c b/src/mono/mono/metadata/webcil-loader.c new file mode 100644 index 0000000000000..985daefb37a3e --- /dev/null +++ b/src/mono/mono/metadata/webcil-loader.c @@ -0,0 +1,135 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// + +#include + +#include + +#include "mono/metadata/metadata-internals.h" +#include "mono/metadata/webcil-loader.h" + +/* keep in sync with webcil-writer */ +enum { + MONO_WEBCIL_VERSION = 0, +}; + +typedef struct MonoWebCilHeader { + uint8_t id[2]; // 'W' 'C' + uint8_t version; + uint8_t reserved0; // 0 + // 4 bytes + uint16_t coff_sections; + uint16_t reserved1; // 0 + // 8 bytes + + uint32_t metadata_rva; + uint32_t metadata_size; + // 16 bytes + + uint32_t cli_flags; + int32_t cli_entry_point; + // 24 bytes + + uint32_t pe_cli_header_rva; + uint32_t pe_cli_header_size; + // 32 bytes +} MonoWebCilHeader; + +static gboolean +webcil_image_match (MonoImage *image) +{ + if (image->raw_data_len >= sizeof (MonoWebCilHeader)) { + return image->raw_data[0] == 'W' && image->raw_data[1] == 'C'; + } + return FALSE; +} + +static gboolean +webcil_image_load_pe_data (MonoImage *image) +{ + MonoCLIImageInfo *iinfo; + MonoDotNetHeader *header; + MonoWebCilHeader wcheader; + gint32 offset = 0; + int i, top; + char d [16 * 8]; + char *p; + + iinfo = image->image_info; + header = &iinfo->cli_header; + + if (offset + sizeof (MonoWebCilHeader) > image->raw_data_len) + goto invalid_image; + memcpy (&wcheader, image->raw_data + offset, sizeof (wcheader)); + + if (!(wcheader.id [0] == 'W' && wcheader.id [1] == 'C' && wcheader.version == MONO_WEBCIL_VERSION)) + goto invalid_image; + + header->coff.coff_sections = GUINT16_FROM_LE (wcheader.coff_sections); + header->datadir.pe_cli_header.rva = GUINT32_FROM_LE (wcheader.pe_cli_header_rva); + header->datadir.pe_cli_header.size = GUINT32_FROM_LE (wcheader.pe_cli_header_size); + + top = iinfo->cli_header.coff.coff_sections; + + iinfo->cli_section_count = top; + iinfo->cli_section_tables = g_new0 (MonoSectionTable, top); + iinfo->cli_sections = g_new0 (void *, top); + + offset += sizeof (wcheader); + g_assert (top < 8); + p = d; + memcpy (d, image->raw_data + offset, top * 16); + for (i = 0; i < top; i++) { + MonoSectionTable *t = &iinfo->cli_section_tables [i]; + guint32 st [4]; + + memcpy (st, p, sizeof (st)); + t->st_virtual_size = GUINT32_FROM_LE (st [0]); + t->st_virtual_address = GUINT32_FROM_LE (st [1]); + t->st_raw_data_size = GUINT32_FROM_LE (st [2]); + t->st_raw_data_ptr = GUINT32_FROM_LE (st [3]); + p += 16; + } + + return TRUE; + +invalid_image: + return FALSE; + +} + +static gboolean +webcil_image_load_cli_data (MonoImage *image) +{ + MonoCLIImageInfo *iinfo; + + iinfo = image->image_info; + + if (!mono_image_load_cli_header (image, iinfo)) + return FALSE; + + if (!mono_image_load_metadata (image, iinfo)) + return FALSE; + + return TRUE; +} + +static gboolean +webcil_image_load_tables (MonoImage *image) +{ + return TRUE; +} + +static const MonoImageLoader webcil_loader = { + webcil_image_match, + webcil_image_load_pe_data, + webcil_image_load_cli_data, + webcil_image_load_tables, +}; + +void +mono_webcil_loader_install (void) +{ + mono_install_image_loader (&webcil_loader); +} diff --git a/src/mono/mono/metadata/webcil-loader.h b/src/mono/mono/metadata/webcil-loader.h new file mode 100644 index 0000000000000..3357d140cb50e --- /dev/null +++ b/src/mono/mono/metadata/webcil-loader.h @@ -0,0 +1,11 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// + +#ifndef _MONO_METADATA_WEBCIL_LOADER_H +#define _MONO_METADATA_WEBCIL_LOADER_H + +void +mono_webcil_loader_install (void); + +#endif /*_MONO_METADATA_WEBCIL_LOADER_H*/ From 45bc3c988dc70617b28019fa8e03d931bbfcd0bf Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Thu, 8 Dec 2022 15:44:26 -0500 Subject: [PATCH 02/57] Checkpoint works on wasm sample; add design doc --- docs/design/mono/webcil.md | 112 +++++++++++ src/mono/mono/metadata/assembly.c | 19 ++ src/mono/mono/metadata/webcil-loader.c | 8 +- src/mono/mono/mini/monovm.c | 16 ++ src/tasks/WasmAppBuilder/WasmAppBuilder.cs | 25 ++- .../WasmAppBuilder/WasmAppBuilder.csproj | 4 + .../Webcil/CoffSectionHeaderBuilder.cs | 27 +++ src/tasks/WasmAppBuilder/Webcil/Constants.cs | 9 + src/tasks/WasmAppBuilder/Webcil/WCHeader.cs | 38 ++++ src/tasks/WasmAppBuilder/WebcilWriter.cs | 180 ++++++++++++++++++ 10 files changed, 431 insertions(+), 7 deletions(-) create mode 100644 docs/design/mono/webcil.md create mode 100644 src/tasks/WasmAppBuilder/Webcil/CoffSectionHeaderBuilder.cs create mode 100644 src/tasks/WasmAppBuilder/Webcil/Constants.cs create mode 100644 src/tasks/WasmAppBuilder/Webcil/WCHeader.cs create mode 100644 src/tasks/WasmAppBuilder/WebcilWriter.cs diff --git a/docs/design/mono/webcil.md b/docs/design/mono/webcil.md new file mode 100644 index 0000000000000..a594cb7162802 --- /dev/null +++ b/docs/design/mono/webcil.md @@ -0,0 +1,112 @@ +# WebCIL assembly format + +## Version + +This is version 0 of the Webcil format. + +## Motivation + +When deploying the .NET runtime to the browser using WebAssembly, we have received some reports from +customers that certain users are unable to use their apps because firewalls and anti-virus software +may prevent browsers from downloading or caching assemblies with a .DLL extension and PE contents. + +This document defines a new container format for ECMA-335 assemblies +that uses the `.webcil` extension and uses a new WebCIL container +format. + + +## Specification + +As our starting point we take section II.25.1 "Structure of the +runtime file format" from ECMA-335 6th Edition. + +|--------| +| PE Headers | +| CLI Header | +| CLI Data | +| Native Image Sections | + + +A Webcil file follows a similar structure + + +|--------| +| Webcil Headers | +| CLI Header | +| CLI Data | + +## Webcil Headers + +The Webcil headers consist of a Webcil header followed by a sequence of section headers. +(All multi-byte integers are in little endian format). + +### Webcil Header + +``` c +struct WebcilHeader { + uint8_t id[2]; // 'W' 'C' + uint8_t version; + uint8_t reserved0; // 0 + // 4 bytes + uint16_t coff_sections; + uint16_t reserved1; // 0 + // 8 bytes + + uint32_t metadata_rva; + uint32_t metadata_size; + // 16 bytes + + uint32_t cli_flags; + int32_t cli_entry_point; + // 24 bytes + + uint32_t pe_cli_header_rva; + uint32_t pe_cli_header_size; + // 32 bytes +}; +``` + +The Webcil header starts with the magic characters 'W' 'C' followed by the version (must be 0). +Then a reserved byte that must be 0 followed by a count of the section headers and 2 more reserved bytes. + +The next 4 integers are a subset of ECMA-335 II.25.3.3 (CLI header) containing the RVA and the size +of the ECMA-335 metadata root, the Flags and EntryPointToken values. The runtime treats all other +CLI header values as their default values or zero. + +**FIXME** why do we need the CLI header fields if we're copying the CLI header anyway? + +The last 2 integers are a subset of the PE Header data directory specifying the RVA and size of the CLI header. + + +### Section header table + +Immediately following the Webcil header is a sequence (whose length is given by `coff_sections` +above) of section headers giving their virtual address and virtual size, as well as the offset in the Webcil +file and the size in the file. + +``` c +struct SectionTable { + uint32_t st_virtual_size; + uint32_t st_virtual_address; + uint32_t st_raw_data_size; + uint32_t st_raw_data_ptr; +}; +``` + +### Sections + +Immediately following the section table are the sections. These are copied verbatim from the PE file. + +## Rationale + +The intention is to include only the information necessary for the runtime to locate the metadata +root, and to resolve the RVA references in the metadata (for locating data declarations and method IL). + +A goal is for the files not to be executable by .NET Framework. + +Unlike PE files, mixing native and managed code is not a goal. + +Lossless conversion from Webcil back to PE is not intended to be supported. The format is being +documented in order to support diagnostic tooling and utilities such as decompilers, disassemblers, +file identification utilities, dependency analyzers, etc. + diff --git a/src/mono/mono/metadata/assembly.c b/src/mono/mono/metadata/assembly.c index c7be11b4c7af0..e5b736c548e9f 100644 --- a/src/mono/mono/metadata/assembly.c +++ b/src/mono/mono/metadata/assembly.c @@ -717,10 +717,21 @@ search_bundle_for_assembly (MonoAssemblyLoadContext *alc, MonoAssemblyName *anam MonoImage *image; MonoAssemblyLoadRequest req; image = mono_assembly_open_from_bundle (alc, aname->name, &status, aname->culture); +#ifndef DISABLE_WEBCIL + if (!image && !g_str_has_suffix (aname->name, ".webcil")) { + char *name = g_strdup_printf ("%s.webcil", aname->name); + image = mono_assembly_open_from_bundle (alc, name, &status, aname->culture); + g_free (name); + } +#endif if (!image && !g_str_has_suffix (aname->name, ".dll")) { char *name = g_strdup_printf ("%s.dll", aname->name); image = mono_assembly_open_from_bundle (alc, name, &status, aname->culture); + g_free (name); } + // Webcil: we could add some code here to look for .webcil files if they ask for .dll, but + // it's not clear if that should be a bug in the callers of mono_assembly_open. They + // shouldn't be passing an extension. if (image) { mono_assembly_request_prepare_load (&req, alc); return mono_assembly_request_load_from (image, aname->name, &req, &status); @@ -2708,6 +2719,14 @@ mono_assembly_load_corlib (void) corlib = mono_assembly_request_open (corlib_name, &req, &status); g_free (corlib_name); } +#ifndef DISABLE_WEBCIL + if (!corlib) { + /* Maybe its in a bundle */ + char *corlib_name = g_strdup_printf ("%s.webcil", MONO_ASSEMBLY_CORLIB_NAME); + corlib = mono_assembly_request_open (corlib_name, &req, &status); + g_free (corlib_name); + } +#endif g_assert (corlib); // exit the process if we weren't able to load corlib diff --git a/src/mono/mono/metadata/webcil-loader.c b/src/mono/mono/metadata/webcil-loader.c index 985daefb37a3e..7af6297d67017 100644 --- a/src/mono/mono/metadata/webcil-loader.c +++ b/src/mono/mono/metadata/webcil-loader.c @@ -23,12 +23,12 @@ typedef struct MonoWebCilHeader { uint16_t reserved1; // 0 // 8 bytes - uint32_t metadata_rva; - uint32_t metadata_size; + uint32_t metadata_rva; // FIXME: unused + uint32_t metadata_size; // FIXME: unused // 16 bytes - uint32_t cli_flags; - int32_t cli_entry_point; + uint32_t cli_flags; // FIXME: unused + int32_t cli_entry_point; // FIXME: unused // 24 bytes uint32_t pe_cli_header_rva; diff --git a/src/mono/mono/mini/monovm.c b/src/mono/mono/mini/monovm.c index 144594276f218..e31b9764d6257 100644 --- a/src/mono/mono/mini/monovm.c +++ b/src/mono/mono/mini/monovm.c @@ -131,6 +131,22 @@ mono_core_preload_hook (MonoAssemblyLoadContext *alc, MonoAssemblyName *aname, c if (result) break; } +#ifndef DISABLE_WEBCIL + else { + /* /path/foo.dll -> /path/foo.webcil */ + size_t n = strlen (fullpath) - 4; + char *fullpath2 = g_malloc (n + 8); + strncpy (fullpath2, fullpath, n); + strncpy (fullpath2 + n, ".webcil", 8); + if (g_file_test (fullpath2, G_FILE_TEST_IS_REGULAR)) { + MonoImageOpenStatus status; + result = mono_assembly_request_open (fullpath2, &req, &status); + } + g_free (fullpath2); + if (result) + break; + } +#endif } } diff --git a/src/tasks/WasmAppBuilder/WasmAppBuilder.cs b/src/tasks/WasmAppBuilder/WasmAppBuilder.cs index 2fead7b9860bf..bc313e3eb1483 100644 --- a/src/tasks/WasmAppBuilder/WasmAppBuilder.cs +++ b/src/tasks/WasmAppBuilder/WasmAppBuilder.cs @@ -48,6 +48,7 @@ public class WasmAppBuilder : Task public string? MainHTMLPath { get; set; } public bool IncludeThreadsWorker {get; set; } public int PThreadPoolSize {get; set; } + public bool DisableWebcil { get; set; } // // Extra json elements to add to mono-config.json @@ -177,10 +178,13 @@ private bool ExecuteInternal () _assemblies.Add(asm); } MainAssemblyName = Path.GetFileName(MainAssemblyName); + var mainAssemblyNameForConfig = MainAssemblyName; + if (!DisableWebcil) + mainAssemblyNameForConfig = Path.ChangeExtension(mainAssemblyNameForConfig, ".webcil"); var config = new WasmAppConfig () { - MainAssemblyName = MainAssemblyName, + MainAssemblyName = mainAssemblyNameForConfig, }; // Create app @@ -189,7 +193,18 @@ private bool ExecuteInternal () Directory.CreateDirectory(asmRootPath); foreach (var assembly in _assemblies) { - FileCopyChecked(assembly, Path.Combine(asmRootPath, Path.GetFileName(assembly)), "Assemblies"); + if (!DisableWebcil) + { + var tmpWebcil = Path.GetTempFileName(); + var webcilWriter = new Microsoft.WebAssembly.Build.Tasks.WebcilWriter(inputPath: assembly, outputPath: tmpWebcil, logger: Log); + webcilWriter.Write(); + var finalWebcil = Path.ChangeExtension(assembly, ".webcil"); + FileCopyChecked(tmpWebcil, Path.Combine(asmRootPath, Path.GetFileName(finalWebcil)), "Assemblies"); + } + else + { + FileCopyChecked(assembly, Path.Combine(asmRootPath, Path.GetFileName(assembly)), "Assemblies"); + } if (DebugLevel != 0) { var pdb = assembly; @@ -234,7 +249,10 @@ private bool ExecuteInternal () foreach (var assembly in _assemblies) { - config.Assets.Add(new AssemblyEntry(Path.GetFileName(assembly))); + string assemblyPath = assembly; + if (!DisableWebcil) + assemblyPath = Path.ChangeExtension(assemblyPath, ".webcil"); + config.Assets.Add(new AssemblyEntry(Path.GetFileName(assemblyPath))); if (DebugLevel != 0) { var pdb = assembly; pdb = Path.ChangeExtension(pdb, ".pdb"); @@ -247,6 +265,7 @@ private bool ExecuteInternal () if (SatelliteAssemblies != null) { + // FIXME: TODO: run WebcilWriter on satellite assemblies too foreach (var assembly in SatelliteAssemblies) { string culture = assembly.GetMetadata("CultureName") ?? string.Empty; diff --git a/src/tasks/WasmAppBuilder/WasmAppBuilder.csproj b/src/tasks/WasmAppBuilder/WasmAppBuilder.csproj index c18167fcd2149..242966a2914d3 100644 --- a/src/tasks/WasmAppBuilder/WasmAppBuilder.csproj +++ b/src/tasks/WasmAppBuilder/WasmAppBuilder.csproj @@ -7,6 +7,7 @@ $(NoWarn),CS8604,CS8602 false true + true @@ -20,6 +21,9 @@ + + + diff --git a/src/tasks/WasmAppBuilder/Webcil/CoffSectionHeaderBuilder.cs b/src/tasks/WasmAppBuilder/Webcil/CoffSectionHeaderBuilder.cs new file mode 100644 index 0000000000000..bc79259f8db98 --- /dev/null +++ b/src/tasks/WasmAppBuilder/Webcil/CoffSectionHeaderBuilder.cs @@ -0,0 +1,27 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Runtime.InteropServices; + +namespace Microsoft.WebAssembly.Build.Tasks.WebCil; + +/// +/// This is System.Reflection.PortableExecutable.CoffSectionHeaderBuilder, but with a public constructor so that +/// we can make our own copies of it. +/// +[StructLayout(LayoutKind.Sequential, Pack = 1)] +public struct CoffSectionHeaderBuilder +{ + public readonly int VirtualSize; + public readonly int VirtualAddress; + public readonly int SizeOfRawData; + public readonly int PointerToRawData; + + public CoffSectionHeaderBuilder(int virtualSize, int virtualAddress, int sizeOfRawData, int pointerToRawData) + { + VirtualSize = virtualSize; + VirtualAddress = virtualAddress; + SizeOfRawData = sizeOfRawData; + PointerToRawData = pointerToRawData; + } +} diff --git a/src/tasks/WasmAppBuilder/Webcil/Constants.cs b/src/tasks/WasmAppBuilder/Webcil/Constants.cs new file mode 100644 index 0000000000000..4ab2b548c34cc --- /dev/null +++ b/src/tasks/WasmAppBuilder/Webcil/Constants.cs @@ -0,0 +1,9 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.WebAssembly.Build.Tasks.WebCil; + +public static unsafe class Constants +{ + public const int WC_VERSION = 0; +} diff --git a/src/tasks/WasmAppBuilder/Webcil/WCHeader.cs b/src/tasks/WasmAppBuilder/Webcil/WCHeader.cs new file mode 100644 index 0000000000000..ef246eb30ba7b --- /dev/null +++ b/src/tasks/WasmAppBuilder/Webcil/WCHeader.cs @@ -0,0 +1,38 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Runtime.InteropServices; + +namespace Microsoft.WebAssembly.Build.Tasks.WebCil; + +/// +/// The header of a WebCIL file. +/// +/// +/// +/// The header is a subset of the PE, COFF and CLI headers that are needed by the mono runtime to load managed assemblies. +/// +[StructLayout(LayoutKind.Sequential, Pack = 1)] +public unsafe struct WCHeader +{ + public fixed byte id[2]; // 'W' 'C' + public byte version; + public byte reserved0; // 0 + // 4 bytes + + public ushort coff_sections; + public ushort reserved1; // 0 + // 8 bytes + + public uint metadata_rva; + public uint metadata_size; + // 16 bytes + + public uint cli_flags; + public int cli_entry_point; + // 24 bytes + + public uint pe_cli_header_rva; + public uint pe_cli_header_size; + // 32 bytes +} diff --git a/src/tasks/WasmAppBuilder/WebcilWriter.cs b/src/tasks/WasmAppBuilder/WebcilWriter.cs new file mode 100644 index 0000000000000..00b44c353214e --- /dev/null +++ b/src/tasks/WasmAppBuilder/WebcilWriter.cs @@ -0,0 +1,180 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; +using System.Collections.Immutable; +using System.Reflection.PortableExecutable; +using System.Runtime.InteropServices; + +using Microsoft.Build.Utilities; + +using Microsoft.WebAssembly.Build.Tasks.WebCil; + +namespace Microsoft.WebAssembly.Build.Tasks; + +/// +/// Reads a .NET assembly in a normal PE COFF file and writes it out as a Webcil file +/// +public class WebcilWriter +{ + private readonly string _inputPath; + private readonly string _outputPath; + + private TaskLoggingHelper Log { get; } + public WebcilWriter(string inputPath, string outputPath, TaskLoggingHelper logger) + { + _inputPath = inputPath; + _outputPath = outputPath; + Log = logger; + } + + public void Write() + { + Log.LogMessage($"Writing Webcil (input {_inputPath}) output to {_outputPath}"); + + using var inputStream = File.Open(_inputPath, FileMode.Open); + ImmutableArray sectionsHeaders; + ImmutableArray peSections; + WCHeader header; + using (var peReader = new PEReader(inputStream, PEStreamOptions.LeaveOpen)) + { + // DumpPE(peReader); + FillHeader(peReader, out header, out peSections, out sectionsHeaders); + } + + using var outputStream = File.Open(_outputPath, FileMode.Create); + WriteHeader(outputStream, header); + WriteSectionHeaders(outputStream, sectionsHeaders); + CopySections(outputStream, inputStream, peSections); + } + + + private record struct FilePosition(int Position) + { + public static implicit operator FilePosition(int position) => new(position); + + public static FilePosition operator +(FilePosition left, int right) => new(left.Position + right); + } + + private static unsafe int SizeOfHeader() + { + return sizeof(WCHeader); + } + + public static unsafe void FillHeader(PEReader peReader, out WCHeader header, out ImmutableArray peSections, out ImmutableArray sectionsHeaders) + { + var headers = peReader.PEHeaders; + var peHeader = headers.PEHeader!; + var coffHeader = headers.CoffHeader!; + var corHeader = headers.CorHeader!; + var sections = headers.SectionHeaders; + header.id[0] = (byte)'W'; + header.id[1] = (byte)'C'; + header.version = Constants.WC_VERSION; + header.reserved0 = 0; + header.coff_sections = (ushort)coffHeader.NumberOfSections; + header.reserved1 = 0; + header.metadata_rva = (uint)corHeader.MetadataDirectory.RelativeVirtualAddress; + header.metadata_size = (uint)corHeader.MetadataDirectory.Size; + header.cli_flags = (uint)corHeader.Flags; + header.cli_entry_point = corHeader.EntryPointTokenOrRelativeVirtualAddress; + header.pe_cli_header_rva = (uint)peHeader.CorHeaderTableDirectory.RelativeVirtualAddress; + header.pe_cli_header_size = (uint)peHeader.CorHeaderTableDirectory.Size; + + // current logical position in the output file + FilePosition pos = SizeOfHeader(); + // position of the current section in the output file + // initially it's after all the section headers + FilePosition curSectionPos = pos + sizeof(CoffSectionHeaderBuilder) * coffHeader.NumberOfSections; + + // TODO: write the sections, but adjust the raw data ptr to the offset after the WCHeader. + ImmutableArray.Builder headerBuilder = ImmutableArray.CreateBuilder(coffHeader.NumberOfSections); + foreach (var sectionHeader in sections) + { + var newHeader = new CoffSectionHeaderBuilder + ( + virtualSize: sectionHeader.VirtualSize, + virtualAddress: sectionHeader.VirtualAddress, + sizeOfRawData: sectionHeader.SizeOfRawData, + pointerToRawData: curSectionPos.Position + ); + + pos += sizeof(CoffSectionHeaderBuilder); + curSectionPos += sectionHeader.SizeOfRawData; + headerBuilder.Add(newHeader); + } + + peSections = sections; + sectionsHeaders = headerBuilder.ToImmutable(); + } + + private static void WriteHeader(Stream s, WCHeader header) + { + WriteStructure(s, header); + } + + private static void WriteSectionHeaders(Stream s, ImmutableArray sectionsHeaders) + { + // FIXME: fixup endianness + if (!BitConverter.IsLittleEndian) + throw new NotImplementedException(); + foreach (var sectionHeader in sectionsHeaders) + { + WriteSectionHeader(s, sectionHeader); + } + } + + private static void WriteSectionHeader(Stream s, CoffSectionHeaderBuilder sectionHeader) + { + WriteStructure(s, sectionHeader); + } + +#if NETCOREAPP2_1_OR_GREATER + private static void WriteStructure(Stream s, T structure) + where T : unmanaged + { + // FIXME: fixup endianness + if (!BitConverter.IsLittleEndian) + throw new NotImplementedException(); + unsafe + { + byte* p = (byte*)&structure; + s.Write(new ReadOnlySpan(p, sizeof(T))); + } + } +#else + private static void WriteStructure(Stream s, T structure) + where T : unmanaged + { + // FIXME: fixup endianness + if (!BitConverter.IsLittleEndian) + throw new NotImplementedException(); + int size = Marshal.SizeOf(); + byte[] buffer = new byte[size]; + IntPtr ptr = IntPtr.Zero; + try + { + ptr = Marshal.AllocHGlobal(size); + Marshal.StructureToPtr(structure, ptr, false); + Marshal.Copy(ptr, buffer, 0, size); + } + finally + { + Marshal.FreeHGlobal(ptr); + } + s.Write(buffer, 0, size); + } +#endif + + private static void CopySections(Stream outStream, Stream inputStream, ImmutableArray peSections) + { + // endianness: ok, we're just copying from one stream to another + foreach (var peHeader in peSections) + { + inputStream.Seek(peHeader.PointerToRawData, SeekOrigin.Begin); + inputStream.CopyTo(outStream, peHeader.SizeOfRawData); + } + } + +} From c2739da7fd3d3ad95f90c97bb9f496de9a96021f Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Thu, 8 Dec 2022 15:51:53 -0500 Subject: [PATCH 03/57] Remove unused fields --- docs/design/mono/webcil.md | 16 +--------------- src/mono/mono/metadata/webcil-loader.c | 10 +--------- src/tasks/WasmAppBuilder/Webcil/WCHeader.cs | 11 +---------- src/tasks/WasmAppBuilder/WebcilWriter.cs | 4 ---- 4 files changed, 3 insertions(+), 38 deletions(-) diff --git a/docs/design/mono/webcil.md b/docs/design/mono/webcil.md index a594cb7162802..ad29ce584f6f4 100644 --- a/docs/design/mono/webcil.md +++ b/docs/design/mono/webcil.md @@ -52,29 +52,15 @@ struct WebcilHeader { uint16_t reserved1; // 0 // 8 bytes - uint32_t metadata_rva; - uint32_t metadata_size; - // 16 bytes - - uint32_t cli_flags; - int32_t cli_entry_point; - // 24 bytes - uint32_t pe_cli_header_rva; uint32_t pe_cli_header_size; - // 32 bytes + // 16 bytes }; ``` The Webcil header starts with the magic characters 'W' 'C' followed by the version (must be 0). Then a reserved byte that must be 0 followed by a count of the section headers and 2 more reserved bytes. -The next 4 integers are a subset of ECMA-335 II.25.3.3 (CLI header) containing the RVA and the size -of the ECMA-335 metadata root, the Flags and EntryPointToken values. The runtime treats all other -CLI header values as their default values or zero. - -**FIXME** why do we need the CLI header fields if we're copying the CLI header anyway? - The last 2 integers are a subset of the PE Header data directory specifying the RVA and size of the CLI header. diff --git a/src/mono/mono/metadata/webcil-loader.c b/src/mono/mono/metadata/webcil-loader.c index 7af6297d67017..bbfc5b5f7917a 100644 --- a/src/mono/mono/metadata/webcil-loader.c +++ b/src/mono/mono/metadata/webcil-loader.c @@ -23,17 +23,9 @@ typedef struct MonoWebCilHeader { uint16_t reserved1; // 0 // 8 bytes - uint32_t metadata_rva; // FIXME: unused - uint32_t metadata_size; // FIXME: unused - // 16 bytes - - uint32_t cli_flags; // FIXME: unused - int32_t cli_entry_point; // FIXME: unused - // 24 bytes - uint32_t pe_cli_header_rva; uint32_t pe_cli_header_size; - // 32 bytes + // 16 bytes } MonoWebCilHeader; static gboolean diff --git a/src/tasks/WasmAppBuilder/Webcil/WCHeader.cs b/src/tasks/WasmAppBuilder/Webcil/WCHeader.cs index ef246eb30ba7b..639d7aab077c9 100644 --- a/src/tasks/WasmAppBuilder/Webcil/WCHeader.cs +++ b/src/tasks/WasmAppBuilder/Webcil/WCHeader.cs @@ -23,16 +23,7 @@ public unsafe struct WCHeader public ushort coff_sections; public ushort reserved1; // 0 // 8 bytes - - public uint metadata_rva; - public uint metadata_size; - // 16 bytes - - public uint cli_flags; - public int cli_entry_point; - // 24 bytes - public uint pe_cli_header_rva; public uint pe_cli_header_size; - // 32 bytes + // 16 bytes } diff --git a/src/tasks/WasmAppBuilder/WebcilWriter.cs b/src/tasks/WasmAppBuilder/WebcilWriter.cs index 00b44c353214e..caa99d7009376 100644 --- a/src/tasks/WasmAppBuilder/WebcilWriter.cs +++ b/src/tasks/WasmAppBuilder/WebcilWriter.cs @@ -75,10 +75,6 @@ public static unsafe void FillHeader(PEReader peReader, out WCHeader header, out header.reserved0 = 0; header.coff_sections = (ushort)coffHeader.NumberOfSections; header.reserved1 = 0; - header.metadata_rva = (uint)corHeader.MetadataDirectory.RelativeVirtualAddress; - header.metadata_size = (uint)corHeader.MetadataDirectory.Size; - header.cli_flags = (uint)corHeader.Flags; - header.cli_entry_point = corHeader.EntryPointTokenOrRelativeVirtualAddress; header.pe_cli_header_rva = (uint)peHeader.CorHeaderTableDirectory.RelativeVirtualAddress; header.pe_cli_header_size = (uint)peHeader.CorHeaderTableDirectory.Size; From c58670bebcd75c55156df7f06ab5810d081f79ae Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Thu, 8 Dec 2022 15:59:21 -0500 Subject: [PATCH 04/57] fixup markdown --- docs/design/mono/webcil.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/design/mono/webcil.md b/docs/design/mono/webcil.md index ad29ce584f6f4..9fd52d1db3d3d 100644 --- a/docs/design/mono/webcil.md +++ b/docs/design/mono/webcil.md @@ -20,20 +20,25 @@ format. As our starting point we take section II.25.1 "Structure of the runtime file format" from ECMA-335 6th Edition. +| | |--------| | PE Headers | | CLI Header | | CLI Data | | Native Image Sections | +| | + A Webcil file follows a similar structure +| | |--------| | Webcil Headers | | CLI Header | | CLI Data | +| | ## Webcil Headers @@ -71,7 +76,7 @@ above) of section headers giving their virtual address and virtual size, as well file and the size in the file. ``` c -struct SectionTable { +struct SectionHeader { uint32_t st_virtual_size; uint32_t st_virtual_address; uint32_t st_raw_data_size; From 851ebdee507d4905bc8ebe4ce147bfaa071115d9 Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Thu, 8 Dec 2022 15:59:46 -0500 Subject: [PATCH 05/57] remove unused var --- src/tasks/WasmAppBuilder/WebcilWriter.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/tasks/WasmAppBuilder/WebcilWriter.cs b/src/tasks/WasmAppBuilder/WebcilWriter.cs index caa99d7009376..e3cfea8ae81e2 100644 --- a/src/tasks/WasmAppBuilder/WebcilWriter.cs +++ b/src/tasks/WasmAppBuilder/WebcilWriter.cs @@ -67,7 +67,6 @@ public static unsafe void FillHeader(PEReader peReader, out WCHeader header, out var headers = peReader.PEHeaders; var peHeader = headers.PEHeader!; var coffHeader = headers.CoffHeader!; - var corHeader = headers.CorHeader!; var sections = headers.SectionHeaders; header.id[0] = (byte)'W'; header.id[1] = (byte)'C'; From fa812bec87eeea13a891317a07a6a77e9f78eb0c Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Thu, 8 Dec 2022 17:45:36 -0500 Subject: [PATCH 06/57] specify FileAccess --- src/tasks/WasmAppBuilder/WebcilWriter.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tasks/WasmAppBuilder/WebcilWriter.cs b/src/tasks/WasmAppBuilder/WebcilWriter.cs index e3cfea8ae81e2..89f6ccb3143ba 100644 --- a/src/tasks/WasmAppBuilder/WebcilWriter.cs +++ b/src/tasks/WasmAppBuilder/WebcilWriter.cs @@ -33,7 +33,7 @@ public void Write() { Log.LogMessage($"Writing Webcil (input {_inputPath}) output to {_outputPath}"); - using var inputStream = File.Open(_inputPath, FileMode.Open); + using var inputStream = File.Open(_inputPath, FileMode.Open, FileAccess.Read); ImmutableArray sectionsHeaders; ImmutableArray peSections; WCHeader header; @@ -43,7 +43,7 @@ public void Write() FillHeader(peReader, out header, out peSections, out sectionsHeaders); } - using var outputStream = File.Open(_outputPath, FileMode.Create); + using var outputStream = File.Open(_outputPath, FileMode.Create, FileAccess.Write); WriteHeader(outputStream, header); WriteSectionHeaders(outputStream, sectionsHeaders); CopySections(outputStream, inputStream, peSections); From 6a7b411b331c456e8d5309fb039e7246bcf07267 Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Thu, 8 Dec 2022 19:03:16 -0500 Subject: [PATCH 07/57] FileShare --- src/tasks/WasmAppBuilder/WebcilWriter.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tasks/WasmAppBuilder/WebcilWriter.cs b/src/tasks/WasmAppBuilder/WebcilWriter.cs index 89f6ccb3143ba..c25da9917f3e5 100644 --- a/src/tasks/WasmAppBuilder/WebcilWriter.cs +++ b/src/tasks/WasmAppBuilder/WebcilWriter.cs @@ -33,7 +33,7 @@ public void Write() { Log.LogMessage($"Writing Webcil (input {_inputPath}) output to {_outputPath}"); - using var inputStream = File.Open(_inputPath, FileMode.Open, FileAccess.Read); + using var inputStream = File.Open(_inputPath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); ImmutableArray sectionsHeaders; ImmutableArray peSections; WCHeader header; From 708368ba5210ab35c58e607db4cf4e6ab732db33 Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Fri, 9 Dec 2022 14:00:22 -0500 Subject: [PATCH 08/57] Push .dll->.webcil probing lower in the bundle logic --- src/mono/mono/metadata/assembly.c | 32 ++++++++++++++++++++----------- src/mono/mono/mini/monovm.c | 4 ++-- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/src/mono/mono/metadata/assembly.c b/src/mono/mono/metadata/assembly.c index e5b736c548e9f..bc7bb04b2a793 100644 --- a/src/mono/mono/metadata/assembly.c +++ b/src/mono/mono/metadata/assembly.c @@ -717,21 +717,11 @@ search_bundle_for_assembly (MonoAssemblyLoadContext *alc, MonoAssemblyName *anam MonoImage *image; MonoAssemblyLoadRequest req; image = mono_assembly_open_from_bundle (alc, aname->name, &status, aname->culture); -#ifndef DISABLE_WEBCIL - if (!image && !g_str_has_suffix (aname->name, ".webcil")) { - char *name = g_strdup_printf ("%s.webcil", aname->name); - image = mono_assembly_open_from_bundle (alc, name, &status, aname->culture); - g_free (name); - } -#endif if (!image && !g_str_has_suffix (aname->name, ".dll")) { char *name = g_strdup_printf ("%s.dll", aname->name); image = mono_assembly_open_from_bundle (alc, name, &status, aname->culture); g_free (name); } - // Webcil: we could add some code here to look for .webcil files if they ask for .dll, but - // it's not clear if that should be a bug in the callers of mono_assembly_open. They - // shouldn't be passing an extension. if (image) { mono_assembly_request_prepare_load (&req, alc); return mono_assembly_request_load_from (image, aname->name, &req, &status); @@ -1460,6 +1450,26 @@ absolute_dir (const gchar *filename) return res; } +static gboolean +bundled_assembly_match (const MonoBundledAssembly *b, const char *name) +{ +#ifdef DISABLE_WEBCIL + return strcmp (b->name, name) == 0; +#else + if (strcmp (b->name, name) == 0) + return TRUE; + /* if they want a .dll and we have the matching .webcil, return it */ + if (g_str_has_suffix (b->name, ".webcil") && g_str_has_suffix (name, ".dll")) { + size_t bprefix = strlen (b->name) - 7; + size_t nprefix = strlen (name) - 4; + size_t longer = MAX(bprefix, nprefix); + if (strncmp (b->name, name, longer) == 0) + return TRUE; + } + return FALSE; +#endif +} + static MonoImage * open_from_bundle_internal (MonoAssemblyLoadContext *alc, const char *filename, MonoImageOpenStatus *status, gboolean is_satellite) { @@ -1469,7 +1479,7 @@ open_from_bundle_internal (MonoAssemblyLoadContext *alc, const char *filename, M MonoImage *image = NULL; char *name = is_satellite ? g_strdup (filename) : g_path_get_basename (filename); for (int i = 0; !image && bundles [i]; ++i) { - if (strcmp (bundles [i]->name, name) == 0) { + if (bundled_assembly_match (bundles[i], name)) { // Since bundled images don't exist on disk, don't give them a legit filename image = mono_image_open_from_data_internal (alc, (char*)bundles [i]->data, bundles [i]->size, FALSE, status, FALSE, name, NULL); break; diff --git a/src/mono/mono/mini/monovm.c b/src/mono/mono/mini/monovm.c index e31b9764d6257..a70f2fd560c41 100644 --- a/src/mono/mono/mini/monovm.c +++ b/src/mono/mono/mini/monovm.c @@ -136,8 +136,8 @@ mono_core_preload_hook (MonoAssemblyLoadContext *alc, MonoAssemblyName *aname, c /* /path/foo.dll -> /path/foo.webcil */ size_t n = strlen (fullpath) - 4; char *fullpath2 = g_malloc (n + 8); - strncpy (fullpath2, fullpath, n); - strncpy (fullpath2 + n, ".webcil", 8); + g_strlcpy (fullpath2, fullpath, n + 1); + g_strlcpy (fullpath2 + n, ".webcil", 8); if (g_file_test (fullpath2, G_FILE_TEST_IS_REGULAR)) { MonoImageOpenStatus status; result = mono_assembly_request_open (fullpath2, &req, &status); From 509c219b7af22d11aa859c05bf4f64d8308b2cb1 Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Fri, 9 Dec 2022 14:51:54 -0500 Subject: [PATCH 09/57] Also convert satellite assemblies --- src/tasks/WasmAppBuilder/WasmAppBuilder.cs | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/tasks/WasmAppBuilder/WasmAppBuilder.cs b/src/tasks/WasmAppBuilder/WasmAppBuilder.cs index bc313e3eb1483..6ca875fe54f94 100644 --- a/src/tasks/WasmAppBuilder/WasmAppBuilder.cs +++ b/src/tasks/WasmAppBuilder/WasmAppBuilder.cs @@ -265,7 +265,6 @@ private bool ExecuteInternal () if (SatelliteAssemblies != null) { - // FIXME: TODO: run WebcilWriter on satellite assemblies too foreach (var assembly in SatelliteAssemblies) { string culture = assembly.GetMetadata("CultureName") ?? string.Empty; @@ -280,7 +279,19 @@ private bool ExecuteInternal () string name = Path.GetFileName(fullPath); string directory = Path.Combine(AppDir, config.AssemblyRootFolder, culture); Directory.CreateDirectory(directory); - FileCopyChecked(fullPath, Path.Combine(directory, name), "SatelliteAssemblies"); + if (!DisableWebcil) + { + var tmpWebcil = Path.GetTempFileName(); + var webcilWriter = new Microsoft.WebAssembly.Build.Tasks.WebcilWriter(inputPath: fullPath, outputPath: tmpWebcil, logger: Log); + webcilWriter.Write(); + var finalWebcil = Path.ChangeExtension(name, ".webcil"); + FileCopyChecked(tmpWebcil, Path.Combine(directory, finalWebcil), "SatelliteAssemblies"); + } + else + { + FileCopyChecked(fullPath, Path.Combine(directory, name), "SatelliteAssemblies"); + } + config.Assets.Add(new SatelliteAssemblyEntry(name, culture)); } } From 79d07704ca4a8abd56525302981650d915bacfae Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Fri, 9 Dec 2022 15:02:51 -0500 Subject: [PATCH 10/57] satellite matching --- src/mono/mono/metadata/assembly.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/mono/mono/metadata/assembly.c b/src/mono/mono/metadata/assembly.c index bc7bb04b2a793..274a55d96605a 100644 --- a/src/mono/mono/metadata/assembly.c +++ b/src/mono/mono/metadata/assembly.c @@ -1451,19 +1451,19 @@ absolute_dir (const gchar *filename) } static gboolean -bundled_assembly_match (const MonoBundledAssembly *b, const char *name) +bundled_assembly_match (const char *bundled_name, const char *name) { #ifdef DISABLE_WEBCIL - return strcmp (b->name, name) == 0; + return strcmp (bundled_name, name) == 0; #else - if (strcmp (b->name, name) == 0) + if (strcmp (bundled_name, name) == 0) return TRUE; /* if they want a .dll and we have the matching .webcil, return it */ - if (g_str_has_suffix (b->name, ".webcil") && g_str_has_suffix (name, ".dll")) { - size_t bprefix = strlen (b->name) - 7; + if (g_str_has_suffix (bundled_name, ".webcil") && g_str_has_suffix (name, ".dll")) { + size_t bprefix = strlen (bundled_name) - 7; size_t nprefix = strlen (name) - 4; size_t longer = MAX(bprefix, nprefix); - if (strncmp (b->name, name, longer) == 0) + if (strncmp (bundled_name, name, longer) == 0) return TRUE; } return FALSE; @@ -1479,7 +1479,7 @@ open_from_bundle_internal (MonoAssemblyLoadContext *alc, const char *filename, M MonoImage *image = NULL; char *name = is_satellite ? g_strdup (filename) : g_path_get_basename (filename); for (int i = 0; !image && bundles [i]; ++i) { - if (bundled_assembly_match (bundles[i], name)) { + if (bundled_assembly_match (bundles[i]->name, name)) { // Since bundled images don't exist on disk, don't give them a legit filename image = mono_image_open_from_data_internal (alc, (char*)bundles [i]->data, bundles [i]->size, FALSE, status, FALSE, name, NULL); break; @@ -1500,7 +1500,7 @@ open_from_satellite_bundle (MonoAssemblyLoadContext *alc, const char *filename, char *name = g_strdup (filename); for (int i = 0; !image && satellite_bundles [i]; ++i) { - if (strcmp (satellite_bundles [i]->name, name) == 0 && strcmp (satellite_bundles [i]->culture, culture) == 0) { + if (bundled_assembly_match (satellite_bundles[i]->name, name) && strcmp (satellite_bundles [i]->culture, culture) == 0) { char *bundle_name = g_strconcat (culture, "/", name, (const char *)NULL); image = mono_image_open_from_data_internal (alc, (char *)satellite_bundles [i]->data, satellite_bundles [i]->size, FALSE, status, FALSE, bundle_name, NULL); g_free (bundle_name); From 38ddc99b6447a379003f05240b86f5f6ad840d32 Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Fri, 9 Dec 2022 15:08:23 -0500 Subject: [PATCH 11/57] fixup asset entry --- src/tasks/WasmAppBuilder/WasmAppBuilder.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/tasks/WasmAppBuilder/WasmAppBuilder.cs b/src/tasks/WasmAppBuilder/WasmAppBuilder.cs index 6ca875fe54f94..e3c46ed75484e 100644 --- a/src/tasks/WasmAppBuilder/WasmAppBuilder.cs +++ b/src/tasks/WasmAppBuilder/WasmAppBuilder.cs @@ -286,13 +286,14 @@ private bool ExecuteInternal () webcilWriter.Write(); var finalWebcil = Path.ChangeExtension(name, ".webcil"); FileCopyChecked(tmpWebcil, Path.Combine(directory, finalWebcil), "SatelliteAssemblies"); + config.Assets.Add(new SatelliteAssemblyEntry(finalWebcil, culture)); } else { FileCopyChecked(fullPath, Path.Combine(directory, name), "SatelliteAssemblies"); + config.Assets.Add(new SatelliteAssemblyEntry(name, culture)); } - config.Assets.Add(new SatelliteAssemblyEntry(name, culture)); } } From 91bbbceffa2162445a4694362294d930017a4287 Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Mon, 19 Dec 2022 13:17:15 -0500 Subject: [PATCH 12/57] [wasm] don't leak .webcil image names to the debugger In particular this will make source and breakpoint URLs look like `dotnet://foo.dll/Foo.cs` which means that grabbing PDBs via source link will work, etc. --- src/mono/wasm/debugger/BrowserDebugProxy/MonoSDBHelper.cs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/mono/wasm/debugger/BrowserDebugProxy/MonoSDBHelper.cs b/src/mono/wasm/debugger/BrowserDebugProxy/MonoSDBHelper.cs index 055c44880842e..91e02eb15fdaa 100644 --- a/src/mono/wasm/debugger/BrowserDebugProxy/MonoSDBHelper.cs +++ b/src/mono/wasm/debugger/BrowserDebugProxy/MonoSDBHelper.cs @@ -1205,7 +1205,13 @@ public async Task GetAssemblyName(int assembly_id, CancellationToken tok commandParamsWriter.Write(assembly_id); using var retDebuggerCmdReader = await SendDebuggerAgentCommand(CmdAssembly.GetLocation, commandParamsWriter, token); - return retDebuggerCmdReader.ReadString(); + string result = retDebuggerCmdReader.ReadString(); + if (result.EndsWith(".webcil")) { + /* don't leak .webcil names to the debugger - work in terms of the original .dlls */ + string baseName = result.Substring(0, result.Length - 7); + result = baseName + ".dll"; + } + return result; } public async Task GetFullAssemblyName(int assemblyId, CancellationToken token) From c2650d261273741f5aac2730f929d319b3c8ea03 Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Tue, 20 Dec 2022 14:28:38 -0500 Subject: [PATCH 13/57] fix one other PE file check --- src/mono/mono/metadata/reflection.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mono/mono/metadata/reflection.c b/src/mono/mono/metadata/reflection.c index 487a31f4ec317..bd437db42a161 100644 --- a/src/mono/mono/metadata/reflection.c +++ b/src/mono/mono/metadata/reflection.c @@ -1285,7 +1285,7 @@ method_body_object_construct (MonoClass *unused_class, MonoMethod *method, gpoin if ((method->flags & METHOD_ATTRIBUTE_PINVOKE_IMPL) || (method->flags & METHOD_ATTRIBUTE_ABSTRACT) || (method->iflags & METHOD_IMPL_ATTRIBUTE_INTERNAL_CALL) || - (image->raw_data && image->raw_data [1] != 'Z') || + (image->raw_data && (image->raw_data [1] != 'Z' && image->raw_data [1] != 'C')) || (method->iflags & METHOD_IMPL_ATTRIBUTE_RUNTIME)) return MONO_HANDLE_CAST (MonoReflectionMethodBody, NULL_HANDLE); From b045d0477bebac766f62c972664073bf7f25a511 Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Thu, 22 Dec 2022 12:27:07 -0500 Subject: [PATCH 14/57] Add PE DebugTableDirectory to webcil This is used to retrieve the PPDB data and/or the PDB checksum from an image. Refactor mono_has_pdb_checksum to support webcil in addition to PE images --- docs/design/mono/webcil.md | 13 ++- src/mono/mono/metadata/image.c | 59 +++++++++---- src/mono/mono/metadata/webcil-loader.c | 92 +++++++++++++++------ src/mono/mono/metadata/webcil-loader.h | 6 ++ src/tasks/WasmAppBuilder/Webcil/WCHeader.cs | 3 + src/tasks/WasmAppBuilder/WebcilWriter.cs | 3 +- 6 files changed, 130 insertions(+), 46 deletions(-) diff --git a/docs/design/mono/webcil.md b/docs/design/mono/webcil.md index 9fd52d1db3d3d..fe6bf8f4c52f2 100644 --- a/docs/design/mono/webcil.md +++ b/docs/design/mono/webcil.md @@ -60,20 +60,27 @@ struct WebcilHeader { uint32_t pe_cli_header_rva; uint32_t pe_cli_header_size; // 16 bytes + + uint32_t pe_debug_rva; + uint32_t pe_debug_size; + // 24 bytes }; ``` The Webcil header starts with the magic characters 'W' 'C' followed by the version (must be 0). Then a reserved byte that must be 0 followed by a count of the section headers and 2 more reserved bytes. -The last 2 integers are a subset of the PE Header data directory specifying the RVA and size of the CLI header. +The next pairs of integers are a subset of the PE Header data directory specifying the RVA and size +of the CLI header, as well as the directory entry for the PE debug directory. ### Section header table Immediately following the Webcil header is a sequence (whose length is given by `coff_sections` -above) of section headers giving their virtual address and virtual size, as well as the offset in the Webcil -file and the size in the file. +above) of section headers giving their virtual address and virtual size, as well as the offset in +the Webcil file and the size in the file. This is a subset of the PE section header that includes +enough information to correctly interpret the RVAs from the webcil header and from the .NET +metadata. Other information (such as the section names) are not included. ``` c struct SectionHeader { diff --git a/src/mono/mono/metadata/image.c b/src/mono/mono/metadata/image.c index 8ea3cbe84464d..6c85f26f7c8fd 100644 --- a/src/mono/mono/metadata/image.c +++ b/src/mono/mono/metadata/image.c @@ -928,26 +928,44 @@ do_load_header (MonoImage *image, MonoDotNetHeader *header, int offset) return offset; } -mono_bool -mono_has_pdb_checksum (char *raw_data, uint32_t raw_data_len) +static int32_t +try_load_pe_cli_header (char *raw_data, uint32_t raw_data_len, MonoDotNetHeader *cli_header) { - MonoDotNetHeader cli_header; MonoMSDOSHeader msdos; - guint8 *data; int offset = 0; memcpy (&msdos, raw_data + offset, sizeof (msdos)); if (!(msdos.msdos_sig [0] == 'M' && msdos.msdos_sig [1] == 'Z')) { - return FALSE; + return -1; } msdos.pe_offset = GUINT32_FROM_LE (msdos.pe_offset); offset = msdos.pe_offset; - int ret = do_load_header_internal (raw_data, raw_data_len, &cli_header, offset, FALSE); - if ( ret >= 0 ) { + int32_t ret = do_load_header_internal (raw_data, raw_data_len, cli_header, offset, FALSE); + return ret; +} + + +mono_bool +mono_has_pdb_checksum (char *raw_data, uint32_t raw_data_len) +{ + guint8 *data; + MonoDotNetHeader cli_header; + gboolean is_pe = TRUE; + + int32_t ret = try_load_pe_cli_header (raw_data, raw_data_len, &cli_header); + +#ifndef DISABLE_WEBCIL + if (ret == -1) { + ret = mono_webcil_load_cli_header (raw_data, raw_data_len, 0, &cli_header); + is_pe = FALSE; + } +#endif + + if (ret > 0) { MonoPEDirEntry *debug_dir_entry = (MonoPEDirEntry *) &cli_header.datadir.pe_debug; ImageDebugDirectory debug_dir; if (!debug_dir_entry->size) @@ -956,28 +974,39 @@ mono_has_pdb_checksum (char *raw_data, uint32_t raw_data_len) const int top = cli_header.coff.coff_sections; guint32 addr = debug_dir_entry->rva; int i = 0; + gboolean found = FALSE; for (i = 0; i < top; i++){ MonoSectionTable t; - if (ret + sizeof (MonoSectionTable) > raw_data_len) { - return FALSE; - } + if (G_LIKELY (is_pe)) { + if (ret + sizeof (MonoSectionTable) > raw_data_len) + return FALSE; - memcpy (&t, raw_data + ret, sizeof (MonoSectionTable)); - ret += sizeof (MonoSectionTable); + memcpy (&t, raw_data + ret, sizeof (MonoSectionTable)); + ret += sizeof (MonoSectionTable); #if G_BYTE_ORDER != G_LITTLE_ENDIAN - t.st_virtual_address = GUINT32_FROM_LE (t.st_virtual_address); - t.st_raw_data_size = GUINT32_FROM_LE (t.st_raw_data_size); - t.st_raw_data_ptr = GUINT32_FROM_LE (t.st_raw_data_ptr); + t.st_virtual_address = GUINT32_FROM_LE (t.st_virtual_address); + t.st_raw_data_size = GUINT32_FROM_LE (t.st_raw_data_size); + t.st_raw_data_ptr = GUINT32_FROM_LE (t.st_raw_data_ptr); #endif + } +#ifndef DISABLE_WEBCIL + else { + ret = mono_webcil_load_section_table (raw_data, raw_data_len, ret, &t); + if (ret == -1) + return FALSE; + } +#endif /* consistency checks here */ if ((addr >= t.st_virtual_address) && (addr < t.st_virtual_address + t.st_raw_data_size)){ addr = addr - t.st_virtual_address + t.st_raw_data_ptr; + found = TRUE; break; } } + g_assert (found); for (guint32 idx = 0; idx < debug_dir_entry->size / sizeof (ImageDebugDirectory); ++idx) { data = (guint8 *) ((ImageDebugDirectory *) (raw_data + addr) + idx); debug_dir.characteristics = read32(data); diff --git a/src/mono/mono/metadata/webcil-loader.c b/src/mono/mono/metadata/webcil-loader.c index bbfc5b5f7917a..915628a0ac9c2 100644 --- a/src/mono/mono/metadata/webcil-loader.c +++ b/src/mono/mono/metadata/webcil-loader.c @@ -26,6 +26,10 @@ typedef struct MonoWebCilHeader { uint32_t pe_cli_header_rva; uint32_t pe_cli_header_size; // 16 bytes + + uint32_t pe_debug_rva; + uint32_t pe_debug_size; + // 24 bytes } MonoWebCilHeader; static gboolean @@ -37,30 +41,67 @@ webcil_image_match (MonoImage *image) return FALSE; } +/* + * Fills the MonoDotNetHeader with data from the given raw_data+offset + * by reading the webcil header. + * most of MonoDotNetHeader is unused and left uninitialized (assumed zero); + */ +static int32_t +do_load_header (const char *raw_data, uint32_t raw_data_len, int32_t offset, MonoDotNetHeader *header) +{ + MonoWebCilHeader wcheader; + if (offset + sizeof (MonoWebCilHeader) > raw_data_len) + return -1; + memcpy (&wcheader, raw_data + offset, sizeof (wcheader)); + + if (!(wcheader.id [0] == 'W' && wcheader.id [1] == 'C' && wcheader.version == MONO_WEBCIL_VERSION)) + return -1; + + memset (header, 0, sizeof(MonoDotNetHeader)); + header->coff.coff_sections = GUINT16_FROM_LE (wcheader.coff_sections); + header->datadir.pe_cli_header.rva = GUINT32_FROM_LE (wcheader.pe_cli_header_rva); + header->datadir.pe_cli_header.size = GUINT32_FROM_LE (wcheader.pe_cli_header_size); + header->datadir.pe_debug.rva = GUINT32_FROM_LE (wcheader.pe_debug_rva); + header->datadir.pe_debug.size = GUINT32_FROM_LE (wcheader.pe_debug_size); + + offset += sizeof (wcheader); + return offset; +} + +int32_t +mono_webcil_load_section_table (const char *raw_data, uint32_t raw_data_len, int32_t offset, MonoSectionTable *t) +{ + /* WebCIL section table entries are a subset of a PE section + * header. Initialize just the parts we have. + */ + uint32_t st [4]; + + if (offset > raw_data_len) + return -1; + memcpy (st, raw_data + offset, sizeof (st)); + t->st_virtual_size = GUINT32_FROM_LE (st [0]); + t->st_virtual_address = GUINT32_FROM_LE (st [1]); + t->st_raw_data_size = GUINT32_FROM_LE (st [2]); + t->st_raw_data_ptr = GUINT32_FROM_LE (st [3]); + offset += sizeof(st); + return offset; +} + + static gboolean webcil_image_load_pe_data (MonoImage *image) { MonoCLIImageInfo *iinfo; MonoDotNetHeader *header; - MonoWebCilHeader wcheader; - gint32 offset = 0; - int i, top; - char d [16 * 8]; - char *p; + int32_t offset = 0; + int top; iinfo = image->image_info; header = &iinfo->cli_header; - if (offset + sizeof (MonoWebCilHeader) > image->raw_data_len) + offset = do_load_header (image->raw_data, image->raw_data_len, offset, header); + if (offset == -1) goto invalid_image; - memcpy (&wcheader, image->raw_data + offset, sizeof (wcheader)); - - if (!(wcheader.id [0] == 'W' && wcheader.id [1] == 'C' && wcheader.version == MONO_WEBCIL_VERSION)) - goto invalid_image; - - header->coff.coff_sections = GUINT16_FROM_LE (wcheader.coff_sections); - header->datadir.pe_cli_header.rva = GUINT32_FROM_LE (wcheader.pe_cli_header_rva); - header->datadir.pe_cli_header.size = GUINT32_FROM_LE (wcheader.pe_cli_header_size); top = iinfo->cli_header.coff.coff_sections; @@ -68,20 +109,11 @@ webcil_image_load_pe_data (MonoImage *image) iinfo->cli_section_tables = g_new0 (MonoSectionTable, top); iinfo->cli_sections = g_new0 (void *, top); - offset += sizeof (wcheader); - g_assert (top < 8); - p = d; - memcpy (d, image->raw_data + offset, top * 16); - for (i = 0; i < top; i++) { + for (int i = 0; i < top; i++) { MonoSectionTable *t = &iinfo->cli_section_tables [i]; - guint32 st [4]; - - memcpy (st, p, sizeof (st)); - t->st_virtual_size = GUINT32_FROM_LE (st [0]); - t->st_virtual_address = GUINT32_FROM_LE (st [1]); - t->st_raw_data_size = GUINT32_FROM_LE (st [2]); - t->st_raw_data_ptr = GUINT32_FROM_LE (st [3]); - p += 16; + offset = mono_webcil_load_section_table (image->raw_data, image->raw_data_len, offset, t); + if (offset == -1) + goto invalid_image; } return TRUE; @@ -125,3 +157,9 @@ mono_webcil_loader_install (void) { mono_install_image_loader (&webcil_loader); } + +int32_t +mono_webcil_load_cli_header (const char *raw_data, uint32_t raw_data_len, int32_t offset, MonoDotNetHeader *header) +{ + return do_load_header (raw_data, raw_data_len, offset, header); +} diff --git a/src/mono/mono/metadata/webcil-loader.h b/src/mono/mono/metadata/webcil-loader.h index 3357d140cb50e..c95c2c5690409 100644 --- a/src/mono/mono/metadata/webcil-loader.h +++ b/src/mono/mono/metadata/webcil-loader.h @@ -8,4 +8,10 @@ void mono_webcil_loader_install (void); +int32_t +mono_webcil_load_cli_header (const char *raw_data, uint32_t raw_data_len, int32_t offset, MonoDotNetHeader *header); + +int32_t +mono_webcil_load_section_table (const char *raw_data, uint32_t raw_data_len, int32_t offset, MonoSectionTable *t); + #endif /*_MONO_METADATA_WEBCIL_LOADER_H*/ diff --git a/src/tasks/WasmAppBuilder/Webcil/WCHeader.cs b/src/tasks/WasmAppBuilder/Webcil/WCHeader.cs index 639d7aab077c9..378efe9918570 100644 --- a/src/tasks/WasmAppBuilder/Webcil/WCHeader.cs +++ b/src/tasks/WasmAppBuilder/Webcil/WCHeader.cs @@ -26,4 +26,7 @@ public unsafe struct WCHeader public uint pe_cli_header_rva; public uint pe_cli_header_size; // 16 bytes + public uint pe_debug_rva; + public uint pe_debug_size; + // 24 bytes } diff --git a/src/tasks/WasmAppBuilder/WebcilWriter.cs b/src/tasks/WasmAppBuilder/WebcilWriter.cs index c25da9917f3e5..ec41ab66969ac 100644 --- a/src/tasks/WasmAppBuilder/WebcilWriter.cs +++ b/src/tasks/WasmAppBuilder/WebcilWriter.cs @@ -76,6 +76,8 @@ public static unsafe void FillHeader(PEReader peReader, out WCHeader header, out header.reserved1 = 0; header.pe_cli_header_rva = (uint)peHeader.CorHeaderTableDirectory.RelativeVirtualAddress; header.pe_cli_header_size = (uint)peHeader.CorHeaderTableDirectory.Size; + header.pe_debug_rva = (uint)peHeader.DebugTableDirectory.RelativeVirtualAddress; + header.pe_debug_size = (uint)peHeader.DebugTableDirectory.Size; // current logical position in the output file FilePosition pos = SizeOfHeader(); @@ -83,7 +85,6 @@ public static unsafe void FillHeader(PEReader peReader, out WCHeader header, out // initially it's after all the section headers FilePosition curSectionPos = pos + sizeof(CoffSectionHeaderBuilder) * coffHeader.NumberOfSections; - // TODO: write the sections, but adjust the raw data ptr to the offset after the WCHeader. ImmutableArray.Builder headerBuilder = ImmutableArray.CreateBuilder(coffHeader.NumberOfSections); foreach (var sectionHeader in sections) { From 7eb32314e1518ddcdf674346399ce9b4623a1846 Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Thu, 22 Dec 2022 13:07:42 -0500 Subject: [PATCH 15/57] fix windows build --- src/mono/mono/metadata/webcil-loader.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/mono/mono/metadata/webcil-loader.c b/src/mono/mono/metadata/webcil-loader.c index 915628a0ac9c2..f325880319cf0 100644 --- a/src/mono/mono/metadata/webcil-loader.c +++ b/src/mono/mono/metadata/webcil-loader.c @@ -76,7 +76,9 @@ mono_webcil_load_section_table (const char *raw_data, uint32_t raw_data_len, int */ uint32_t st [4]; - if (offset > raw_data_len) + if (G_UNLIKELY (offset < 0)) + return offset; + if ((uint32_t)offset > raw_data_len) return -1; memcpy (st, raw_data + offset, sizeof (st)); t->st_virtual_size = GUINT32_FROM_LE (st [0]); From 599500b7c45a4ecb0eff8392e1aad09b67d51181 Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Thu, 22 Dec 2022 15:31:00 -0500 Subject: [PATCH 16/57] Implement a WebcilReader for BorwserDebugProxy like PEReader This needs some improvements: - add support for reading CodeView and EmbeddedPDB data - copy/paste less from the WebcilWriter task - copy/paste less from PEReader (will require moving WebcilReader to SRM) --- .../debugger/BrowserDebugProxy/DebugStore.cs | 124 ++++++++-- .../BrowserDebugProxy/MonoSDBHelper.cs | 2 +- .../Webcil/CoffSectionHeaderBuilder.cs | 27 +++ .../BrowserDebugProxy/Webcil/Constants.cs | 9 + .../BrowserDebugProxy/Webcil/WCHeader.cs | 32 +++ .../BrowserDebugProxy/Webcil/WebcilReader.cs | 225 ++++++++++++++++++ .../Webcil/CoffSectionHeaderBuilder.cs | 4 +- 7 files changed, 401 insertions(+), 22 deletions(-) create mode 100644 src/mono/wasm/debugger/BrowserDebugProxy/Webcil/CoffSectionHeaderBuilder.cs create mode 100644 src/mono/wasm/debugger/BrowserDebugProxy/Webcil/Constants.cs create mode 100644 src/mono/wasm/debugger/BrowserDebugProxy/Webcil/WCHeader.cs create mode 100644 src/mono/wasm/debugger/BrowserDebugProxy/Webcil/WebcilReader.cs diff --git a/src/mono/wasm/debugger/BrowserDebugProxy/DebugStore.cs b/src/mono/wasm/debugger/BrowserDebugProxy/DebugStore.cs index 1e0b19dd435ee..d5909762bec2f 100644 --- a/src/mono/wasm/debugger/BrowserDebugProxy/DebugStore.cs +++ b/src/mono/wasm/debugger/BrowserDebugProxy/DebugStore.cs @@ -854,7 +854,7 @@ internal sealed class AssemblyInfo private readonly List sources = new List(); internal string Url { get; } //The caller must keep the PEReader alive and undisposed throughout the lifetime of the metadata reader - internal PEReader peReader; + private readonly IDisposable peReaderOrWebcilReader; internal MetadataReader asmMetadataReader { get; } internal MetadataReader pdbMetadataReader { get; set; } internal List> enCMetadataReader = new List>(); @@ -867,36 +867,54 @@ internal sealed class AssemblyInfo private readonly Dictionary _documentIdToSourceFileTable = new Dictionary(); - public AssemblyInfo(ILogger logger) + public static AssemblyInfo FromBytes(MonoProxy monoProxy, SessionId sessionId, byte[] assembly, byte[] pdb, ILogger logger, CancellationToken token) { - debugId = -1; - this.id = Interlocked.Increment(ref next_id); - this.logger = logger; + // First try to read it as a PE file, otherwise try it as a WebCIL file + using var asmStream = new MemoryStream(assembly); + try + { + var peReader = new PEReader(asmStream); + if (!peReader.HasMetadata) + throw new BadImageFormatException(); + return FromPEReader(monoProxy, sessionId, peReader, pdb, logger, token); + } + catch (BadImageFormatException) + { + // This is a WebAssembly file + asmStream.Seek(0, SeekOrigin.Begin); + var webcilReader = new Webcil.WebcilReader(asmStream); + return FromWebcilReader(monoProxy, sessionId, webcilReader, pdb, logger, token); + } + } + + public static AssemblyInfo WithoutDebugInfo(ILogger logger) + { + return new AssemblyInfo(logger); } - public unsafe AssemblyInfo(MonoProxy monoProxy, SessionId sessionId, byte[] assembly, byte[] pdb, ILogger logger, CancellationToken token) + private AssemblyInfo(ILogger logger) { debugId = -1; this.id = Interlocked.Increment(ref next_id); this.logger = logger; - using var asmStream = new MemoryStream(assembly); - peReader = new PEReader(asmStream); + } + + private static AssemblyInfo FromPEReader(MonoProxy monoProxy, SessionId sessionId, PEReader peReader, byte[] pdb, ILogger logger, CancellationToken token) + { var entries = peReader.ReadDebugDirectory(); + CodeViewDebugDirectoryData? codeViewData = null; if (entries.Length > 0) { var codeView = entries[0]; if (codeView.Type == DebugDirectoryEntryType.CodeView) { - CodeViewDebugDirectoryData codeViewData = peReader.ReadCodeViewDebugDirectoryData(codeView); - PdbAge = codeViewData.Age; - PdbGuid = codeViewData.Guid; - PdbName = codeViewData.Path; - CodeViewInformationAvailable = true; + codeViewData = peReader.ReadCodeViewDebugDirectoryData(codeView); } } - asmMetadataReader = PEReaderExtensions.GetMetadataReader(peReader); - var asmDef = asmMetadataReader.GetAssemblyDefinition(); - Name = asmDef.GetAssemblyName().Name + ".dll"; + var asmMetadataReader = PEReaderExtensions.GetMetadataReader(peReader); + string name = ReadAssemblyName(asmMetadataReader); + + MetadataReader pdbMetadataReader = null; if (pdb != null) { var pdbStream = new MemoryStream(pdb); @@ -907,7 +925,7 @@ public unsafe AssemblyInfo(MonoProxy monoProxy, SessionId sessionId, byte[] asse } catch (BadImageFormatException) { - monoProxy.SendLog(sessionId, $"Warning: Unable to read debug information of: {Name} (use DebugType=Portable/Embedded)", token); + monoProxy.SendLog(sessionId, $"Warning: Unable to read debug information of: {name} (use DebugType=Portable/Embedded)", token); } } else @@ -918,6 +936,74 @@ public unsafe AssemblyInfo(MonoProxy monoProxy, SessionId sessionId, byte[] asse pdbMetadataReader = peReader.ReadEmbeddedPortablePdbDebugDirectoryData(embeddedPdbEntry).GetMetadataReader(); } } + + var assemblyInfo = new AssemblyInfo(peReader, name, asmMetadataReader, codeViewData, pdbMetadataReader, logger); + return assemblyInfo; + } + + private static AssemblyInfo FromWebcilReader(MonoProxy monoProxy, SessionId sessionId, Webcil.WebcilReader wcReader, byte[] pdb, ILogger logger, CancellationToken token) + { + var entries = wcReader.ReadDebugDirectory(); + CodeViewDebugDirectoryData? codeViewData = null; + if (entries.Length > 0) + { + var codeView = entries[0]; + if (codeView.Type == DebugDirectoryEntryType.CodeView) + { + codeViewData = wcReader.ReadCodeViewDebugDirectoryData(codeView); + } + } + var asmMetadataReader = wcReader.GetMetadataReader(); + string name = ReadAssemblyName(asmMetadataReader); + + MetadataReader pdbMetadataReader = null; + if (pdb != null) + { + var pdbStream = new MemoryStream(pdb); + try + { + // MetadataReaderProvider.FromPortablePdbStream takes ownership of the stream + pdbMetadataReader = MetadataReaderProvider.FromPortablePdbStream(pdbStream).GetMetadataReader(); + } + catch (BadImageFormatException) + { + monoProxy.SendLog(sessionId, $"Warning: Unable to read debug information of: {name} (use DebugType=Portable/Embedded)", token); + } + } + else + { + var embeddedPdbEntry = entries.FirstOrDefault(e => e.Type == DebugDirectoryEntryType.EmbeddedPortablePdb); + if (embeddedPdbEntry.DataSize != 0) + { + pdbMetadataReader = wcReader.ReadEmbeddedPortablePdbDebugDirectoryData(embeddedPdbEntry).GetMetadataReader(); + } + } + + var assemblyInfo = new AssemblyInfo(wcReader, name, asmMetadataReader, codeViewData, pdbMetadataReader, logger); + return assemblyInfo; + } + + private static string ReadAssemblyName(MetadataReader asmMetadataReader) + { + var asmDef = asmMetadataReader.GetAssemblyDefinition(); + return asmDef.GetAssemblyName().Name + ".dll"; + } + + private unsafe AssemblyInfo(IDisposable owningReader, string name, MetadataReader asmMetadataReader, CodeViewDebugDirectoryData? codeViewData, MetadataReader pdbMetadataReader, ILogger logger) + : this(logger) + { + peReaderOrWebcilReader = owningReader; + if (codeViewData != null) + { + PdbAge = codeViewData.Value.Age; + PdbGuid = codeViewData.Value.Guid; + PdbName = codeViewData.Value.Path; + CodeViewInformationAvailable = true; + } + this.asmMetadataReader = asmMetadataReader; + Name = name; + logger.LogDebug($"Info: loading AssemblyInfo with name {Name}"); + this.pdbMetadataReader = pdbMetadataReader; Populate(); } @@ -1410,7 +1496,7 @@ public IEnumerable Add(SessionId id, byte[] assembly_data, byte[] pd AssemblyInfo assembly; try { - assembly = new AssemblyInfo(monoProxy, id, assembly_data, pdb_data, logger, token); + assembly = AssemblyInfo.FromBytes(monoProxy, id, assembly_data, pdb_data, logger, token); } catch (Exception e) { @@ -1504,7 +1590,7 @@ public async IAsyncEnumerable Load(SessionId id, string[] loaded_fil logger.LogDebug($"Bytes from assembly {step.Url} is NULL"); continue; } - assembly = new AssemblyInfo(monoProxy, id, bytes[0], bytes[1], logger, token); + assembly = AssemblyInfo.FromBytes(monoProxy, id, bytes[0], bytes[1], logger, token); } catch (Exception e) { diff --git a/src/mono/wasm/debugger/BrowserDebugProxy/MonoSDBHelper.cs b/src/mono/wasm/debugger/BrowserDebugProxy/MonoSDBHelper.cs index 91e02eb15fdaa..29d6356acb09e 100644 --- a/src/mono/wasm/debugger/BrowserDebugProxy/MonoSDBHelper.cs +++ b/src/mono/wasm/debugger/BrowserDebugProxy/MonoSDBHelper.cs @@ -855,7 +855,7 @@ public async Task GetAssemblyInfo(int assemblyId, CancellationToke asm = store.GetAssemblyByName(assemblyName); if (asm == null) { - asm = new AssemblyInfo(logger); + asm = AssemblyInfo.WithoutDebugInfo(logger); logger.LogDebug($"Created assembly without debug information: {assemblyName}"); } } diff --git a/src/mono/wasm/debugger/BrowserDebugProxy/Webcil/CoffSectionHeaderBuilder.cs b/src/mono/wasm/debugger/BrowserDebugProxy/Webcil/CoffSectionHeaderBuilder.cs new file mode 100644 index 0000000000000..d8e428bea6b6d --- /dev/null +++ b/src/mono/wasm/debugger/BrowserDebugProxy/Webcil/CoffSectionHeaderBuilder.cs @@ -0,0 +1,27 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Runtime.InteropServices; + +namespace Microsoft.WebAssembly.Diagnostics.Webcil; + +/// +/// This is System.Reflection.PortableExecutable.CoffHeader, but with a public constructor so that +/// we can make our own copies of it, and with fewer fields +/// +[StructLayout(LayoutKind.Sequential, Pack = 1)] +public struct CoffSectionHeaderBuilder +{ + public readonly int VirtualSize; + public readonly int VirtualAddress; + public readonly int SizeOfRawData; + public readonly int PointerToRawData; + + public CoffSectionHeaderBuilder(int virtualSize, int virtualAddress, int sizeOfRawData, int pointerToRawData) + { + VirtualSize = virtualSize; + VirtualAddress = virtualAddress; + SizeOfRawData = sizeOfRawData; + PointerToRawData = pointerToRawData; + } +} diff --git a/src/mono/wasm/debugger/BrowserDebugProxy/Webcil/Constants.cs b/src/mono/wasm/debugger/BrowserDebugProxy/Webcil/Constants.cs new file mode 100644 index 0000000000000..b7634199538f7 --- /dev/null +++ b/src/mono/wasm/debugger/BrowserDebugProxy/Webcil/Constants.cs @@ -0,0 +1,9 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.WebAssembly.Diagnostics.Webcil; + +public static unsafe class Constants +{ + public const int WC_VERSION = 0; +} diff --git a/src/mono/wasm/debugger/BrowserDebugProxy/Webcil/WCHeader.cs b/src/mono/wasm/debugger/BrowserDebugProxy/Webcil/WCHeader.cs new file mode 100644 index 0000000000000..905407f7b0165 --- /dev/null +++ b/src/mono/wasm/debugger/BrowserDebugProxy/Webcil/WCHeader.cs @@ -0,0 +1,32 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Runtime.InteropServices; + +namespace Microsoft.WebAssembly.Diagnostics.Webcil; + +/// +/// The header of a WebCIL file. +/// +/// +/// +/// The header is a subset of the PE, COFF and CLI headers that are needed by the mono runtime to load managed assemblies. +/// +[StructLayout(LayoutKind.Sequential, Pack = 1)] +public unsafe struct WCHeader +{ + public fixed byte id[2]; // 'W' 'C' + public byte version; + public byte reserved0; // 0 + // 4 bytes + + public ushort coff_sections; + public ushort reserved1; // 0 + // 8 bytes + public uint pe_cli_header_rva; + public uint pe_cli_header_size; + // 16 bytes + public uint pe_debug_rva; + public uint pe_debug_size; + // 24 bytes +} diff --git a/src/mono/wasm/debugger/BrowserDebugProxy/Webcil/WebcilReader.cs b/src/mono/wasm/debugger/BrowserDebugProxy/Webcil/WebcilReader.cs new file mode 100644 index 0000000000000..764feb3b90866 --- /dev/null +++ b/src/mono/wasm/debugger/BrowserDebugProxy/Webcil/WebcilReader.cs @@ -0,0 +1,225 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Immutable; +using System.IO; +using System.Runtime.InteropServices; + +using System.Reflection.Metadata; +using System.Reflection.PortableExecutable; + +namespace Microsoft.WebAssembly.Diagnostics.Webcil; + + +public sealed class WebcilReader : IDisposable +{ + // TODO: WISH: + // This should be implemented in terms of System.Reflection.Internal.MemoryBlockProvider like the PEReader, + // but the memory block classes are internal to S.R.M. + + private readonly Stream _stream; + private WCHeader? _header; + private DirectoryEntry? _corHeaderMetadataDirectory; + private MetadataReaderProvider _metadataReaderProvider; + private ImmutableArray? _sections; + + public WebcilReader(Stream stream) + { + this._stream = stream; + if (!stream.CanRead || !stream.CanSeek) + { + throw new ArgumentException("Stream must be readable and seekable", nameof(stream)); + } + if (!ReadHeader()) + { + throw new BadImageFormatException("Stream does not contain a valid Webcil file", nameof(stream)); + } + if (!ReadCorHeader()) + { + throw new BadImageFormatException("Stream does not contain a valid COR header in the Webcil file", nameof(stream)); + } + } + + private unsafe bool ReadHeader() + { + WCHeader header; + var buffer = new byte[Marshal.SizeOf()]; + if (_stream.Read(buffer, 0, buffer.Length) != buffer.Length) + { + return false; + } + fixed (byte* p = buffer) + { + header = *(WCHeader*)p; + } + if (header.id[0] != 'W' || header.id[1] != 'C' || header.version != Constants.WC_VERSION) + { + return false; + } + _header = header; + if (!BitConverter.IsLittleEndian) + { + throw new NotImplementedException("TODO: implement big endian support"); + } + return true; + } + + private unsafe bool ReadCorHeader() + { + // we can't construct CorHeader because it's constructor is internal + // but we don't care, really, we only want the metadata directory entry + var pos = TranslateRVA(_header.Value.pe_cli_header_rva); + if (_stream.Seek(pos, SeekOrigin.Begin) != pos) + { + return false; + } + using var reader = new BinaryReader(_stream, System.Text.Encoding.UTF8, leaveOpen: true); + reader.ReadInt32(); // byte count + reader.ReadUInt16(); // major version + reader.ReadUInt16(); // minor version + _corHeaderMetadataDirectory = new DirectoryEntry(reader.ReadInt32(), reader.ReadInt32()); + return true; + } + + public MetadataReaderProvider GetMetadataReaderProvider() + { + // FIXME threading + if (_metadataReaderProvider == null) + { + long pos = TranslateRVA((uint)_corHeaderMetadataDirectory.Value.RelativeVirtualAddress); + if (_stream.Seek(pos, SeekOrigin.Begin) != pos) + { + throw new BadImageFormatException("Could not seek to metadata", nameof(_stream)); + } + _metadataReaderProvider = MetadataReaderProvider.FromMetadataStream(_stream, MetadataStreamOptions.LeaveOpen); + } + return _metadataReaderProvider; + } + + public MetadataReader GetMetadataReader() => GetMetadataReaderProvider().GetMetadataReader(); + + public ImmutableArray ReadDebugDirectory() + { + var debugRVA = _header.Value.pe_debug_rva; + if (debugRVA == 0) + { + return ImmutableArray.Empty; + } + var debugSize = _header.Value.pe_debug_size; + if (debugSize == 0) + { + return ImmutableArray.Empty; + } + var debugOffset = TranslateRVA(debugRVA); + _stream.Seek(debugOffset, SeekOrigin.Begin); + var buffer = new byte[debugSize]; + if (_stream.Read(buffer, 0, buffer.Length) != buffer.Length) + { + throw new BadImageFormatException("Could not read debug directory", nameof(_stream)); + } + unsafe + { + fixed (byte* p = buffer) + { + return ReadDebugDirectoryEntries(new BlobReader(p, buffer.Length)); + } + } + } + + // FIXME: copied from DebugDirectoryEntry.Size + internal const int DebugDirectoryEntrySize = + sizeof(uint) + // Characteristics + sizeof(uint) + // TimeDataStamp + sizeof(uint) + // Version + sizeof(uint) + // Type + sizeof(uint) + // SizeOfData + sizeof(uint) + // AddressOfRawData + sizeof(uint); // PointerToRawData + + + // FIXME: copy-pasted from PEReader + private static ImmutableArray ReadDebugDirectoryEntries(BlobReader reader) + { + int entryCount = reader.Length / DebugDirectoryEntrySize; + var builder = ImmutableArray.CreateBuilder(entryCount); + for (int i = 0; i < entryCount; i++) + { + // Reserved, must be zero. + int characteristics = reader.ReadInt32(); + if (characteristics != 0) + { + throw new BadImageFormatException(); + } + + uint stamp = reader.ReadUInt32(); + ushort majorVersion = reader.ReadUInt16(); + ushort minorVersion = reader.ReadUInt16(); + + var type = (DebugDirectoryEntryType)reader.ReadInt32(); + + int dataSize = reader.ReadInt32(); + int dataRva = reader.ReadInt32(); + int dataPointer = reader.ReadInt32(); + + builder.Add(new DebugDirectoryEntry(stamp, majorVersion, minorVersion, type, dataSize, dataRva, dataPointer)); + } + + return builder.MoveToImmutable(); + } + + public CodeViewDebugDirectoryData ReadCodeViewDebugDirectoryData(DebugDirectoryEntry entry) + { + // TODO: implement by copying PEReader.DecodeCodeViewDebugDirectoryData + throw new NotImplementedException("FIXME: implement ReadCodeViewDebugDirectoryData"); + } + + public MetadataReaderProvider ReadEmbeddedPortablePdbDebugDirectoryData(DebugDirectoryEntry entry) + { + // TODO: implement by copying PEReader.DecodeEmbeddedPortablePdbDebugDirectoryData + throw new NotImplementedException("FIXME: Implement ReadEmbeddedPortablePdbDebugDirectoryData"); + } + + private long TranslateRVA(uint rva) + { + if (_sections == null) + { + _sections = ReadSections(); + } + foreach (var section in _sections.Value) + { + if (rva >= section.VirtualAddress && rva < section.VirtualAddress + section.VirtualSize) + { + return section.PointerToRawData + (rva - section.VirtualAddress); + } + } + throw new BadImageFormatException("RVA not found in any section", nameof(_stream)); + } + + private static long SectionDirectoryOffset => Marshal.SizeOf(); + + private unsafe ImmutableArray ReadSections() + { + var sections = ImmutableArray.CreateBuilder(_header.Value.coff_sections); + var buffer = new byte[Marshal.SizeOf()]; + _stream.Seek(SectionDirectoryOffset, SeekOrigin.Begin); + for (int i = 0; i < _header.Value.coff_sections; i++) + { + if (_stream.Read(buffer, 0, buffer.Length) != buffer.Length) + { + throw new BadImageFormatException("Stream does not contain a valid Webcil file", nameof(_stream)); + } + fixed (byte* p = buffer) + { + // FIXME endianness + sections.Add(*(CoffSectionHeaderBuilder*)p); + } + } + return sections.MoveToImmutable(); + } + + public void Dispose() + { + _stream.Dispose(); + } +} diff --git a/src/tasks/WasmAppBuilder/Webcil/CoffSectionHeaderBuilder.cs b/src/tasks/WasmAppBuilder/Webcil/CoffSectionHeaderBuilder.cs index bc79259f8db98..47be4733f36a6 100644 --- a/src/tasks/WasmAppBuilder/Webcil/CoffSectionHeaderBuilder.cs +++ b/src/tasks/WasmAppBuilder/Webcil/CoffSectionHeaderBuilder.cs @@ -6,8 +6,8 @@ namespace Microsoft.WebAssembly.Build.Tasks.WebCil; /// -/// This is System.Reflection.PortableExecutable.CoffSectionHeaderBuilder, but with a public constructor so that -/// we can make our own copies of it. +/// This is System.Reflection.PortableExecutable.CoffHeader, but with a public constructor so that +/// we can make our own copies of it, and with fewer fields /// [StructLayout(LayoutKind.Sequential, Pack = 1)] public struct CoffSectionHeaderBuilder From 7d5aa7748a187d95bb8d55c651d8fcea0b839993 Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Thu, 22 Dec 2022 16:16:22 -0500 Subject: [PATCH 17/57] REVERT ME: temporarily don't try to read codeview data for webcil --- src/mono/wasm/debugger/BrowserDebugProxy/DebugStore.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mono/wasm/debugger/BrowserDebugProxy/DebugStore.cs b/src/mono/wasm/debugger/BrowserDebugProxy/DebugStore.cs index d5909762bec2f..60812d7d5a867 100644 --- a/src/mono/wasm/debugger/BrowserDebugProxy/DebugStore.cs +++ b/src/mono/wasm/debugger/BrowserDebugProxy/DebugStore.cs @@ -948,7 +948,7 @@ private static AssemblyInfo FromWebcilReader(MonoProxy monoProxy, SessionId sess if (entries.Length > 0) { var codeView = entries[0]; - if (codeView.Type == DebugDirectoryEntryType.CodeView) + if (codeView.Type == DebugDirectoryEntryType.CodeView && false) // FIXME: implement this { codeViewData = wcReader.ReadCodeViewDebugDirectoryData(codeView); } From 68905aab3de0cf12365deb0cb38817c0a1fe047d Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Thu, 22 Dec 2022 22:23:08 -0500 Subject: [PATCH 18/57] [debug] Match bundled pdbs if we're looking up .webcil files The pdbs are registered by wasm with a notional .dll filename. if the debugger does a lookup using a .webcil name instead, allow the match --- src/mono/mono/metadata/mono-debug.c | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/src/mono/mono/metadata/mono-debug.c b/src/mono/mono/metadata/mono-debug.c index 5c81475ccb244..95e486ac93260 100644 --- a/src/mono/mono/metadata/mono-debug.c +++ b/src/mono/mono/metadata/mono-debug.c @@ -1097,13 +1097,31 @@ mono_register_symfile_for_assembly (const char *assembly_name, const mono_byte * bundled_symfiles = bsymfile; } +static gboolean +bsymfile_match (BundledSymfile *bsymfile, const char *assembly_name) +{ + if (!strcmp (bsymfile->aname, assembly_name)) + return TRUE; +#ifndef DISABLE_WEBCIL + const char *p = strstr (assembly_name, ".webcil"); + /* if assembly_name ends with .webcil, check if aname matches, with a .dll extension instead */ + if (p && *(p + 7) == 0) { + size_t n = p - assembly_name; + if (!strncmp (bsymfile->aname, assembly_name, n) + && !strcmp (bsymfile->aname + n, ".dll")) + return TRUE; + } +#endif + return FALSE; +} + static MonoDebugHandle * open_symfile_from_bundle (MonoImage *image) { BundledSymfile *bsymfile; for (bsymfile = bundled_symfiles; bsymfile; bsymfile = bsymfile->next) { - if (strcmp (bsymfile->aname, image->module_name)) + if (!bsymfile_match (bsymfile, image->module_name)) continue; return mono_debug_open_image (image, bsymfile->raw_contents, bsymfile->size); @@ -1117,7 +1135,7 @@ mono_get_symfile_bytes_from_bundle (const char *assembly_name, int *size) { BundledSymfile *bsymfile; for (bsymfile = bundled_symfiles; bsymfile; bsymfile = bsymfile->next) { - if (strcmp (bsymfile->aname, assembly_name)) + if (!bsymfile_match (bsymfile, assembly_name)) continue; *size = bsymfile->size; return bsymfile->raw_contents; From 68da344ec74c8ae721669b479878e90f649f75e9 Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Thu, 22 Dec 2022 22:31:07 -0500 Subject: [PATCH 19/57] be less verbose --- src/mono/wasm/debugger/BrowserDebugProxy/DebugStore.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mono/wasm/debugger/BrowserDebugProxy/DebugStore.cs b/src/mono/wasm/debugger/BrowserDebugProxy/DebugStore.cs index 60812d7d5a867..ffde2f6c53384 100644 --- a/src/mono/wasm/debugger/BrowserDebugProxy/DebugStore.cs +++ b/src/mono/wasm/debugger/BrowserDebugProxy/DebugStore.cs @@ -1002,7 +1002,7 @@ private unsafe AssemblyInfo(IDisposable owningReader, string name, MetadataReade } this.asmMetadataReader = asmMetadataReader; Name = name; - logger.LogDebug($"Info: loading AssemblyInfo with name {Name}"); + logger.LogTrace($"Info: loading AssemblyInfo with name {Name}"); this.pdbMetadataReader = pdbMetadataReader; Populate(); } From 1db11d1fd3f4d9da3fe4f4ef63c2d32057e801f0 Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Wed, 28 Dec 2022 16:04:58 -0500 Subject: [PATCH 20/57] Adjust debug directory entries when writing webcil files the PE COFF debug directory entries contain a 'pointer' field which is an offset from the start of the file. When writing the webcil file, the header is typically smaller than a PE file, so the offsets are wrong. Adjust the offsets by the size of the file. We assume (and assert) the debug directory entries actually point at some PE COFF sections in the PE file (as opposed to somewhere past the end of the known PE data). When writing, we initially just copy all the sections directly, then seek to where the debug directory entries are, and overwrite them with updated entries that have the correct 'pointer' --- ...eaderBuilder.cs => WebcilSectionHeader.cs} | 7 +- src/tasks/WasmAppBuilder/WebcilWriter.cs | 178 ++++++++++++++++-- 2 files changed, 163 insertions(+), 22 deletions(-) rename src/tasks/WasmAppBuilder/Webcil/{CoffSectionHeaderBuilder.cs => WebcilSectionHeader.cs} (67%) diff --git a/src/tasks/WasmAppBuilder/Webcil/CoffSectionHeaderBuilder.cs b/src/tasks/WasmAppBuilder/Webcil/WebcilSectionHeader.cs similarity index 67% rename from src/tasks/WasmAppBuilder/Webcil/CoffSectionHeaderBuilder.cs rename to src/tasks/WasmAppBuilder/Webcil/WebcilSectionHeader.cs index 47be4733f36a6..ea8e3a1b30063 100644 --- a/src/tasks/WasmAppBuilder/Webcil/CoffSectionHeaderBuilder.cs +++ b/src/tasks/WasmAppBuilder/Webcil/WebcilSectionHeader.cs @@ -6,18 +6,17 @@ namespace Microsoft.WebAssembly.Build.Tasks.WebCil; /// -/// This is System.Reflection.PortableExecutable.CoffHeader, but with a public constructor so that -/// we can make our own copies of it, and with fewer fields +/// This is the Webcil analog of System.Reflection.PortableExecutable.SectionHeader, but with fewer fields /// [StructLayout(LayoutKind.Sequential, Pack = 1)] -public struct CoffSectionHeaderBuilder +public readonly struct WebcilSectionHeader { public readonly int VirtualSize; public readonly int VirtualAddress; public readonly int SizeOfRawData; public readonly int PointerToRawData; - public CoffSectionHeaderBuilder(int virtualSize, int virtualAddress, int sizeOfRawData, int pointerToRawData) + public WebcilSectionHeader(int virtualSize, int virtualAddress, int sizeOfRawData, int pointerToRawData) { VirtualSize = virtualSize; VirtualAddress = virtualAddress; diff --git a/src/tasks/WasmAppBuilder/WebcilWriter.cs b/src/tasks/WasmAppBuilder/WebcilWriter.cs index ec41ab66969ac..f3372dea5c4bf 100644 --- a/src/tasks/WasmAppBuilder/WebcilWriter.cs +++ b/src/tasks/WasmAppBuilder/WebcilWriter.cs @@ -18,6 +18,29 @@ namespace Microsoft.WebAssembly.Build.Tasks; /// public class WebcilWriter { + + // Interesting stuff we've learned about the input PE file + public record PEFileInfo( + // The sections in the PE file + ImmutableArray SectionHeaders, + // The location of the debug directory entries + DirectoryEntry DebugTableDirectory, + // The file offset of the sections, following the section directory + FilePosition SectionStart, + // The debug directory entries + ImmutableArray DebugDirectoryEntries + ); + + // Intersting stuff we know about the webcil file we're writing + public record WCFileInfo( + // The header of the webcil file + WCHeader Header, + // The section directory of the webcil file + ImmutableArray SectionHeaders, + // The file offset of the sections, following the section directory + FilePosition SectionStart + ); + private readonly string _inputPath; private readonly string _outputPath; @@ -34,23 +57,24 @@ public void Write() Log.LogMessage($"Writing Webcil (input {_inputPath}) output to {_outputPath}"); using var inputStream = File.Open(_inputPath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); - ImmutableArray sectionsHeaders; - ImmutableArray peSections; - WCHeader header; + PEFileInfo peInfo; + WCFileInfo wcInfo; using (var peReader = new PEReader(inputStream, PEStreamOptions.LeaveOpen)) { // DumpPE(peReader); - FillHeader(peReader, out header, out peSections, out sectionsHeaders); + GatherInfo(peReader, out wcInfo, out peInfo); } using var outputStream = File.Open(_outputPath, FileMode.Create, FileAccess.Write); - WriteHeader(outputStream, header); - WriteSectionHeaders(outputStream, sectionsHeaders); - CopySections(outputStream, inputStream, peSections); + WriteHeader(outputStream, wcInfo.Header); + WriteSectionHeaders(outputStream, wcInfo.SectionHeaders); + CopySections(outputStream, inputStream, peInfo.SectionHeaders); + var wcDebugDirectoryEntries = FixupDebugDirectoryEntries(peInfo, wcInfo); + OverwriteDebugDirectoryEntries(outputStream, wcInfo, wcDebugDirectoryEntries); + outputStream.Flush(); } - - private record struct FilePosition(int Position) + public record struct FilePosition(int Position) { public static implicit operator FilePosition(int position) => new(position); @@ -62,12 +86,13 @@ private static unsafe int SizeOfHeader() return sizeof(WCHeader); } - public static unsafe void FillHeader(PEReader peReader, out WCHeader header, out ImmutableArray peSections, out ImmutableArray sectionsHeaders) + public static unsafe void GatherInfo(PEReader peReader, out WCFileInfo wcInfo, out PEFileInfo peInfo) { var headers = peReader.PEHeaders; var peHeader = headers.PEHeader!; var coffHeader = headers.CoffHeader!; var sections = headers.SectionHeaders; + WCHeader header; header.id[0] = (byte)'W'; header.id[1] = (byte)'C'; header.version = Constants.WC_VERSION; @@ -83,12 +108,25 @@ public static unsafe void FillHeader(PEReader peReader, out WCHeader header, out FilePosition pos = SizeOfHeader(); // position of the current section in the output file // initially it's after all the section headers - FilePosition curSectionPos = pos + sizeof(CoffSectionHeaderBuilder) * coffHeader.NumberOfSections; + FilePosition curSectionPos = pos + sizeof(WebcilSectionHeader) * coffHeader.NumberOfSections; - ImmutableArray.Builder headerBuilder = ImmutableArray.CreateBuilder(coffHeader.NumberOfSections); + FilePosition firstWCSection = curSectionPos; + FilePosition firstPESection = 0; + + ImmutableArray.Builder headerBuilder = ImmutableArray.CreateBuilder(coffHeader.NumberOfSections); foreach (var sectionHeader in sections) { - var newHeader = new CoffSectionHeaderBuilder + // The first section is the one with the lowest file offset + if (firstPESection.Position == 0) + { + firstPESection = sectionHeader.PointerToRawData; + } + else + { + firstPESection = Math.Min(firstPESection.Position, sectionHeader.PointerToRawData); + } + + var newHeader = new WebcilSectionHeader ( virtualSize: sectionHeader.VirtualSize, virtualAddress: sectionHeader.VirtualAddress, @@ -96,13 +134,19 @@ public static unsafe void FillHeader(PEReader peReader, out WCHeader header, out pointerToRawData: curSectionPos.Position ); - pos += sizeof(CoffSectionHeaderBuilder); + pos += sizeof(WebcilSectionHeader); curSectionPos += sectionHeader.SizeOfRawData; headerBuilder.Add(newHeader); } - peSections = sections; - sectionsHeaders = headerBuilder.ToImmutable(); + peInfo = new PEFileInfo(SectionHeaders: sections, + DebugTableDirectory: peHeader.DebugTableDirectory, + SectionStart: firstPESection, + DebugDirectoryEntries: peReader.ReadDebugDirectory()); + + wcInfo = new WCFileInfo(Header: header, + SectionHeaders: headerBuilder.MoveToImmutable(), + SectionStart: firstWCSection); } private static void WriteHeader(Stream s, WCHeader header) @@ -110,7 +154,7 @@ private static void WriteHeader(Stream s, WCHeader header) WriteStructure(s, header); } - private static void WriteSectionHeaders(Stream s, ImmutableArray sectionsHeaders) + private static void WriteSectionHeaders(Stream s, ImmutableArray sectionsHeaders) { // FIXME: fixup endianness if (!BitConverter.IsLittleEndian) @@ -121,7 +165,7 @@ private static void WriteSectionHeaders(Stream s, ImmutableArray wcSections, uint relativeVirtualAddress) + { + foreach (var section in wcSections) + { + if (relativeVirtualAddress >= section.VirtualAddress && relativeVirtualAddress < section.VirtualAddress + section.VirtualSize) + { + return section.PointerToRawData + (int)(relativeVirtualAddress - section.VirtualAddress); + } + } + + throw new InvalidOperationException("relative virtual address not in any section"); + } + + // Given a physical file offset, return the section and the offset within the section. + private static (WebcilSectionHeader section, int offset) GetSectionFromFileOffset(ImmutableArray peSections, FilePosition fileOffset) + { + foreach (var section in peSections) + { + if (fileOffset.Position >= section.PointerToRawData && fileOffset.Position < section.PointerToRawData + section.SizeOfRawData) + { + return (section, fileOffset.Position - section.PointerToRawData); + } + } + + throw new InvalidOperationException("file offset not in any section (Webcil)"); + } + + private static void GetSectionFromFileOffset(ImmutableArray sections, FilePosition fileOffset) + { + foreach (var section in sections) + { + if (fileOffset.Position >= section.PointerToRawData && fileOffset.Position < section.PointerToRawData + section.SizeOfRawData) + { + return; + } + } + + throw new InvalidOperationException($"file offset {fileOffset.Position} not in any section (PE)"); + } + + // Make a new set of debug directory entries that + // have their data pointers adjusted to be relative to the start of the webcil file. + // This is necessary because the debug directory entires in the PE file are relative to the start of the PE file, + // and a PE header is bigger than a webcil header. + private static ImmutableArray FixupDebugDirectoryEntries(PEFileInfo peInfo, WCFileInfo wcInfo) + { + int dataPointerAdjustment = peInfo.SectionStart.Position - wcInfo.SectionStart.Position; + ImmutableArray entries = peInfo.DebugDirectoryEntries; + ImmutableArray.Builder newEntries = ImmutableArray.CreateBuilder(entries.Length); + foreach (var entry in entries) + { + DebugDirectoryEntry newEntry; + if (entry.Type == DebugDirectoryEntryType.Reproducible) + { + // this entry doesn't have an associated data pointer, so just copy it + newEntry = entry; + } + else + { + // the "pointer" field is a file offset, find the corresponding entry in the Webcil file and overwrite with the correct file position + var newDataPointer = entry.DataPointer - dataPointerAdjustment; + newEntry = new DebugDirectoryEntry(entry.Stamp, entry.MajorVersion, entry.MinorVersion, entry.Type, entry.DataSize, entry.DataRelativeVirtualAddress, newDataPointer); + GetSectionFromFileOffset(peInfo.SectionHeaders, entry.DataPointer); + // validate that the new entry is in some section + GetSectionFromFileOffset(wcInfo.SectionHeaders, newDataPointer); + } + newEntries.Add(newEntry); + } + return newEntries.MoveToImmutable(); + } + + private static void OverwriteDebugDirectoryEntries(Stream s, WCFileInfo wcInfo, ImmutableArray entries) + { + s.Seek(GetPositionOfRelativeVirtualAddress(wcInfo.SectionHeaders, wcInfo.Header.pe_cli_header_rva).Position, SeekOrigin.Begin); + // endianness: ok, we're just copying from one stream to another + foreach (var entry in entries) + { + WriteDebugDirectoryEntry(s, entry); + } + // TODO check that we overwrite with the same size as the original + + // restore the stream position + s.Seek(0, SeekOrigin.End); + } + + private static void WriteDebugDirectoryEntry(Stream s, DebugDirectoryEntry entry) + { + using var writer = new BinaryWriter(s, System.Text.Encoding.UTF8, leaveOpen: true); + writer.Write((uint)0); // Characteristics + writer.Write(entry.Stamp); + writer.Write(entry.MajorVersion); + writer.Write(entry.MinorVersion); + writer.Write((uint)entry.Type); + writer.Write(entry.DataSize); + writer.Write(entry.DataRelativeVirtualAddress); + writer.Write(entry.DataPointer); + writer.Flush(); + } } From 784f809bff92f7e52de47678d478f13f73cd8089 Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Thu, 29 Dec 2022 12:04:27 -0500 Subject: [PATCH 21/57] Fix bug in WebcilWriter Stream.CopyTo takes a buffer size, not the number of bytes to copy. --- src/tasks/WasmAppBuilder/WebcilWriter.cs | 31 ++++++++++++++++++++---- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/src/tasks/WasmAppBuilder/WebcilWriter.cs b/src/tasks/WasmAppBuilder/WebcilWriter.cs index f3372dea5c4bf..f3c70698094a2 100644 --- a/src/tasks/WasmAppBuilder/WebcilWriter.cs +++ b/src/tasks/WasmAppBuilder/WebcilWriter.cs @@ -212,11 +212,32 @@ private static void CopySections(Stream outStream, Stream inputStream, Immutable // endianness: ok, we're just copying from one stream to another foreach (var peHeader in peSections) { + var buffer = new byte[peHeader.SizeOfRawData]; inputStream.Seek(peHeader.PointerToRawData, SeekOrigin.Begin); - inputStream.CopyTo(outStream, peHeader.SizeOfRawData); + ReadExactly(inputStream, buffer); + outStream.Write(buffer, 0, buffer.Length); } } +#if NETCOREAPP2_1_OR_GREATER + private static void ReadExactly(Stream s, Span buffer) + { + s.ReadExactly(buffer); + } +#else + private static void ReadExactly(Stream s, byte[] buffer) + { + int offset = 0; + while (offset < buffer.Length) + { + int read = s.Read(buffer, offset, buffer.Length - offset); + if (read == 0) + throw new EndOfStreamException(); + offset += read; + } + } +#endif + private static FilePosition GetPositionOfRelativeVirtualAddress(ImmutableArray wcSections, uint relativeVirtualAddress) { foreach (var section in wcSections) @@ -291,20 +312,21 @@ private static ImmutableArray FixupDebugDirectoryEntries(PE private static void OverwriteDebugDirectoryEntries(Stream s, WCFileInfo wcInfo, ImmutableArray entries) { s.Seek(GetPositionOfRelativeVirtualAddress(wcInfo.SectionHeaders, wcInfo.Header.pe_cli_header_rva).Position, SeekOrigin.Begin); + using var writer = new BinaryWriter(s, System.Text.Encoding.UTF8, leaveOpen: true); // endianness: ok, we're just copying from one stream to another foreach (var entry in entries) { - WriteDebugDirectoryEntry(s, entry); + WriteDebugDirectoryEntry(writer, entry); } + writer.Flush(); // TODO check that we overwrite with the same size as the original // restore the stream position s.Seek(0, SeekOrigin.End); } - private static void WriteDebugDirectoryEntry(Stream s, DebugDirectoryEntry entry) + private static void WriteDebugDirectoryEntry(BinaryWriter writer, DebugDirectoryEntry entry) { - using var writer = new BinaryWriter(s, System.Text.Encoding.UTF8, leaveOpen: true); writer.Write((uint)0); // Characteristics writer.Write(entry.Stamp); writer.Write(entry.MajorVersion); @@ -313,6 +335,5 @@ private static void WriteDebugDirectoryEntry(Stream s, DebugDirectoryEntry entry writer.Write(entry.DataSize); writer.Write(entry.DataRelativeVirtualAddress); writer.Write(entry.DataPointer); - writer.Flush(); } } From 111d398ffbad4ae50b0a733c5509423ed916719d Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Thu, 29 Dec 2022 14:04:15 -0500 Subject: [PATCH 22/57] bugfix: the debug directory is at pe_debug_rva not at the CLI header --- src/tasks/WasmAppBuilder/WebcilWriter.cs | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/tasks/WasmAppBuilder/WebcilWriter.cs b/src/tasks/WasmAppBuilder/WebcilWriter.cs index f3c70698094a2..c0cea12a64d99 100644 --- a/src/tasks/WasmAppBuilder/WebcilWriter.cs +++ b/src/tasks/WasmAppBuilder/WebcilWriter.cs @@ -86,7 +86,7 @@ private static unsafe int SizeOfHeader() return sizeof(WCHeader); } - public static unsafe void GatherInfo(PEReader peReader, out WCFileInfo wcInfo, out PEFileInfo peInfo) + public unsafe void GatherInfo(PEReader peReader, out WCFileInfo wcInfo, out PEFileInfo peInfo) { var headers = peReader.PEHeaders; var peHeader = headers.PEHeader!; @@ -109,8 +109,9 @@ public static unsafe void GatherInfo(PEReader peReader, out WCFileInfo wcInfo, o // position of the current section in the output file // initially it's after all the section headers FilePosition curSectionPos = pos + sizeof(WebcilSectionHeader) * coffHeader.NumberOfSections; - + // The first WC section is immediately after the section directory FilePosition firstWCSection = curSectionPos; + FilePosition firstPESection = 0; ImmutableArray.Builder headerBuilder = ImmutableArray.CreateBuilder(coffHeader.NumberOfSections); @@ -139,10 +140,13 @@ public static unsafe void GatherInfo(PEReader peReader, out WCFileInfo wcInfo, o headerBuilder.Add(newHeader); } + + ImmutableArray debugDirectoryEntries = peReader.ReadDebugDirectory(); + peInfo = new PEFileInfo(SectionHeaders: sections, DebugTableDirectory: peHeader.DebugTableDirectory, SectionStart: firstPESection, - DebugDirectoryEntries: peReader.ReadDebugDirectory()); + DebugDirectoryEntries: debugDirectoryEntries); wcInfo = new WCFileInfo(Header: header, SectionHeaders: headerBuilder.MoveToImmutable(), @@ -244,7 +248,8 @@ private static FilePosition GetPositionOfRelativeVirtualAddress(ImmutableArray= section.VirtualAddress && relativeVirtualAddress < section.VirtualAddress + section.VirtualSize) { - return section.PointerToRawData + (int)(relativeVirtualAddress - section.VirtualAddress); + FilePosition pos = section.PointerToRawData + ((int)relativeVirtualAddress - section.VirtualAddress); + return pos; } } @@ -290,7 +295,7 @@ private static ImmutableArray FixupDebugDirectoryEntries(PE foreach (var entry in entries) { DebugDirectoryEntry newEntry; - if (entry.Type == DebugDirectoryEntryType.Reproducible) + if (entry.Type == DebugDirectoryEntryType.Reproducible || entry.DataPointer == 0 || entry.DataSize == 0) { // this entry doesn't have an associated data pointer, so just copy it newEntry = entry; @@ -311,18 +316,17 @@ private static ImmutableArray FixupDebugDirectoryEntries(PE private static void OverwriteDebugDirectoryEntries(Stream s, WCFileInfo wcInfo, ImmutableArray entries) { - s.Seek(GetPositionOfRelativeVirtualAddress(wcInfo.SectionHeaders, wcInfo.Header.pe_cli_header_rva).Position, SeekOrigin.Begin); + FilePosition debugDirectoryPos = GetPositionOfRelativeVirtualAddress(wcInfo.SectionHeaders, wcInfo.Header.pe_debug_rva); using var writer = new BinaryWriter(s, System.Text.Encoding.UTF8, leaveOpen: true); - // endianness: ok, we're just copying from one stream to another + writer.Seek(debugDirectoryPos.Position, SeekOrigin.Begin); foreach (var entry in entries) { WriteDebugDirectoryEntry(writer, entry); } - writer.Flush(); // TODO check that we overwrite with the same size as the original // restore the stream position - s.Seek(0, SeekOrigin.End); + writer.Seek(0, SeekOrigin.End); } private static void WriteDebugDirectoryEntry(BinaryWriter writer, DebugDirectoryEntry entry) From 5541a3b6ac793a4dd1c15e8a04b0755bca3144a1 Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Thu, 29 Dec 2022 14:47:30 -0500 Subject: [PATCH 23/57] skip debug fixups if there's no debug directory --- src/tasks/WasmAppBuilder/WebcilWriter.cs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/tasks/WasmAppBuilder/WebcilWriter.cs b/src/tasks/WasmAppBuilder/WebcilWriter.cs index c0cea12a64d99..8ec82b51f8ee0 100644 --- a/src/tasks/WasmAppBuilder/WebcilWriter.cs +++ b/src/tasks/WasmAppBuilder/WebcilWriter.cs @@ -69,9 +69,11 @@ public void Write() WriteHeader(outputStream, wcInfo.Header); WriteSectionHeaders(outputStream, wcInfo.SectionHeaders); CopySections(outputStream, inputStream, peInfo.SectionHeaders); - var wcDebugDirectoryEntries = FixupDebugDirectoryEntries(peInfo, wcInfo); - OverwriteDebugDirectoryEntries(outputStream, wcInfo, wcDebugDirectoryEntries); - outputStream.Flush(); + if (wcInfo.Header.pe_debug_size != 0 && wcInfo.Header.pe_debug_rva != 0) + { + var wcDebugDirectoryEntries = FixupDebugDirectoryEntries(peInfo, wcInfo); + OverwriteDebugDirectoryEntries(outputStream, wcInfo, wcDebugDirectoryEntries); + } } public record struct FilePosition(int Position) From c0bb6c6c0ac5b47cc72918fb98a5c0806f527bdb Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Thu, 29 Dec 2022 16:32:11 -0500 Subject: [PATCH 24/57] WebcilReader: implement CodeView and Emebedded PPDB support --- .../debugger/BrowserDebugProxy/DebugStore.cs | 2 +- .../BrowserDebugProxy/Webcil/WebcilReader.cs | 106 +++++++++++++++++- 2 files changed, 102 insertions(+), 6 deletions(-) diff --git a/src/mono/wasm/debugger/BrowserDebugProxy/DebugStore.cs b/src/mono/wasm/debugger/BrowserDebugProxy/DebugStore.cs index ffde2f6c53384..20a93e5111d5b 100644 --- a/src/mono/wasm/debugger/BrowserDebugProxy/DebugStore.cs +++ b/src/mono/wasm/debugger/BrowserDebugProxy/DebugStore.cs @@ -948,7 +948,7 @@ private static AssemblyInfo FromWebcilReader(MonoProxy monoProxy, SessionId sess if (entries.Length > 0) { var codeView = entries[0]; - if (codeView.Type == DebugDirectoryEntryType.CodeView && false) // FIXME: implement this + if (codeView.Type == DebugDirectoryEntryType.CodeView) { codeViewData = wcReader.ReadCodeViewDebugDirectoryData(codeView); } diff --git a/src/mono/wasm/debugger/BrowserDebugProxy/Webcil/WebcilReader.cs b/src/mono/wasm/debugger/BrowserDebugProxy/Webcil/WebcilReader.cs index 764feb3b90866..6ae3c41bedbde 100644 --- a/src/mono/wasm/debugger/BrowserDebugProxy/Webcil/WebcilReader.cs +++ b/src/mono/wasm/debugger/BrowserDebugProxy/Webcil/WebcilReader.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Immutable; using System.IO; +using System.Reflection; using System.Runtime.InteropServices; using System.Reflection.Metadata; @@ -14,7 +15,7 @@ namespace Microsoft.WebAssembly.Diagnostics.Webcil; public sealed class WebcilReader : IDisposable { - // TODO: WISH: + // WISH: // This should be implemented in terms of System.Reflection.Internal.MemoryBlockProvider like the PEReader, // but the memory block classes are internal to S.R.M. @@ -170,14 +171,109 @@ private static ImmutableArray ReadDebugDirectoryEntries(Blo public CodeViewDebugDirectoryData ReadCodeViewDebugDirectoryData(DebugDirectoryEntry entry) { - // TODO: implement by copying PEReader.DecodeCodeViewDebugDirectoryData - throw new NotImplementedException("FIXME: implement ReadCodeViewDebugDirectoryData"); + var pos = entry.DataPointer; + var buffer = new byte[entry.DataSize]; + if (_stream.Seek(pos, SeekOrigin.Begin) != pos) + { + throw new BadImageFormatException("Could not seek to CodeView debug directory data", nameof(_stream)); + } + if (_stream.Read(buffer, 0, buffer.Length) != buffer.Length) + { + throw new BadImageFormatException("Could not read CodeView debug directory data", nameof(_stream)); + } + unsafe + { + fixed (byte* p = buffer) + { + return DecodeCodeViewDebugDirectoryData(new BlobReader(p, buffer.Length)); + } + } + } + + private static CodeViewDebugDirectoryData DecodeCodeViewDebugDirectoryData(BlobReader reader) + { + // FIXME: copy-pasted from PEReader.DecodeCodeViewDebugDirectoryData + + if (reader.ReadByte() != (byte)'R' || + reader.ReadByte() != (byte)'S' || + reader.ReadByte() != (byte)'D' || + reader.ReadByte() != (byte)'S') + { + throw new BadImageFormatException("Unexpected CodeView data signature"); + } + + Guid guid = reader.ReadGuid(); + int age = reader.ReadInt32(); + string path = ReadUtf8NullTerminated(reader); + + return MakeCodeViewDebugDirectoryData(guid, age, path); + + } + + private static string ReadUtf8NullTerminated(BlobReader reader) + { + var mi = typeof(BlobReader).GetMethod("ReadUtf8NullTerminated", BindingFlags.NonPublic | BindingFlags.Instance); + return (string)mi.Invoke(reader, null); + } + + private static CodeViewDebugDirectoryData MakeCodeViewDebugDirectoryData(Guid guid, int age, string path) + { + var types = new Type[] { typeof(Guid), typeof(int), typeof(string) }; + var mi = typeof(CodeViewDebugDirectoryData).GetConstructor(BindingFlags.NonPublic | BindingFlags.Instance, types); + return (CodeViewDebugDirectoryData)mi.Invoke(new object[] { guid, age, path }); } public MetadataReaderProvider ReadEmbeddedPortablePdbDebugDirectoryData(DebugDirectoryEntry entry) { - // TODO: implement by copying PEReader.DecodeEmbeddedPortablePdbDebugDirectoryData - throw new NotImplementedException("FIXME: Implement ReadEmbeddedPortablePdbDebugDirectoryData"); + var pos = entry.DataPointer; + var buffer = new byte[entry.DataSize]; + if (_stream.Seek(pos, SeekOrigin.Begin) != pos) + { + throw new BadImageFormatException("Could not seek to Embedded Portable PDB debug directory data", nameof(_stream)); + } + if (_stream.Read(buffer, 0, buffer.Length) != buffer.Length) + { + throw new BadImageFormatException("Could not read Embedded Portable PDB debug directory data", nameof(_stream)); + } + unsafe + { + fixed (byte* p = buffer) + { + return DecodeEmbeddedPortablePdbDirectoryData(new BlobReader(p, buffer.Length)); + } + } + } + + private const uint PortablePdbVersions_DebugDirectoryEmbeddedSignature = 0x4244504d; + private static MetadataReaderProvider DecodeEmbeddedPortablePdbDirectoryData(BlobReader reader) + { + // FIXME: inspired by PEReader.DecodeEmbeddedPortablePdbDebugDirectoryData + // but not using its internal utility classes. + + if (reader.ReadUInt32() != PortablePdbVersions_DebugDirectoryEmbeddedSignature) + { + throw new BadImageFormatException("Unexpected embedded portable PDB data signature"); + } + + int decompressedSize = reader.ReadInt32(); + + byte[] decompressedBuffer; + + byte[] compressedBuffer = reader.ReadBytes(reader.RemainingBytes); + + using (var compressedStream = new MemoryStream(compressedBuffer, writable: false)) + using (var deflateStream = new System.IO.Compression.DeflateStream(compressedStream, System.IO.Compression.CompressionMode.Decompress, leaveOpen: true)) + { + decompressedBuffer = GC.AllocateUninitializedArray(decompressedSize); + using (var decompressedStream = new MemoryStream(decompressedBuffer, writable: true)) + { + deflateStream.CopyTo(decompressedStream); + } + } + + + return MetadataReaderProvider.FromPortablePdbStream(new MemoryStream(decompressedBuffer, writable: false)); + } private long TranslateRVA(uint rva) From e44841c4f5f4ea8b3c6640bc297f816ebd97268d Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Tue, 3 Jan 2023 10:21:40 -0500 Subject: [PATCH 25/57] [WBT] Add UseWebcil option (default to true) --- src/mono/wasm/Wasm.Build.Tests/BuildTestBase.cs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/mono/wasm/Wasm.Build.Tests/BuildTestBase.cs b/src/mono/wasm/Wasm.Build.Tests/BuildTestBase.cs index 0b4777ae7ac31..23502a8849cb9 100644 --- a/src/mono/wasm/Wasm.Build.Tests/BuildTestBase.cs +++ b/src/mono/wasm/Wasm.Build.Tests/BuildTestBase.cs @@ -423,7 +423,8 @@ protected static BuildArgs ExpandBuildArgs(BuildArgs buildArgs, string extraProp options.HasV8Script, options.TargetFramework ?? DefaultTargetFramework, options.HasIcudt, - options.DotnetWasmFromRuntimePack ?? !buildArgs.AOT); + options.DotnetWasmFromRuntimePack ?? !buildArgs.AOT, + options.UseWebcil); } if (options.UseCache) @@ -616,7 +617,8 @@ protected static void AssertBasicAppBundle(string bundleDir, bool hasV8Script, string targetFramework, bool hasIcudt = true, - bool dotnetWasmFromRuntimePack = true) + bool dotnetWasmFromRuntimePack = true, + bool useWebcil = true) { AssertFilesExist(bundleDir, new [] { @@ -632,7 +634,9 @@ protected static void AssertBasicAppBundle(string bundleDir, AssertFilesExist(bundleDir, new[] { "icudt.dat" }, expectToExist: hasIcudt); string managedDir = Path.Combine(bundleDir, "managed"); - AssertFilesExist(managedDir, new[] { $"{projectName}.dll" }); + string bundledMainAppAssembly = + useWebcil ? $"{projectName}.webcil" : $"{projectName}.dll"; + AssertFilesExist(managedDir, new[] { bundledMainAppAssembly }); bool is_debug = config == "Debug"; if (is_debug) @@ -1095,6 +1099,7 @@ public record BuildProjectOptions bool Publish = true, bool BuildOnlyAfterPublish = true, bool HasV8Script = true, + bool UseWebcil = true, string? Verbosity = null, string? Label = null, string? TargetFramework = null, From 96b6a1deca9fff71b74001b9237d32d2251fb8f1 Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Tue, 3 Jan 2023 12:46:54 -0500 Subject: [PATCH 26/57] rename WebcilWriter -> WebcilConverter [NFC] --- src/tasks/WasmAppBuilder/WasmAppBuilder.cs | 8 ++++---- .../{WebcilWriter.cs => WebcilConverter.cs} | 13 +++++++++---- 2 files changed, 13 insertions(+), 8 deletions(-) rename src/tasks/WasmAppBuilder/{WebcilWriter.cs => WebcilConverter.cs} (96%) diff --git a/src/tasks/WasmAppBuilder/WasmAppBuilder.cs b/src/tasks/WasmAppBuilder/WasmAppBuilder.cs index e3c46ed75484e..1ec016f6c92a5 100644 --- a/src/tasks/WasmAppBuilder/WasmAppBuilder.cs +++ b/src/tasks/WasmAppBuilder/WasmAppBuilder.cs @@ -196,8 +196,8 @@ private bool ExecuteInternal () if (!DisableWebcil) { var tmpWebcil = Path.GetTempFileName(); - var webcilWriter = new Microsoft.WebAssembly.Build.Tasks.WebcilWriter(inputPath: assembly, outputPath: tmpWebcil, logger: Log); - webcilWriter.Write(); + var webcilWriter = Microsoft.WebAssembly.Build.Tasks.WebcilConverter.FromPortableExecutable(inputPath: assembly, outputPath: tmpWebcil, logger: Log); + webcilWriter.ConvertToWebcil(); var finalWebcil = Path.ChangeExtension(assembly, ".webcil"); FileCopyChecked(tmpWebcil, Path.Combine(asmRootPath, Path.GetFileName(finalWebcil)), "Assemblies"); } @@ -282,8 +282,8 @@ private bool ExecuteInternal () if (!DisableWebcil) { var tmpWebcil = Path.GetTempFileName(); - var webcilWriter = new Microsoft.WebAssembly.Build.Tasks.WebcilWriter(inputPath: fullPath, outputPath: tmpWebcil, logger: Log); - webcilWriter.Write(); + var webcilWriter = Microsoft.WebAssembly.Build.Tasks.WebcilConverter.FromPortableExecutable(inputPath: fullPath, outputPath: tmpWebcil, logger: Log); + webcilWriter.ConvertToWebcil(); var finalWebcil = Path.ChangeExtension(name, ".webcil"); FileCopyChecked(tmpWebcil, Path.Combine(directory, finalWebcil), "SatelliteAssemblies"); config.Assets.Add(new SatelliteAssemblyEntry(finalWebcil, culture)); diff --git a/src/tasks/WasmAppBuilder/WebcilWriter.cs b/src/tasks/WasmAppBuilder/WebcilConverter.cs similarity index 96% rename from src/tasks/WasmAppBuilder/WebcilWriter.cs rename to src/tasks/WasmAppBuilder/WebcilConverter.cs index 8ec82b51f8ee0..93050abbc4516 100644 --- a/src/tasks/WasmAppBuilder/WebcilWriter.cs +++ b/src/tasks/WasmAppBuilder/WebcilConverter.cs @@ -16,7 +16,7 @@ namespace Microsoft.WebAssembly.Build.Tasks; /// /// Reads a .NET assembly in a normal PE COFF file and writes it out as a Webcil file /// -public class WebcilWriter +public class WebcilConverter { // Interesting stuff we've learned about the input PE file @@ -45,16 +45,21 @@ FilePosition SectionStart private readonly string _outputPath; private TaskLoggingHelper Log { get; } - public WebcilWriter(string inputPath, string outputPath, TaskLoggingHelper logger) + private WebcilConverter(string inputPath, string outputPath, TaskLoggingHelper logger) { _inputPath = inputPath; _outputPath = outputPath; Log = logger; } - public void Write() + public static WebcilConverter FromPortableExecutable(string inputPath, string outputPath, TaskLoggingHelper logger) { - Log.LogMessage($"Writing Webcil (input {_inputPath}) output to {_outputPath}"); + return new WebcilConverter(inputPath, outputPath, logger); + } + + public void ConvertToWebcil() + { + Log.LogMessage($"Converting to Webcil: input {_inputPath} output: {_outputPath}"); using var inputStream = File.Open(_inputPath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); PEFileInfo peInfo; From 2cd2c927429af4f7c19e16c65b70bb7278e341e2 Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Tue, 3 Jan 2023 16:43:37 -0500 Subject: [PATCH 27/57] fixup AssemblyLoadedEventTest --- src/mono/wasm/debugger/DebuggerTestSuite/MonoJsTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mono/wasm/debugger/DebuggerTestSuite/MonoJsTests.cs b/src/mono/wasm/debugger/DebuggerTestSuite/MonoJsTests.cs index 93d7221c51f7e..139daae4ae326 100644 --- a/src/mono/wasm/debugger/DebuggerTestSuite/MonoJsTests.cs +++ b/src/mono/wasm/debugger/DebuggerTestSuite/MonoJsTests.cs @@ -136,7 +136,7 @@ async Task AssemblyLoadedEventTest(string asm_name, string asm_path, string pdb_ : await Task.FromResult(ProtocolEventHandlerReturn.KeepHandler); }); - byte[] bytes = File.ReadAllBytes(asm_path); + byte[] bytes = File.Exists(asm_path) ? File.ReadAllBytes(asm_path) : File.ReadAllBytes(Path.ChangeExtension(asm_path, ".webcil")); // hack! string asm_base64 = Convert.ToBase64String(bytes); string pdb_base64 = String.Empty; From c3b7726fdaac863fefe755bcede16a38b677f84e Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Tue, 3 Jan 2023 16:46:33 -0500 Subject: [PATCH 28/57] hack: no extension on assembly for breakpoint --- src/mono/wasm/debugger/DebuggerTestSuite/DebuggerTestBase.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mono/wasm/debugger/DebuggerTestSuite/DebuggerTestBase.cs b/src/mono/wasm/debugger/DebuggerTestSuite/DebuggerTestBase.cs index e2ca58fe62054..d5cac7e9a3d4f 100644 --- a/src/mono/wasm/debugger/DebuggerTestSuite/DebuggerTestBase.cs +++ b/src/mono/wasm/debugger/DebuggerTestSuite/DebuggerTestBase.cs @@ -275,7 +275,7 @@ public async Task WaitForScriptParsedEventsAsync(params string[] paths) // sets breakpoint by method name and line offset internal async Task CheckInspectLocalsAtBreakpointSite(string type, string method, int line_offset, string bp_function_name, string eval_expression, - Func locals_fn = null, Func wait_for_event_fn = null, bool use_cfo = false, string assembly = "debugger-test.dll", int col = 0) + Func locals_fn = null, Func wait_for_event_fn = null, bool use_cfo = false, string assembly = "debugger-test", int col = 0) { UseCallFunctionOnBeforeGetProperties = use_cfo; From 4f257601276a648bf0fbd3127011eab0154358ae Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Wed, 4 Jan 2023 14:42:41 -0500 Subject: [PATCH 29/57] pass normal .dll name for MainAssemblyName in config let the runtime deal with it - bundle matching will resolve it to the .webcil file --- src/tasks/WasmAppBuilder/WasmAppBuilder.cs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/tasks/WasmAppBuilder/WasmAppBuilder.cs b/src/tasks/WasmAppBuilder/WasmAppBuilder.cs index 1ec016f6c92a5..c121e226ce48e 100644 --- a/src/tasks/WasmAppBuilder/WasmAppBuilder.cs +++ b/src/tasks/WasmAppBuilder/WasmAppBuilder.cs @@ -178,13 +178,10 @@ private bool ExecuteInternal () _assemblies.Add(asm); } MainAssemblyName = Path.GetFileName(MainAssemblyName); - var mainAssemblyNameForConfig = MainAssemblyName; - if (!DisableWebcil) - mainAssemblyNameForConfig = Path.ChangeExtension(mainAssemblyNameForConfig, ".webcil"); var config = new WasmAppConfig () { - MainAssemblyName = mainAssemblyNameForConfig, + MainAssemblyName = MainAssemblyName, }; // Create app From d82140f763d2e8721db51b6c07b5f3608606449f Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Fri, 6 Jan 2023 11:00:38 -0500 Subject: [PATCH 30/57] Wasm.Debugger.Tests: give CI 10 more minutes --- src/mono/wasm/debugger/Wasm.Debugger.Tests/wasm.helix.targets | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mono/wasm/debugger/Wasm.Debugger.Tests/wasm.helix.targets b/src/mono/wasm/debugger/Wasm.Debugger.Tests/wasm.helix.targets index e874cc4239df4..b9a4726d94f5c 100644 --- a/src/mono/wasm/debugger/Wasm.Debugger.Tests/wasm.helix.targets +++ b/src/mono/wasm/debugger/Wasm.Debugger.Tests/wasm.helix.targets @@ -3,8 +3,8 @@ true $(DebuggerHost)- true - <_DebuggerTestsWorkItemTimeout Condition="'$(Scenario)' == 'WasmDebuggerTests'">00:20:00 - <_DebuggerTestsWorkItemTimeout Condition="'$(Scenario)' == 'WasmDebuggerTests' and '$(BrowserHost)' == 'windows'">00:30:00 + <_DebuggerTestsWorkItemTimeout Condition="'$(Scenario)' == 'WasmDebuggerTests'">00:30:00 + <_DebuggerTestsWorkItemTimeout Condition="'$(Scenario)' == 'WasmDebuggerTests' and '$(BrowserHost)' == 'windows'">00:40:00 $(HelixExtensionTargets);_AddWorkItemsForWasmDebuggerTests From c1d5169ff268063ade190825a732103b2ea4c557 Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Fri, 6 Jan 2023 13:24:34 -0500 Subject: [PATCH 31/57] Add Microsoft.NET.WebAssembly.Webcil assembly project Mark it as shipping, but not shipping a nuget package. The idea is that it will be shipped along with the WasmAppBuilder msbuild task, and with the BrowserDebugProxy tool. --- .../Directory.Build.props | 9 +++++++++ .../Microsoft.NET.WebAssembly.Webcil.csproj | 18 +++++++++++++++++ .../src/Webcil/WebcilConverter.cs | 20 +++++++++++++++++++ .../BrowserDebugProxy.csproj | 4 ++++ .../WasmAppBuilder/WasmAppBuilder.csproj | 7 +++++++ 5 files changed, 58 insertions(+) create mode 100644 src/libraries/Microsoft.NET.WebAssembly.Webcil/Directory.Build.props create mode 100644 src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Microsoft.NET.WebAssembly.Webcil.csproj create mode 100644 src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilConverter.cs diff --git a/src/libraries/Microsoft.NET.WebAssembly.Webcil/Directory.Build.props b/src/libraries/Microsoft.NET.WebAssembly.Webcil/Directory.Build.props new file mode 100644 index 0000000000000..fe703f0232848 --- /dev/null +++ b/src/libraries/Microsoft.NET.WebAssembly.Webcil/Directory.Build.props @@ -0,0 +1,9 @@ + + + + true + + false + + diff --git a/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Microsoft.NET.WebAssembly.Webcil.csproj b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Microsoft.NET.WebAssembly.Webcil.csproj new file mode 100644 index 0000000000000..731237e691626 --- /dev/null +++ b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Microsoft.NET.WebAssembly.Webcil.csproj @@ -0,0 +1,18 @@ + + + $(NetCoreAppToolCurrent);$(NetFrameworkToolCurrent) + Abstractions for modifying .NET webcil binary images + true + true + true + + + + + + + + + + + diff --git a/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilConverter.cs b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilConverter.cs new file mode 100644 index 0000000000000..43d1c950f7d31 --- /dev/null +++ b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilConverter.cs @@ -0,0 +1,20 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; +using System.Collections.Immutable; +using System.Reflection.PortableExecutable; +using System.Runtime.InteropServices; + +using Microsoft.Build.Utilities; + +namespace Microsoft.NET.WebAssembly.Webcil; + +public class WebcilConverter +{ + public WebcilConverter() + { + } +} + diff --git a/src/mono/wasm/debugger/BrowserDebugProxy/BrowserDebugProxy.csproj b/src/mono/wasm/debugger/BrowserDebugProxy/BrowserDebugProxy.csproj index c05515cdd1062..a4cc5817c76af 100644 --- a/src/mono/wasm/debugger/BrowserDebugProxy/BrowserDebugProxy.csproj +++ b/src/mono/wasm/debugger/BrowserDebugProxy/BrowserDebugProxy.csproj @@ -14,4 +14,8 @@ + + + + diff --git a/src/tasks/WasmAppBuilder/WasmAppBuilder.csproj b/src/tasks/WasmAppBuilder/WasmAppBuilder.csproj index 242966a2914d3..3fd2f17c30a25 100644 --- a/src/tasks/WasmAppBuilder/WasmAppBuilder.csproj +++ b/src/tasks/WasmAppBuilder/WasmAppBuilder.csproj @@ -26,6 +26,10 @@ + + + + @@ -33,6 +37,9 @@ TargetPath="tasks\$(TargetFrameworkForNETCoreTasks)" /> + + Date: Fri, 6 Jan 2023 14:13:37 -0500 Subject: [PATCH 32/57] Move WebcilConverter to Microsoft.NET.WebAssembly.Webcil --- .../Directory.Build.props | 2 + .../src/Common/IsExternalInit.cs | 7 + .../Microsoft.NET.WebAssembly.Webcil.csproj | 10 + .../src/Webcil/Internal}/Constants.cs | 4 +- .../src/Webcil/WebciHeader.cs} | 4 +- .../src/Webcil/WebcilConverter.cs | 329 +++++++++++++++++- .../src}/Webcil/WebcilSectionHeader.cs | 2 +- src/tasks/WasmAppBuilder/WebcilConverter.cs | 320 +---------------- 8 files changed, 356 insertions(+), 322 deletions(-) create mode 100644 src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Common/IsExternalInit.cs rename src/{tasks/WasmAppBuilder/Webcil => libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/Internal}/Constants.cs (66%) rename src/{tasks/WasmAppBuilder/Webcil/WCHeader.cs => libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebciHeader.cs} (90%) rename src/{tasks/WasmAppBuilder => libraries/Microsoft.NET.WebAssembly.Webcil/src}/Webcil/WebcilSectionHeader.cs (94%) diff --git a/src/libraries/Microsoft.NET.WebAssembly.Webcil/Directory.Build.props b/src/libraries/Microsoft.NET.WebAssembly.Webcil/Directory.Build.props index fe703f0232848..6bda1e7da2544 100644 --- a/src/libraries/Microsoft.NET.WebAssembly.Webcil/Directory.Build.props +++ b/src/libraries/Microsoft.NET.WebAssembly.Webcil/Directory.Build.props @@ -5,5 +5,7 @@ false + + false diff --git a/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Common/IsExternalInit.cs b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Common/IsExternalInit.cs new file mode 100644 index 0000000000000..d7be691310347 --- /dev/null +++ b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Common/IsExternalInit.cs @@ -0,0 +1,7 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace System.Runtime.CompilerServices +{ + internal sealed class IsExternalInit { } +} diff --git a/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Microsoft.NET.WebAssembly.Webcil.csproj b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Microsoft.NET.WebAssembly.Webcil.csproj index 731237e691626..6da8aac28b686 100644 --- a/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Microsoft.NET.WebAssembly.Webcil.csproj +++ b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Microsoft.NET.WebAssembly.Webcil.csproj @@ -5,6 +5,7 @@ true true true + false @@ -13,6 +14,15 @@ + + + + + + + + + diff --git a/src/tasks/WasmAppBuilder/Webcil/Constants.cs b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/Internal/Constants.cs similarity index 66% rename from src/tasks/WasmAppBuilder/Webcil/Constants.cs rename to src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/Internal/Constants.cs index 4ab2b548c34cc..2324dbdf633f9 100644 --- a/src/tasks/WasmAppBuilder/Webcil/Constants.cs +++ b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/Internal/Constants.cs @@ -1,9 +1,9 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -namespace Microsoft.WebAssembly.Build.Tasks.WebCil; +namespace Microsoft.NET.WebAssembly.Webcil.Internal; -public static unsafe class Constants +internal static unsafe class Constants { public const int WC_VERSION = 0; } diff --git a/src/tasks/WasmAppBuilder/Webcil/WCHeader.cs b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebciHeader.cs similarity index 90% rename from src/tasks/WasmAppBuilder/Webcil/WCHeader.cs rename to src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebciHeader.cs index 378efe9918570..8c326b7f7b6ce 100644 --- a/src/tasks/WasmAppBuilder/Webcil/WCHeader.cs +++ b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebciHeader.cs @@ -3,7 +3,7 @@ using System.Runtime.InteropServices; -namespace Microsoft.WebAssembly.Build.Tasks.WebCil; +namespace Microsoft.NET.WebAssembly.Webcil; /// /// The header of a WebCIL file. @@ -13,7 +13,7 @@ namespace Microsoft.WebAssembly.Build.Tasks.WebCil; /// The header is a subset of the PE, COFF and CLI headers that are needed by the mono runtime to load managed assemblies. /// [StructLayout(LayoutKind.Sequential, Pack = 1)] -public unsafe struct WCHeader +public unsafe struct WebcilHeader { public fixed byte id[2]; // 'W' 'C' public byte version; diff --git a/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilConverter.cs b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilConverter.cs index 43d1c950f7d31..44bf4a0aad6b2 100644 --- a/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilConverter.cs +++ b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilConverter.cs @@ -7,14 +7,335 @@ using System.Reflection.PortableExecutable; using System.Runtime.InteropServices; -using Microsoft.Build.Utilities; - namespace Microsoft.NET.WebAssembly.Webcil; +/// +/// Reads a .NET assembly in a normal PE COFF file and writes it out as a Webcil file +/// public class WebcilConverter { - public WebcilConverter() + + // Interesting stuff we've learned about the input PE file + public record PEFileInfo( + // The sections in the PE file + ImmutableArray SectionHeaders, + // The location of the debug directory entries + DirectoryEntry DebugTableDirectory, + // The file offset of the sections, following the section directory + FilePosition SectionStart, + // The debug directory entries + ImmutableArray DebugDirectoryEntries + ); + + // Intersting stuff we know about the webcil file we're writing + public record WCFileInfo( + // The header of the webcil file + WebcilHeader Header, + // The section directory of the webcil file + ImmutableArray SectionHeaders, + // The file offset of the sections, following the section directory + FilePosition SectionStart + ); + + private readonly string _inputPath; + private readonly string _outputPath; + + private WebcilConverter(string inputPath, string outputPath) { + _inputPath = inputPath; + _outputPath = outputPath; + } + + public static WebcilConverter FromPortableExecutable(string inputPath, string outputPath) + { + return new WebcilConverter(inputPath, outputPath); + } + + public void ConvertToWebcil() + { + using var inputStream = File.Open(_inputPath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); + PEFileInfo peInfo; + WCFileInfo wcInfo; + using (var peReader = new PEReader(inputStream, PEStreamOptions.LeaveOpen)) + { + GatherInfo(peReader, out wcInfo, out peInfo); + } + + using var outputStream = File.Open(_outputPath, FileMode.Create, FileAccess.Write); + WriteHeader(outputStream, wcInfo.Header); + WriteSectionHeaders(outputStream, wcInfo.SectionHeaders); + CopySections(outputStream, inputStream, peInfo.SectionHeaders); + if (wcInfo.Header.pe_debug_size != 0 && wcInfo.Header.pe_debug_rva != 0) + { + var wcDebugDirectoryEntries = FixupDebugDirectoryEntries(peInfo, wcInfo); + OverwriteDebugDirectoryEntries(outputStream, wcInfo, wcDebugDirectoryEntries); + } + } + + public record struct FilePosition(int Position) + { + public static implicit operator FilePosition(int position) => new(position); + + public static FilePosition operator +(FilePosition left, int right) => new(left.Position + right); + } + + private static unsafe int SizeOfHeader() + { + return sizeof(WebcilHeader); + } + + public unsafe void GatherInfo(PEReader peReader, out WCFileInfo wcInfo, out PEFileInfo peInfo) + { + var headers = peReader.PEHeaders; + var peHeader = headers.PEHeader!; + var coffHeader = headers.CoffHeader!; + var sections = headers.SectionHeaders; + WebcilHeader header; + header.id[0] = (byte)'W'; + header.id[1] = (byte)'C'; + header.version = Internal.Constants.WC_VERSION; + header.reserved0 = 0; + header.coff_sections = (ushort)coffHeader.NumberOfSections; + header.reserved1 = 0; + header.pe_cli_header_rva = (uint)peHeader.CorHeaderTableDirectory.RelativeVirtualAddress; + header.pe_cli_header_size = (uint)peHeader.CorHeaderTableDirectory.Size; + header.pe_debug_rva = (uint)peHeader.DebugTableDirectory.RelativeVirtualAddress; + header.pe_debug_size = (uint)peHeader.DebugTableDirectory.Size; + + // current logical position in the output file + FilePosition pos = SizeOfHeader(); + // position of the current section in the output file + // initially it's after all the section headers + FilePosition curSectionPos = pos + sizeof(WebcilSectionHeader) * coffHeader.NumberOfSections; + // The first WC section is immediately after the section directory + FilePosition firstWCSection = curSectionPos; + + FilePosition firstPESection = 0; + + ImmutableArray.Builder headerBuilder = ImmutableArray.CreateBuilder(coffHeader.NumberOfSections); + foreach (var sectionHeader in sections) + { + // The first section is the one with the lowest file offset + if (firstPESection.Position == 0) + { + firstPESection = sectionHeader.PointerToRawData; + } + else + { + firstPESection = Math.Min(firstPESection.Position, sectionHeader.PointerToRawData); + } + + var newHeader = new WebcilSectionHeader + ( + virtualSize: sectionHeader.VirtualSize, + virtualAddress: sectionHeader.VirtualAddress, + sizeOfRawData: sectionHeader.SizeOfRawData, + pointerToRawData: curSectionPos.Position + ); + + pos += sizeof(WebcilSectionHeader); + curSectionPos += sectionHeader.SizeOfRawData; + headerBuilder.Add(newHeader); + } + + + ImmutableArray debugDirectoryEntries = peReader.ReadDebugDirectory(); + + peInfo = new PEFileInfo(SectionHeaders: sections, + DebugTableDirectory: peHeader.DebugTableDirectory, + SectionStart: firstPESection, + DebugDirectoryEntries: debugDirectoryEntries); + + wcInfo = new WCFileInfo(Header: header, + SectionHeaders: headerBuilder.MoveToImmutable(), + SectionStart: firstWCSection); + } + + private static void WriteHeader(Stream s, WebcilHeader header) + { + WriteStructure(s, header); + } + + private static void WriteSectionHeaders(Stream s, ImmutableArray sectionsHeaders) + { + // FIXME: fixup endianness + if (!BitConverter.IsLittleEndian) + throw new NotImplementedException(); + foreach (var sectionHeader in sectionsHeaders) + { + WriteSectionHeader(s, sectionHeader); + } } -} + private static void WriteSectionHeader(Stream s, WebcilSectionHeader sectionHeader) + { + WriteStructure(s, sectionHeader); + } + +#if NETCOREAPP2_1_OR_GREATER + private static void WriteStructure(Stream s, T structure) + where T : unmanaged + { + // FIXME: fixup endianness + if (!BitConverter.IsLittleEndian) + throw new NotImplementedException(); + unsafe + { + byte* p = (byte*)&structure; + s.Write(new ReadOnlySpan(p, sizeof(T))); + } + } +#else + private static void WriteStructure(Stream s, T structure) + where T : unmanaged + { + // FIXME: fixup endianness + if (!BitConverter.IsLittleEndian) + throw new NotImplementedException(); + int size = Marshal.SizeOf(); + byte[] buffer = new byte[size]; + IntPtr ptr = IntPtr.Zero; + try + { + ptr = Marshal.AllocHGlobal(size); + Marshal.StructureToPtr(structure, ptr, false); + Marshal.Copy(ptr, buffer, 0, size); + } + finally + { + Marshal.FreeHGlobal(ptr); + } + s.Write(buffer, 0, size); + } +#endif + + private static void CopySections(Stream outStream, Stream inputStream, ImmutableArray peSections) + { + // endianness: ok, we're just copying from one stream to another + foreach (var peHeader in peSections) + { + var buffer = new byte[peHeader.SizeOfRawData]; + inputStream.Seek(peHeader.PointerToRawData, SeekOrigin.Begin); + ReadExactly(inputStream, buffer); + outStream.Write(buffer, 0, buffer.Length); + } + } + +#if NETCOREAPP2_1_OR_GREATER + private static void ReadExactly(Stream s, Span buffer) + { + s.ReadExactly(buffer); + } +#else + private static void ReadExactly(Stream s, byte[] buffer) + { + int offset = 0; + while (offset < buffer.Length) + { + int read = s.Read(buffer, offset, buffer.Length - offset); + if (read == 0) + throw new EndOfStreamException(); + offset += read; + } + } +#endif + + private static FilePosition GetPositionOfRelativeVirtualAddress(ImmutableArray wcSections, uint relativeVirtualAddress) + { + foreach (var section in wcSections) + { + if (relativeVirtualAddress >= section.VirtualAddress && relativeVirtualAddress < section.VirtualAddress + section.VirtualSize) + { + FilePosition pos = section.PointerToRawData + ((int)relativeVirtualAddress - section.VirtualAddress); + return pos; + } + } + + throw new InvalidOperationException("relative virtual address not in any section"); + } + + // Given a physical file offset, return the section and the offset within the section. + private static (WebcilSectionHeader section, int offset) GetSectionFromFileOffset(ImmutableArray peSections, FilePosition fileOffset) + { + foreach (var section in peSections) + { + if (fileOffset.Position >= section.PointerToRawData && fileOffset.Position < section.PointerToRawData + section.SizeOfRawData) + { + return (section, fileOffset.Position - section.PointerToRawData); + } + } + + throw new InvalidOperationException("file offset not in any section (Webcil)"); + } + + private static void GetSectionFromFileOffset(ImmutableArray sections, FilePosition fileOffset) + { + foreach (var section in sections) + { + if (fileOffset.Position >= section.PointerToRawData && fileOffset.Position < section.PointerToRawData + section.SizeOfRawData) + { + return; + } + } + + throw new InvalidOperationException($"file offset {fileOffset.Position} not in any section (PE)"); + } + + // Make a new set of debug directory entries that + // have their data pointers adjusted to be relative to the start of the webcil file. + // This is necessary because the debug directory entires in the PE file are relative to the start of the PE file, + // and a PE header is bigger than a webcil header. + private static ImmutableArray FixupDebugDirectoryEntries(PEFileInfo peInfo, WCFileInfo wcInfo) + { + int dataPointerAdjustment = peInfo.SectionStart.Position - wcInfo.SectionStart.Position; + ImmutableArray entries = peInfo.DebugDirectoryEntries; + ImmutableArray.Builder newEntries = ImmutableArray.CreateBuilder(entries.Length); + foreach (var entry in entries) + { + DebugDirectoryEntry newEntry; + if (entry.Type == DebugDirectoryEntryType.Reproducible || entry.DataPointer == 0 || entry.DataSize == 0) + { + // this entry doesn't have an associated data pointer, so just copy it + newEntry = entry; + } + else + { + // the "pointer" field is a file offset, find the corresponding entry in the Webcil file and overwrite with the correct file position + var newDataPointer = entry.DataPointer - dataPointerAdjustment; + newEntry = new DebugDirectoryEntry(entry.Stamp, entry.MajorVersion, entry.MinorVersion, entry.Type, entry.DataSize, entry.DataRelativeVirtualAddress, newDataPointer); + GetSectionFromFileOffset(peInfo.SectionHeaders, entry.DataPointer); + // validate that the new entry is in some section + GetSectionFromFileOffset(wcInfo.SectionHeaders, newDataPointer); + } + newEntries.Add(newEntry); + } + return newEntries.MoveToImmutable(); + } + + private static void OverwriteDebugDirectoryEntries(Stream s, WCFileInfo wcInfo, ImmutableArray entries) + { + FilePosition debugDirectoryPos = GetPositionOfRelativeVirtualAddress(wcInfo.SectionHeaders, wcInfo.Header.pe_debug_rva); + using var writer = new BinaryWriter(s, System.Text.Encoding.UTF8, leaveOpen: true); + writer.Seek(debugDirectoryPos.Position, SeekOrigin.Begin); + foreach (var entry in entries) + { + WriteDebugDirectoryEntry(writer, entry); + } + // TODO check that we overwrite with the same size as the original + + // restore the stream position + writer.Seek(0, SeekOrigin.End); + } + + private static void WriteDebugDirectoryEntry(BinaryWriter writer, DebugDirectoryEntry entry) + { + writer.Write((uint)0); // Characteristics + writer.Write(entry.Stamp); + writer.Write(entry.MajorVersion); + writer.Write(entry.MinorVersion); + writer.Write((uint)entry.Type); + writer.Write(entry.DataSize); + writer.Write(entry.DataRelativeVirtualAddress); + writer.Write(entry.DataPointer); + } +} diff --git a/src/tasks/WasmAppBuilder/Webcil/WebcilSectionHeader.cs b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilSectionHeader.cs similarity index 94% rename from src/tasks/WasmAppBuilder/Webcil/WebcilSectionHeader.cs rename to src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilSectionHeader.cs index ea8e3a1b30063..8571fb8ac325c 100644 --- a/src/tasks/WasmAppBuilder/Webcil/WebcilSectionHeader.cs +++ b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilSectionHeader.cs @@ -3,7 +3,7 @@ using System.Runtime.InteropServices; -namespace Microsoft.WebAssembly.Build.Tasks.WebCil; +namespace Microsoft.NET.WebAssembly.Webcil; /// /// This is the Webcil analog of System.Reflection.PortableExecutable.SectionHeader, but with fewer fields diff --git a/src/tasks/WasmAppBuilder/WebcilConverter.cs b/src/tasks/WasmAppBuilder/WebcilConverter.cs index 93050abbc4516..3cd8b1fc74514 100644 --- a/src/tasks/WasmAppBuilder/WebcilConverter.cs +++ b/src/tasks/WasmAppBuilder/WebcilConverter.cs @@ -5,12 +5,9 @@ using System.IO; using System.Collections.Immutable; using System.Reflection.PortableExecutable; -using System.Runtime.InteropServices; using Microsoft.Build.Utilities; -using Microsoft.WebAssembly.Build.Tasks.WebCil; - namespace Microsoft.WebAssembly.Build.Tasks; /// @@ -18,35 +15,15 @@ namespace Microsoft.WebAssembly.Build.Tasks; /// public class WebcilConverter { - - // Interesting stuff we've learned about the input PE file - public record PEFileInfo( - // The sections in the PE file - ImmutableArray SectionHeaders, - // The location of the debug directory entries - DirectoryEntry DebugTableDirectory, - // The file offset of the sections, following the section directory - FilePosition SectionStart, - // The debug directory entries - ImmutableArray DebugDirectoryEntries - ); - - // Intersting stuff we know about the webcil file we're writing - public record WCFileInfo( - // The header of the webcil file - WCHeader Header, - // The section directory of the webcil file - ImmutableArray SectionHeaders, - // The file offset of the sections, following the section directory - FilePosition SectionStart - ); - private readonly string _inputPath; private readonly string _outputPath; + private readonly NET.WebAssembly.Webcil.WebcilConverter _converter; + private TaskLoggingHelper Log { get; } - private WebcilConverter(string inputPath, string outputPath, TaskLoggingHelper logger) + private WebcilConverter(NET.WebAssembly.Webcil.WebcilConverter converter, string inputPath, string outputPath, TaskLoggingHelper logger) { + _converter = converter; _inputPath = inputPath; _outputPath = outputPath; Log = logger; @@ -54,297 +31,14 @@ private WebcilConverter(string inputPath, string outputPath, TaskLoggingHelper l public static WebcilConverter FromPortableExecutable(string inputPath, string outputPath, TaskLoggingHelper logger) { - return new WebcilConverter(inputPath, outputPath, logger); + var converter = NET.WebAssembly.Webcil.WebcilConverter.FromPortableExecutable(inputPath, outputPath); + return new WebcilConverter(converter, inputPath, outputPath, logger); } public void ConvertToWebcil() { Log.LogMessage($"Converting to Webcil: input {_inputPath} output: {_outputPath}"); - - using var inputStream = File.Open(_inputPath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); - PEFileInfo peInfo; - WCFileInfo wcInfo; - using (var peReader = new PEReader(inputStream, PEStreamOptions.LeaveOpen)) - { - // DumpPE(peReader); - GatherInfo(peReader, out wcInfo, out peInfo); - } - - using var outputStream = File.Open(_outputPath, FileMode.Create, FileAccess.Write); - WriteHeader(outputStream, wcInfo.Header); - WriteSectionHeaders(outputStream, wcInfo.SectionHeaders); - CopySections(outputStream, inputStream, peInfo.SectionHeaders); - if (wcInfo.Header.pe_debug_size != 0 && wcInfo.Header.pe_debug_rva != 0) - { - var wcDebugDirectoryEntries = FixupDebugDirectoryEntries(peInfo, wcInfo); - OverwriteDebugDirectoryEntries(outputStream, wcInfo, wcDebugDirectoryEntries); - } - } - - public record struct FilePosition(int Position) - { - public static implicit operator FilePosition(int position) => new(position); - - public static FilePosition operator +(FilePosition left, int right) => new(left.Position + right); - } - - private static unsafe int SizeOfHeader() - { - return sizeof(WCHeader); - } - - public unsafe void GatherInfo(PEReader peReader, out WCFileInfo wcInfo, out PEFileInfo peInfo) - { - var headers = peReader.PEHeaders; - var peHeader = headers.PEHeader!; - var coffHeader = headers.CoffHeader!; - var sections = headers.SectionHeaders; - WCHeader header; - header.id[0] = (byte)'W'; - header.id[1] = (byte)'C'; - header.version = Constants.WC_VERSION; - header.reserved0 = 0; - header.coff_sections = (ushort)coffHeader.NumberOfSections; - header.reserved1 = 0; - header.pe_cli_header_rva = (uint)peHeader.CorHeaderTableDirectory.RelativeVirtualAddress; - header.pe_cli_header_size = (uint)peHeader.CorHeaderTableDirectory.Size; - header.pe_debug_rva = (uint)peHeader.DebugTableDirectory.RelativeVirtualAddress; - header.pe_debug_size = (uint)peHeader.DebugTableDirectory.Size; - - // current logical position in the output file - FilePosition pos = SizeOfHeader(); - // position of the current section in the output file - // initially it's after all the section headers - FilePosition curSectionPos = pos + sizeof(WebcilSectionHeader) * coffHeader.NumberOfSections; - // The first WC section is immediately after the section directory - FilePosition firstWCSection = curSectionPos; - - FilePosition firstPESection = 0; - - ImmutableArray.Builder headerBuilder = ImmutableArray.CreateBuilder(coffHeader.NumberOfSections); - foreach (var sectionHeader in sections) - { - // The first section is the one with the lowest file offset - if (firstPESection.Position == 0) - { - firstPESection = sectionHeader.PointerToRawData; - } - else - { - firstPESection = Math.Min(firstPESection.Position, sectionHeader.PointerToRawData); - } - - var newHeader = new WebcilSectionHeader - ( - virtualSize: sectionHeader.VirtualSize, - virtualAddress: sectionHeader.VirtualAddress, - sizeOfRawData: sectionHeader.SizeOfRawData, - pointerToRawData: curSectionPos.Position - ); - - pos += sizeof(WebcilSectionHeader); - curSectionPos += sectionHeader.SizeOfRawData; - headerBuilder.Add(newHeader); - } - - - ImmutableArray debugDirectoryEntries = peReader.ReadDebugDirectory(); - - peInfo = new PEFileInfo(SectionHeaders: sections, - DebugTableDirectory: peHeader.DebugTableDirectory, - SectionStart: firstPESection, - DebugDirectoryEntries: debugDirectoryEntries); - - wcInfo = new WCFileInfo(Header: header, - SectionHeaders: headerBuilder.MoveToImmutable(), - SectionStart: firstWCSection); - } - - private static void WriteHeader(Stream s, WCHeader header) - { - WriteStructure(s, header); - } - - private static void WriteSectionHeaders(Stream s, ImmutableArray sectionsHeaders) - { - // FIXME: fixup endianness - if (!BitConverter.IsLittleEndian) - throw new NotImplementedException(); - foreach (var sectionHeader in sectionsHeaders) - { - WriteSectionHeader(s, sectionHeader); - } - } - - private static void WriteSectionHeader(Stream s, WebcilSectionHeader sectionHeader) - { - WriteStructure(s, sectionHeader); + _converter.ConvertToWebcil(); } -#if NETCOREAPP2_1_OR_GREATER - private static void WriteStructure(Stream s, T structure) - where T : unmanaged - { - // FIXME: fixup endianness - if (!BitConverter.IsLittleEndian) - throw new NotImplementedException(); - unsafe - { - byte* p = (byte*)&structure; - s.Write(new ReadOnlySpan(p, sizeof(T))); - } - } -#else - private static void WriteStructure(Stream s, T structure) - where T : unmanaged - { - // FIXME: fixup endianness - if (!BitConverter.IsLittleEndian) - throw new NotImplementedException(); - int size = Marshal.SizeOf(); - byte[] buffer = new byte[size]; - IntPtr ptr = IntPtr.Zero; - try - { - ptr = Marshal.AllocHGlobal(size); - Marshal.StructureToPtr(structure, ptr, false); - Marshal.Copy(ptr, buffer, 0, size); - } - finally - { - Marshal.FreeHGlobal(ptr); - } - s.Write(buffer, 0, size); - } -#endif - - private static void CopySections(Stream outStream, Stream inputStream, ImmutableArray peSections) - { - // endianness: ok, we're just copying from one stream to another - foreach (var peHeader in peSections) - { - var buffer = new byte[peHeader.SizeOfRawData]; - inputStream.Seek(peHeader.PointerToRawData, SeekOrigin.Begin); - ReadExactly(inputStream, buffer); - outStream.Write(buffer, 0, buffer.Length); - } - } - -#if NETCOREAPP2_1_OR_GREATER - private static void ReadExactly(Stream s, Span buffer) - { - s.ReadExactly(buffer); - } -#else - private static void ReadExactly(Stream s, byte[] buffer) - { - int offset = 0; - while (offset < buffer.Length) - { - int read = s.Read(buffer, offset, buffer.Length - offset); - if (read == 0) - throw new EndOfStreamException(); - offset += read; - } - } -#endif - - private static FilePosition GetPositionOfRelativeVirtualAddress(ImmutableArray wcSections, uint relativeVirtualAddress) - { - foreach (var section in wcSections) - { - if (relativeVirtualAddress >= section.VirtualAddress && relativeVirtualAddress < section.VirtualAddress + section.VirtualSize) - { - FilePosition pos = section.PointerToRawData + ((int)relativeVirtualAddress - section.VirtualAddress); - return pos; - } - } - - throw new InvalidOperationException("relative virtual address not in any section"); - } - - // Given a physical file offset, return the section and the offset within the section. - private static (WebcilSectionHeader section, int offset) GetSectionFromFileOffset(ImmutableArray peSections, FilePosition fileOffset) - { - foreach (var section in peSections) - { - if (fileOffset.Position >= section.PointerToRawData && fileOffset.Position < section.PointerToRawData + section.SizeOfRawData) - { - return (section, fileOffset.Position - section.PointerToRawData); - } - } - - throw new InvalidOperationException("file offset not in any section (Webcil)"); - } - - private static void GetSectionFromFileOffset(ImmutableArray sections, FilePosition fileOffset) - { - foreach (var section in sections) - { - if (fileOffset.Position >= section.PointerToRawData && fileOffset.Position < section.PointerToRawData + section.SizeOfRawData) - { - return; - } - } - - throw new InvalidOperationException($"file offset {fileOffset.Position} not in any section (PE)"); - } - - // Make a new set of debug directory entries that - // have their data pointers adjusted to be relative to the start of the webcil file. - // This is necessary because the debug directory entires in the PE file are relative to the start of the PE file, - // and a PE header is bigger than a webcil header. - private static ImmutableArray FixupDebugDirectoryEntries(PEFileInfo peInfo, WCFileInfo wcInfo) - { - int dataPointerAdjustment = peInfo.SectionStart.Position - wcInfo.SectionStart.Position; - ImmutableArray entries = peInfo.DebugDirectoryEntries; - ImmutableArray.Builder newEntries = ImmutableArray.CreateBuilder(entries.Length); - foreach (var entry in entries) - { - DebugDirectoryEntry newEntry; - if (entry.Type == DebugDirectoryEntryType.Reproducible || entry.DataPointer == 0 || entry.DataSize == 0) - { - // this entry doesn't have an associated data pointer, so just copy it - newEntry = entry; - } - else - { - // the "pointer" field is a file offset, find the corresponding entry in the Webcil file and overwrite with the correct file position - var newDataPointer = entry.DataPointer - dataPointerAdjustment; - newEntry = new DebugDirectoryEntry(entry.Stamp, entry.MajorVersion, entry.MinorVersion, entry.Type, entry.DataSize, entry.DataRelativeVirtualAddress, newDataPointer); - GetSectionFromFileOffset(peInfo.SectionHeaders, entry.DataPointer); - // validate that the new entry is in some section - GetSectionFromFileOffset(wcInfo.SectionHeaders, newDataPointer); - } - newEntries.Add(newEntry); - } - return newEntries.MoveToImmutable(); - } - - private static void OverwriteDebugDirectoryEntries(Stream s, WCFileInfo wcInfo, ImmutableArray entries) - { - FilePosition debugDirectoryPos = GetPositionOfRelativeVirtualAddress(wcInfo.SectionHeaders, wcInfo.Header.pe_debug_rva); - using var writer = new BinaryWriter(s, System.Text.Encoding.UTF8, leaveOpen: true); - writer.Seek(debugDirectoryPos.Position, SeekOrigin.Begin); - foreach (var entry in entries) - { - WriteDebugDirectoryEntry(writer, entry); - } - // TODO check that we overwrite with the same size as the original - - // restore the stream position - writer.Seek(0, SeekOrigin.End); - } - - private static void WriteDebugDirectoryEntry(BinaryWriter writer, DebugDirectoryEntry entry) - { - writer.Write((uint)0); // Characteristics - writer.Write(entry.Stamp); - writer.Write(entry.MajorVersion); - writer.Write(entry.MinorVersion); - writer.Write((uint)entry.Type); - writer.Write(entry.DataSize); - writer.Write(entry.DataRelativeVirtualAddress); - writer.Write(entry.DataPointer); - } } From d507f64faea1148b0a3733031bb9fd955f1e9b26 Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Fri, 6 Jan 2023 14:34:52 -0500 Subject: [PATCH 33/57] Move WebcilReader to Microsoft.NET.WebAssembly.Webcil delete the duplicated utility classes --- .../src}/Webcil/WebcilReader.cs | 59 +++++++++++-------- .../debugger/BrowserDebugProxy/DebugStore.cs | 5 +- .../Webcil/CoffSectionHeaderBuilder.cs | 27 --------- .../BrowserDebugProxy/Webcil/Constants.cs | 9 --- .../BrowserDebugProxy/Webcil/WCHeader.cs | 32 ---------- 5 files changed, 38 insertions(+), 94 deletions(-) rename src/{mono/wasm/debugger/BrowserDebugProxy => libraries/Microsoft.NET.WebAssembly.Webcil/src}/Webcil/WebcilReader.cs (86%) delete mode 100644 src/mono/wasm/debugger/BrowserDebugProxy/Webcil/CoffSectionHeaderBuilder.cs delete mode 100644 src/mono/wasm/debugger/BrowserDebugProxy/Webcil/Constants.cs delete mode 100644 src/mono/wasm/debugger/BrowserDebugProxy/Webcil/WCHeader.cs diff --git a/src/mono/wasm/debugger/BrowserDebugProxy/Webcil/WebcilReader.cs b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilReader.cs similarity index 86% rename from src/mono/wasm/debugger/BrowserDebugProxy/Webcil/WebcilReader.cs rename to src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilReader.cs index 6ae3c41bedbde..05f05cf4f93d8 100644 --- a/src/mono/wasm/debugger/BrowserDebugProxy/Webcil/WebcilReader.cs +++ b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilReader.cs @@ -10,7 +10,7 @@ using System.Reflection.Metadata; using System.Reflection.PortableExecutable; -namespace Microsoft.WebAssembly.Diagnostics.Webcil; +namespace Microsoft.NET.WebAssembly.Webcil; public sealed class WebcilReader : IDisposable @@ -20,10 +20,10 @@ public sealed class WebcilReader : IDisposable // but the memory block classes are internal to S.R.M. private readonly Stream _stream; - private WCHeader? _header; - private DirectoryEntry? _corHeaderMetadataDirectory; - private MetadataReaderProvider _metadataReaderProvider; - private ImmutableArray? _sections; + private WebcilHeader _header; + private DirectoryEntry _corHeaderMetadataDirectory; + private MetadataReaderProvider? _metadataReaderProvider; + private ImmutableArray? _sections; public WebcilReader(Stream stream) { @@ -44,17 +44,17 @@ public WebcilReader(Stream stream) private unsafe bool ReadHeader() { - WCHeader header; - var buffer = new byte[Marshal.SizeOf()]; + WebcilHeader header; + var buffer = new byte[Marshal.SizeOf()]; if (_stream.Read(buffer, 0, buffer.Length) != buffer.Length) { return false; } fixed (byte* p = buffer) { - header = *(WCHeader*)p; + header = *(WebcilHeader*)p; } - if (header.id[0] != 'W' || header.id[1] != 'C' || header.version != Constants.WC_VERSION) + if (header.id[0] != 'W' || header.id[1] != 'C' || header.version != Internal.Constants.WC_VERSION) { return false; } @@ -70,7 +70,7 @@ private unsafe bool ReadCorHeader() { // we can't construct CorHeader because it's constructor is internal // but we don't care, really, we only want the metadata directory entry - var pos = TranslateRVA(_header.Value.pe_cli_header_rva); + var pos = TranslateRVA(_header.pe_cli_header_rva); if (_stream.Seek(pos, SeekOrigin.Begin) != pos) { return false; @@ -88,7 +88,7 @@ public MetadataReaderProvider GetMetadataReaderProvider() // FIXME threading if (_metadataReaderProvider == null) { - long pos = TranslateRVA((uint)_corHeaderMetadataDirectory.Value.RelativeVirtualAddress); + long pos = TranslateRVA((uint)_corHeaderMetadataDirectory.RelativeVirtualAddress); if (_stream.Seek(pos, SeekOrigin.Begin) != pos) { throw new BadImageFormatException("Could not seek to metadata", nameof(_stream)); @@ -102,12 +102,12 @@ public MetadataReaderProvider GetMetadataReaderProvider() public ImmutableArray ReadDebugDirectory() { - var debugRVA = _header.Value.pe_debug_rva; + var debugRVA = _header.pe_debug_rva; if (debugRVA == 0) { return ImmutableArray.Empty; } - var debugSize = _header.Value.pe_debug_size; + var debugSize = _header.pe_debug_size; if (debugSize == 0) { return ImmutableArray.Empty; @@ -204,22 +204,29 @@ private static CodeViewDebugDirectoryData DecodeCodeViewDebugDirectoryData(BlobR Guid guid = reader.ReadGuid(); int age = reader.ReadInt32(); - string path = ReadUtf8NullTerminated(reader); + string path = ReadUtf8NullTerminated(reader)!; return MakeCodeViewDebugDirectoryData(guid, age, path); - } - private static string ReadUtf8NullTerminated(BlobReader reader) + private static string? ReadUtf8NullTerminated(BlobReader reader) { var mi = typeof(BlobReader).GetMethod("ReadUtf8NullTerminated", BindingFlags.NonPublic | BindingFlags.Instance); - return (string)mi.Invoke(reader, null); + if (mi == null) + { + throw new InvalidOperationException("Could not find BlobReader.ReadUtf8NullTerminated"); + } + return (string?)mi.Invoke(reader, null); } private static CodeViewDebugDirectoryData MakeCodeViewDebugDirectoryData(Guid guid, int age, string path) { var types = new Type[] { typeof(Guid), typeof(int), typeof(string) }; - var mi = typeof(CodeViewDebugDirectoryData).GetConstructor(BindingFlags.NonPublic | BindingFlags.Instance, types); + var mi = typeof(CodeViewDebugDirectoryData).GetConstructor(BindingFlags.NonPublic | BindingFlags.Instance, null, types, null); + if (mi == null) + { + throw new InvalidOperationException("Could not find CodeViewDebugDirectoryData constructor"); + } return (CodeViewDebugDirectoryData)mi.Invoke(new object[] { guid, age, path }); } @@ -264,7 +271,11 @@ private static MetadataReaderProvider DecodeEmbeddedPortablePdbDirectoryData(Blo using (var compressedStream = new MemoryStream(compressedBuffer, writable: false)) using (var deflateStream = new System.IO.Compression.DeflateStream(compressedStream, System.IO.Compression.CompressionMode.Decompress, leaveOpen: true)) { +#if NETCOREAPP1_1_OR_GREATER decompressedBuffer = GC.AllocateUninitializedArray(decompressedSize); +#else + decompressedBuffer = new byte[decompressedSize]; +#endif using (var decompressedStream = new MemoryStream(decompressedBuffer, writable: true)) { deflateStream.CopyTo(decompressedStream); @@ -292,14 +303,14 @@ private long TranslateRVA(uint rva) throw new BadImageFormatException("RVA not found in any section", nameof(_stream)); } - private static long SectionDirectoryOffset => Marshal.SizeOf(); + private static long SectionDirectoryOffset => Marshal.SizeOf(); - private unsafe ImmutableArray ReadSections() + private unsafe ImmutableArray ReadSections() { - var sections = ImmutableArray.CreateBuilder(_header.Value.coff_sections); - var buffer = new byte[Marshal.SizeOf()]; + var sections = ImmutableArray.CreateBuilder(_header.coff_sections); + var buffer = new byte[Marshal.SizeOf()]; _stream.Seek(SectionDirectoryOffset, SeekOrigin.Begin); - for (int i = 0; i < _header.Value.coff_sections; i++) + for (int i = 0; i < _header.coff_sections; i++) { if (_stream.Read(buffer, 0, buffer.Length) != buffer.Length) { @@ -308,7 +319,7 @@ private unsafe ImmutableArray ReadSections() fixed (byte* p = buffer) { // FIXME endianness - sections.Add(*(CoffSectionHeaderBuilder*)p); + sections.Add(*(WebcilSectionHeader*)p); } } return sections.MoveToImmutable(); diff --git a/src/mono/wasm/debugger/BrowserDebugProxy/DebugStore.cs b/src/mono/wasm/debugger/BrowserDebugProxy/DebugStore.cs index 20a93e5111d5b..d053921f05415 100644 --- a/src/mono/wasm/debugger/BrowserDebugProxy/DebugStore.cs +++ b/src/mono/wasm/debugger/BrowserDebugProxy/DebugStore.cs @@ -21,6 +21,7 @@ using System.Reflection; using System.Diagnostics; using System.Text; +using Microsoft.NET.WebAssembly.Webcil; namespace Microsoft.WebAssembly.Diagnostics { @@ -882,7 +883,7 @@ public static AssemblyInfo FromBytes(MonoProxy monoProxy, SessionId sessionId, b { // This is a WebAssembly file asmStream.Seek(0, SeekOrigin.Begin); - var webcilReader = new Webcil.WebcilReader(asmStream); + var webcilReader = new WebcilReader(asmStream); return FromWebcilReader(monoProxy, sessionId, webcilReader, pdb, logger, token); } } @@ -941,7 +942,7 @@ private static AssemblyInfo FromPEReader(MonoProxy monoProxy, SessionId sessionI return assemblyInfo; } - private static AssemblyInfo FromWebcilReader(MonoProxy monoProxy, SessionId sessionId, Webcil.WebcilReader wcReader, byte[] pdb, ILogger logger, CancellationToken token) + private static AssemblyInfo FromWebcilReader(MonoProxy monoProxy, SessionId sessionId, WebcilReader wcReader, byte[] pdb, ILogger logger, CancellationToken token) { var entries = wcReader.ReadDebugDirectory(); CodeViewDebugDirectoryData? codeViewData = null; diff --git a/src/mono/wasm/debugger/BrowserDebugProxy/Webcil/CoffSectionHeaderBuilder.cs b/src/mono/wasm/debugger/BrowserDebugProxy/Webcil/CoffSectionHeaderBuilder.cs deleted file mode 100644 index d8e428bea6b6d..0000000000000 --- a/src/mono/wasm/debugger/BrowserDebugProxy/Webcil/CoffSectionHeaderBuilder.cs +++ /dev/null @@ -1,27 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.InteropServices; - -namespace Microsoft.WebAssembly.Diagnostics.Webcil; - -/// -/// This is System.Reflection.PortableExecutable.CoffHeader, but with a public constructor so that -/// we can make our own copies of it, and with fewer fields -/// -[StructLayout(LayoutKind.Sequential, Pack = 1)] -public struct CoffSectionHeaderBuilder -{ - public readonly int VirtualSize; - public readonly int VirtualAddress; - public readonly int SizeOfRawData; - public readonly int PointerToRawData; - - public CoffSectionHeaderBuilder(int virtualSize, int virtualAddress, int sizeOfRawData, int pointerToRawData) - { - VirtualSize = virtualSize; - VirtualAddress = virtualAddress; - SizeOfRawData = sizeOfRawData; - PointerToRawData = pointerToRawData; - } -} diff --git a/src/mono/wasm/debugger/BrowserDebugProxy/Webcil/Constants.cs b/src/mono/wasm/debugger/BrowserDebugProxy/Webcil/Constants.cs deleted file mode 100644 index b7634199538f7..0000000000000 --- a/src/mono/wasm/debugger/BrowserDebugProxy/Webcil/Constants.cs +++ /dev/null @@ -1,9 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -namespace Microsoft.WebAssembly.Diagnostics.Webcil; - -public static unsafe class Constants -{ - public const int WC_VERSION = 0; -} diff --git a/src/mono/wasm/debugger/BrowserDebugProxy/Webcil/WCHeader.cs b/src/mono/wasm/debugger/BrowserDebugProxy/Webcil/WCHeader.cs deleted file mode 100644 index 905407f7b0165..0000000000000 --- a/src/mono/wasm/debugger/BrowserDebugProxy/Webcil/WCHeader.cs +++ /dev/null @@ -1,32 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.InteropServices; - -namespace Microsoft.WebAssembly.Diagnostics.Webcil; - -/// -/// The header of a WebCIL file. -/// -/// -/// -/// The header is a subset of the PE, COFF and CLI headers that are needed by the mono runtime to load managed assemblies. -/// -[StructLayout(LayoutKind.Sequential, Pack = 1)] -public unsafe struct WCHeader -{ - public fixed byte id[2]; // 'W' 'C' - public byte version; - public byte reserved0; // 0 - // 4 bytes - - public ushort coff_sections; - public ushort reserved1; // 0 - // 8 bytes - public uint pe_cli_header_rva; - public uint pe_cli_header_size; - // 16 bytes - public uint pe_debug_rva; - public uint pe_debug_size; - // 24 bytes -} From 6c6cae5b8e58a7b02bf4d45f3591fabe598bd46f Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Fri, 6 Jan 2023 15:57:29 -0500 Subject: [PATCH 34/57] make the webcil magic and version longer --- docs/design/mono/webcil.md | 21 +++++++++--------- .../src/Webcil/Internal/Constants.cs | 3 ++- .../src/Webcil/WebciHeader.cs | 15 +++++++------ .../src/Webcil/WebcilConverter.cs | 10 +++++---- .../src/Webcil/WebcilReader.cs | 13 ++++++----- src/mono/mono/metadata/reflection.c | 2 +- src/mono/mono/metadata/webcil-loader.c | 22 ++++++++++--------- 7 files changed, 48 insertions(+), 38 deletions(-) diff --git a/docs/design/mono/webcil.md b/docs/design/mono/webcil.md index fe6bf8f4c52f2..3bcb7d6365353 100644 --- a/docs/design/mono/webcil.md +++ b/docs/design/mono/webcil.md @@ -2,7 +2,7 @@ ## Version -This is version 0 of the Webcil format. +This is version 0.0 of the Webcil format. ## Motivation @@ -49,26 +49,27 @@ The Webcil headers consist of a Webcil header followed by a sequence of section ``` c struct WebcilHeader { - uint8_t id[2]; // 'W' 'C' - uint8_t version; - uint8_t reserved0; // 0 + uint8_t id[4]; // 'W' 'b' 'I' 'L' // 4 bytes - uint16_t coff_sections; - uint16_t reserved1; // 0 + uint16_t version_major; // 0 + uint16_t version_minor; // 0 // 8 bytes + uint16_t coff_sections; + uint16_t reserved0; // 0 + // 12 bytes uint32_t pe_cli_header_rva; uint32_t pe_cli_header_size; - // 16 bytes + // 20 bytes uint32_t pe_debug_rva; uint32_t pe_debug_size; - // 24 bytes + // 28 bytes }; ``` -The Webcil header starts with the magic characters 'W' 'C' followed by the version (must be 0). -Then a reserved byte that must be 0 followed by a count of the section headers and 2 more reserved bytes. +The Webcil header starts with the magic characters 'W' 'b' 'I' 'L' followed by the version in major +minor format (must be 0 and 0). Then a count of the section headers and two reserved bytes. The next pairs of integers are a subset of the PE Header data directory specifying the RVA and size of the CLI header, as well as the directory entry for the PE debug directory. diff --git a/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/Internal/Constants.cs b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/Internal/Constants.cs index 2324dbdf633f9..2d486645d23b6 100644 --- a/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/Internal/Constants.cs +++ b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/Internal/Constants.cs @@ -5,5 +5,6 @@ namespace Microsoft.NET.WebAssembly.Webcil.Internal; internal static unsafe class Constants { - public const int WC_VERSION = 0; + public const int WC_VERSION_MAJOR = 0; + public const int WC_VERSION_MINOR = 0; } diff --git a/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebciHeader.cs b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebciHeader.cs index 8c326b7f7b6ce..33dcc85791fff 100644 --- a/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebciHeader.cs +++ b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebciHeader.cs @@ -15,18 +15,19 @@ namespace Microsoft.NET.WebAssembly.Webcil; [StructLayout(LayoutKind.Sequential, Pack = 1)] public unsafe struct WebcilHeader { - public fixed byte id[2]; // 'W' 'C' - public byte version; - public byte reserved0; // 0 + public fixed byte id[4]; // 'W' 'b' 'I' 'L' // 4 bytes + public ushort version_major; // 0 + public ushort version_minor; // 0 + // 8 bytes public ushort coff_sections; - public ushort reserved1; // 0 - // 8 bytes + public ushort reserved0; // 0 + // 12 bytes public uint pe_cli_header_rva; public uint pe_cli_header_size; - // 16 bytes + // 20 bytes public uint pe_debug_rva; public uint pe_debug_size; - // 24 bytes + // 28 bytes } diff --git a/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilConverter.cs b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilConverter.cs index 44bf4a0aad6b2..73bdfe52c841e 100644 --- a/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilConverter.cs +++ b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilConverter.cs @@ -92,11 +92,13 @@ public unsafe void GatherInfo(PEReader peReader, out WCFileInfo wcInfo, out PEFi var sections = headers.SectionHeaders; WebcilHeader header; header.id[0] = (byte)'W'; - header.id[1] = (byte)'C'; - header.version = Internal.Constants.WC_VERSION; - header.reserved0 = 0; + header.id[1] = (byte)'b'; + header.id[2] = (byte)'I'; + header.id[3] = (byte)'L'; + header.version_major = Internal.Constants.WC_VERSION_MAJOR; + header.version_minor = Internal.Constants.WC_VERSION_MINOR; header.coff_sections = (ushort)coffHeader.NumberOfSections; - header.reserved1 = 0; + header.reserved0 = 0; header.pe_cli_header_rva = (uint)peHeader.CorHeaderTableDirectory.RelativeVirtualAddress; header.pe_cli_header_size = (uint)peHeader.CorHeaderTableDirectory.Size; header.pe_debug_rva = (uint)peHeader.DebugTableDirectory.RelativeVirtualAddress; diff --git a/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilReader.cs b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilReader.cs index 05f05cf4f93d8..d734d9c2e72e6 100644 --- a/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilReader.cs +++ b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilReader.cs @@ -50,19 +50,22 @@ private unsafe bool ReadHeader() { return false; } + if (!BitConverter.IsLittleEndian) + { + throw new NotImplementedException("TODO: implement big endian support"); + } fixed (byte* p = buffer) { header = *(WebcilHeader*)p; } - if (header.id[0] != 'W' || header.id[1] != 'C' || header.version != Internal.Constants.WC_VERSION) + if (header.id[0] != 'W' || header.id[1] != 'b' + || header.id[1] != 'I' || header.id[1] != 'L' + || header.version_major != Internal.Constants.WC_VERSION_MAJOR + || header.version_minor != Internal.Constants.WC_VERSION_MINOR) { return false; } _header = header; - if (!BitConverter.IsLittleEndian) - { - throw new NotImplementedException("TODO: implement big endian support"); - } return true; } diff --git a/src/mono/mono/metadata/reflection.c b/src/mono/mono/metadata/reflection.c index bd437db42a161..37179bf641a3e 100644 --- a/src/mono/mono/metadata/reflection.c +++ b/src/mono/mono/metadata/reflection.c @@ -1285,7 +1285,7 @@ method_body_object_construct (MonoClass *unused_class, MonoMethod *method, gpoin if ((method->flags & METHOD_ATTRIBUTE_PINVOKE_IMPL) || (method->flags & METHOD_ATTRIBUTE_ABSTRACT) || (method->iflags & METHOD_IMPL_ATTRIBUTE_INTERNAL_CALL) || - (image->raw_data && (image->raw_data [1] != 'Z' && image->raw_data [1] != 'C')) || + (image->raw_data && (image->raw_data [1] != 'Z' && image->raw_data [1] != 'b')) || (method->iflags & METHOD_IMPL_ATTRIBUTE_RUNTIME)) return MONO_HANDLE_CAST (MonoReflectionMethodBody, NULL_HANDLE); diff --git a/src/mono/mono/metadata/webcil-loader.c b/src/mono/mono/metadata/webcil-loader.c index f325880319cf0..6ed12280a31a8 100644 --- a/src/mono/mono/metadata/webcil-loader.c +++ b/src/mono/mono/metadata/webcil-loader.c @@ -11,32 +11,34 @@ /* keep in sync with webcil-writer */ enum { - MONO_WEBCIL_VERSION = 0, + MONO_WEBCIL_VERSION_MAJOR = 0, + MONO_WEBCIL_VERSION_MINOR = 0, }; typedef struct MonoWebCilHeader { - uint8_t id[2]; // 'W' 'C' - uint8_t version; - uint8_t reserved0; // 0 + uint8_t id[4]; // 'W' 'b' 'I' 'L' // 4 bytes - uint16_t coff_sections; - uint16_t reserved1; // 0 + uint16_t version_major; // 0 + uint16_t version_minor; // 0 // 8 bytes + uint16_t coff_sections; + uint16_t reserved0; // 0 + // 12 bytes uint32_t pe_cli_header_rva; uint32_t pe_cli_header_size; - // 16 bytes + // 20 bytes uint32_t pe_debug_rva; uint32_t pe_debug_size; - // 24 bytes + // 28 bytes } MonoWebCilHeader; static gboolean webcil_image_match (MonoImage *image) { if (image->raw_data_len >= sizeof (MonoWebCilHeader)) { - return image->raw_data[0] == 'W' && image->raw_data[1] == 'C'; + return image->raw_data[0] == 'W' && image->raw_data[1] == 'b' && image->raw_data[2] == 'I' && image->raw_data[3] == 'L'; } return FALSE; } @@ -54,7 +56,7 @@ do_load_header (const char *raw_data, uint32_t raw_data_len, int32_t offset, Mon return -1; memcpy (&wcheader, raw_data + offset, sizeof (wcheader)); - if (!(wcheader.id [0] == 'W' && wcheader.id [1] == 'C' && wcheader.version == MONO_WEBCIL_VERSION)) + if (!(wcheader.id [0] == 'W' && wcheader.id [1] == 'b' && wcheader.id[2] == 'I' && wcheader.id[3] == 'L' && wcheader.version_major == MONO_WEBCIL_VERSION_MAJOR && wcheader.version_minor == MONO_WEBCIL_VERSION_MINOR)) return -1; memset (header, 0, sizeof(MonoDotNetHeader)); From d9a4d25bd922df1e838428aa0d1c5744929c37d5 Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Fri, 6 Jan 2023 22:01:12 -0500 Subject: [PATCH 35/57] fixups --- .../src/Webcil/WebcilReader.cs | 2 +- src/mono/mono/metadata/webcil-loader.c | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilReader.cs b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilReader.cs index d734d9c2e72e6..41c63560f7bf0 100644 --- a/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilReader.cs +++ b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilReader.cs @@ -59,7 +59,7 @@ private unsafe bool ReadHeader() header = *(WebcilHeader*)p; } if (header.id[0] != 'W' || header.id[1] != 'b' - || header.id[1] != 'I' || header.id[1] != 'L' + || header.id[2] != 'I' || header.id[3] != 'L' || header.version_major != Internal.Constants.WC_VERSION_MAJOR || header.version_minor != Internal.Constants.WC_VERSION_MINOR) { diff --git a/src/mono/mono/metadata/webcil-loader.c b/src/mono/mono/metadata/webcil-loader.c index 6ed12280a31a8..1323c0f3ff137 100644 --- a/src/mono/mono/metadata/webcil-loader.c +++ b/src/mono/mono/metadata/webcil-loader.c @@ -56,7 +56,8 @@ do_load_header (const char *raw_data, uint32_t raw_data_len, int32_t offset, Mon return -1; memcpy (&wcheader, raw_data + offset, sizeof (wcheader)); - if (!(wcheader.id [0] == 'W' && wcheader.id [1] == 'b' && wcheader.id[2] == 'I' && wcheader.id[3] == 'L' && wcheader.version_major == MONO_WEBCIL_VERSION_MAJOR && wcheader.version_minor == MONO_WEBCIL_VERSION_MINOR)) + if (!(wcheader.id [0] == 'W' && wcheader.id [1] == 'b' && wcheader.id[2] == 'I' && wcheader.id[3] == 'L' && + GUINT16_FROM_LE (wcheader.version_major) == MONO_WEBCIL_VERSION_MAJOR && GUINT16_FROM_LE (wcheader.version_minor) == MONO_WEBCIL_VERSION_MINOR)) return -1; memset (header, 0, sizeof(MonoDotNetHeader)); From ac0404bacefe839d201a924f53ce725aa1edb20b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleksey=20Kliger=20=28=CE=BBgeek=29?= Date: Fri, 6 Jan 2023 22:15:50 -0500 Subject: [PATCH 36/57] Code style improvements from review Co-authored-by: Ankit Jain --- .../src/Microsoft.NET.WebAssembly.Webcil.csproj | 4 ---- .../src/Webcil/WebcilConverter.cs | 5 +---- src/mono/mono/metadata/image.c | 1 - src/mono/mono/metadata/mono-debug.c | 2 +- src/mono/mono/metadata/reflection.c | 2 +- 5 files changed, 3 insertions(+), 11 deletions(-) diff --git a/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Microsoft.NET.WebAssembly.Webcil.csproj b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Microsoft.NET.WebAssembly.Webcil.csproj index 6da8aac28b686..6c66737e5535d 100644 --- a/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Microsoft.NET.WebAssembly.Webcil.csproj +++ b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Microsoft.NET.WebAssembly.Webcil.csproj @@ -21,8 +21,4 @@ - - - - diff --git a/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilConverter.cs b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilConverter.cs index 73bdfe52c841e..07dcdf1a816f8 100644 --- a/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilConverter.cs +++ b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilConverter.cs @@ -47,9 +47,7 @@ private WebcilConverter(string inputPath, string outputPath) } public static WebcilConverter FromPortableExecutable(string inputPath, string outputPath) - { - return new WebcilConverter(inputPath, outputPath); - } + => new WebcilConverter(inputPath, outputPath); public void ConvertToWebcil() { @@ -140,7 +138,6 @@ public unsafe void GatherInfo(PEReader peReader, out WCFileInfo wcInfo, out PEFi headerBuilder.Add(newHeader); } - ImmutableArray debugDirectoryEntries = peReader.ReadDebugDirectory(); peInfo = new PEFileInfo(SectionHeaders: sections, diff --git a/src/mono/mono/metadata/image.c b/src/mono/mono/metadata/image.c index 6c85f26f7c8fd..fa5c3b09758ac 100644 --- a/src/mono/mono/metadata/image.c +++ b/src/mono/mono/metadata/image.c @@ -948,7 +948,6 @@ try_load_pe_cli_header (char *raw_data, uint32_t raw_data_len, MonoDotNetHeader return ret; } - mono_bool mono_has_pdb_checksum (char *raw_data, uint32_t raw_data_len) { diff --git a/src/mono/mono/metadata/mono-debug.c b/src/mono/mono/metadata/mono-debug.c index 95e486ac93260..9e6f97c235599 100644 --- a/src/mono/mono/metadata/mono-debug.c +++ b/src/mono/mono/metadata/mono-debug.c @@ -1108,7 +1108,7 @@ bsymfile_match (BundledSymfile *bsymfile, const char *assembly_name) if (p && *(p + 7) == 0) { size_t n = p - assembly_name; if (!strncmp (bsymfile->aname, assembly_name, n) - && !strcmp (bsymfile->aname + n, ".dll")) + && !strcmp (bsymfile->aname + n, ".dll")) return TRUE; } #endif diff --git a/src/mono/mono/metadata/reflection.c b/src/mono/mono/metadata/reflection.c index 37179bf641a3e..ad4d9795b6877 100644 --- a/src/mono/mono/metadata/reflection.c +++ b/src/mono/mono/metadata/reflection.c @@ -1285,7 +1285,7 @@ method_body_object_construct (MonoClass *unused_class, MonoMethod *method, gpoin if ((method->flags & METHOD_ATTRIBUTE_PINVOKE_IMPL) || (method->flags & METHOD_ATTRIBUTE_ABSTRACT) || (method->iflags & METHOD_IMPL_ATTRIBUTE_INTERNAL_CALL) || - (image->raw_data && (image->raw_data [1] != 'Z' && image->raw_data [1] != 'b')) || + (image->raw_data && (image->raw_data [1] != 'Z' && image->raw_data [1] != 'b')) || (method->iflags & METHOD_IMPL_ATTRIBUTE_RUNTIME)) return MONO_HANDLE_CAST (MonoReflectionMethodBody, NULL_HANDLE); From 0cedaccd3b55ca818b1b81626012c86f9d790b59 Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Sat, 7 Jan 2023 10:13:16 -0500 Subject: [PATCH 37/57] Improve some exception messages, when possible --- .../src/Webcil/WebcilConverter.cs | 14 ++++++++------ .../src/Webcil/WebcilReader.cs | 11 +++++++++-- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilConverter.cs b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilConverter.cs index 07dcdf1a816f8..421b62439e190 100644 --- a/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilConverter.cs +++ b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilConverter.cs @@ -40,6 +40,8 @@ FilePosition SectionStart private readonly string _inputPath; private readonly string _outputPath; + private string InputPath => _inputPath; + private WebcilConverter(string inputPath, string outputPath) { _inputPath = inputPath; @@ -254,7 +256,7 @@ private static FilePosition GetPositionOfRelativeVirtualAddress(ImmutableArray peSections, FilePosition fileOffset) + private (WebcilSectionHeader section, int offset) GetSectionFromFileOffset(ImmutableArray peSections, FilePosition fileOffset) { foreach (var section in peSections) { @@ -264,10 +266,10 @@ private static (WebcilSectionHeader section, int offset) GetSectionFromFileOffse } } - throw new InvalidOperationException("file offset not in any section (Webcil)"); + throw new InvalidOperationException($"file offset not in any section (Webcil) for {InputPath}"); } - private static void GetSectionFromFileOffset(ImmutableArray sections, FilePosition fileOffset) + private void GetSectionFromFileOffset(ImmutableArray sections, FilePosition fileOffset) { foreach (var section in sections) { @@ -277,14 +279,14 @@ private static void GetSectionFromFileOffset(ImmutableArray secti } } - throw new InvalidOperationException($"file offset {fileOffset.Position} not in any section (PE)"); + throw new InvalidOperationException($"file offset {fileOffset.Position} not in any section (PE) for {InputPath}"); } // Make a new set of debug directory entries that // have their data pointers adjusted to be relative to the start of the webcil file. // This is necessary because the debug directory entires in the PE file are relative to the start of the PE file, // and a PE header is bigger than a webcil header. - private static ImmutableArray FixupDebugDirectoryEntries(PEFileInfo peInfo, WCFileInfo wcInfo) + private ImmutableArray FixupDebugDirectoryEntries(PEFileInfo peInfo, WCFileInfo wcInfo) { int dataPointerAdjustment = peInfo.SectionStart.Position - wcInfo.SectionStart.Position; ImmutableArray entries = peInfo.DebugDirectoryEntries; @@ -299,7 +301,7 @@ private static ImmutableArray FixupDebugDirectoryEntries(PE } else { - // the "pointer" field is a file offset, find the corresponding entry in the Webcil file and overwrite with the correct file position + // the "DataPointer" field is a file offset in the PE file, adjust the entry wit the corresponding offset in the Webcil file var newDataPointer = entry.DataPointer - dataPointerAdjustment; newEntry = new DebugDirectoryEntry(entry.Stamp, entry.MajorVersion, entry.MinorVersion, entry.Type, entry.DataSize, entry.DataRelativeVirtualAddress, newDataPointer); GetSectionFromFileOffset(peInfo.SectionHeaders, entry.DataPointer); diff --git a/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilReader.cs b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilReader.cs index 41c63560f7bf0..6782ebf4a5aae 100644 --- a/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilReader.cs +++ b/src/libraries/Microsoft.NET.WebAssembly.Webcil/src/Webcil/WebcilReader.cs @@ -25,6 +25,8 @@ public sealed class WebcilReader : IDisposable private MetadataReaderProvider? _metadataReaderProvider; private ImmutableArray? _sections; + private string? InputPath { get; } + public WebcilReader(Stream stream) { this._stream = stream; @@ -42,6 +44,11 @@ public WebcilReader(Stream stream) } } + public WebcilReader (Stream stream, string inputPath) : this(stream) + { + InputPath = inputPath; + } + private unsafe bool ReadHeader() { WebcilHeader header; @@ -94,7 +101,7 @@ public MetadataReaderProvider GetMetadataReaderProvider() long pos = TranslateRVA((uint)_corHeaderMetadataDirectory.RelativeVirtualAddress); if (_stream.Seek(pos, SeekOrigin.Begin) != pos) { - throw new BadImageFormatException("Could not seek to metadata", nameof(_stream)); + throw new BadImageFormatException("Could not seek to metadata in ", InputPath); } _metadataReaderProvider = MetadataReaderProvider.FromMetadataStream(_stream, MetadataStreamOptions.LeaveOpen); } @@ -120,7 +127,7 @@ public ImmutableArray ReadDebugDirectory() var buffer = new byte[debugSize]; if (_stream.Read(buffer, 0, buffer.Length) != buffer.Length) { - throw new BadImageFormatException("Could not read debug directory", nameof(_stream)); + throw new BadImageFormatException("Could not read debug directory", InputPath); } unsafe { From dadbf4ad6aa0c48ee96cd11865d6fc18c9b5396a Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Mon, 9 Jan 2023 13:39:51 -0500 Subject: [PATCH 38/57] Suggestings from code review --- .../Microsoft.NETCore.BrowserDebugHost.Transport.pkgproj | 1 + .../wasm/debugger/BrowserDebugProxy/BrowserDebugProxy.csproj | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/mono/nuget/Microsoft.NETCore.BrowserDebugHost.Transport/Microsoft.NETCore.BrowserDebugHost.Transport.pkgproj b/src/mono/nuget/Microsoft.NETCore.BrowserDebugHost.Transport/Microsoft.NETCore.BrowserDebugHost.Transport.pkgproj index 7a6b8326bcfb3..2119425f85f0c 100644 --- a/src/mono/nuget/Microsoft.NETCore.BrowserDebugHost.Transport/Microsoft.NETCore.BrowserDebugHost.Transport.pkgproj +++ b/src/mono/nuget/Microsoft.NETCore.BrowserDebugHost.Transport/Microsoft.NETCore.BrowserDebugHost.Transport.pkgproj @@ -14,6 +14,7 @@ <_browserDebugHostFiles Include="$(ArtifactsDir)bin\BrowserDebugHost\$(TargetArchitecture)\$(Configuration)\BrowserDebugHost.dll" /> <_browserDebugHostFiles Include="$(ArtifactsDir)bin\BrowserDebugHost\$(TargetArchitecture)\$(Configuration)\BrowserDebugHost.runtimeconfig.json" /> <_browserDebugHostFiles Include="$(ArtifactsDir)bin\BrowserDebugHost\$(TargetArchitecture)\$(Configuration)\BrowserDebugProxy.dll" /> + <_browserDebugHostFiles Include="$(ArtifactsDir)bin\BrowserDebugHost\$(TargetArchitecture)\$(Configuration)\Microsoft.NET.WebAssembly.Webcil.dll" /> <_browserDebugHostFiles Include="$(ArtifactsDir)bin\BrowserDebugHost\$(TargetArchitecture)\$(Configuration)\Microsoft.CodeAnalysis.CSharp.dll" /> <_browserDebugHostFiles Include="$(ArtifactsDir)bin\BrowserDebugHost\$(TargetArchitecture)\$(Configuration)\Microsoft.CodeAnalysis.dll" /> <_browserDebugHostFiles Include="$(ArtifactsDir)bin\BrowserDebugHost\$(TargetArchitecture)\$(Configuration)\Newtonsoft.Json.dll" /> diff --git a/src/mono/wasm/debugger/BrowserDebugProxy/BrowserDebugProxy.csproj b/src/mono/wasm/debugger/BrowserDebugProxy/BrowserDebugProxy.csproj index a4cc5817c76af..eb93b1a34e23f 100644 --- a/src/mono/wasm/debugger/BrowserDebugProxy/BrowserDebugProxy.csproj +++ b/src/mono/wasm/debugger/BrowserDebugProxy/BrowserDebugProxy.csproj @@ -15,7 +15,7 @@ - + From 73d57a4d1a36933b2b59d94d330bf6283572b8c3 Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Wed, 11 Jan 2023 10:13:33 -0500 Subject: [PATCH 39/57] Add WasmEnableWebcil msbuild property. Off by default --- src/mono/wasm/build/WasmApp.targets | 5 +++++ src/tasks/WasmAppBuilder/WasmAppBuilder.cs | 8 ++++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/mono/wasm/build/WasmApp.targets b/src/mono/wasm/build/WasmApp.targets index e248125876003..8d47e512d6ec5 100644 --- a/src/mono/wasm/build/WasmApp.targets +++ b/src/mono/wasm/build/WasmApp.targets @@ -67,6 +67,7 @@ - $(WasmAotProfilePath) - Path to an AOT profile file. - $(WasmEnableExceptionHandling) - Enable support for the WASM Exception Handling feature. - $(WasmEnableSIMD) - Enable support for the WASM SIMD feature. + - $(WasmEnableWebcil) - Enable conversion of assembly .dlls to .webcil Public items: - @(WasmExtraFilesToDeploy) - Files to copy to $(WasmAppDir). @@ -117,6 +118,9 @@ -1 + + + diff --git a/src/tasks/WasmAppBuilder/WasmAppBuilder.cs b/src/tasks/WasmAppBuilder/WasmAppBuilder.cs index c121e226ce48e..9d60875c8e05d 100644 --- a/src/tasks/WasmAppBuilder/WasmAppBuilder.cs +++ b/src/tasks/WasmAppBuilder/WasmAppBuilder.cs @@ -48,7 +48,7 @@ public class WasmAppBuilder : Task public string? MainHTMLPath { get; set; } public bool IncludeThreadsWorker {get; set; } public int PThreadPoolSize {get; set; } - public bool DisableWebcil { get; set; } + public bool UseWebcil { get; set; } // // Extra json elements to add to mono-config.json @@ -190,7 +190,7 @@ private bool ExecuteInternal () Directory.CreateDirectory(asmRootPath); foreach (var assembly in _assemblies) { - if (!DisableWebcil) + if (UseWebcil) { var tmpWebcil = Path.GetTempFileName(); var webcilWriter = Microsoft.WebAssembly.Build.Tasks.WebcilConverter.FromPortableExecutable(inputPath: assembly, outputPath: tmpWebcil, logger: Log); @@ -247,7 +247,7 @@ private bool ExecuteInternal () foreach (var assembly in _assemblies) { string assemblyPath = assembly; - if (!DisableWebcil) + if (UseWebcil) assemblyPath = Path.ChangeExtension(assemblyPath, ".webcil"); config.Assets.Add(new AssemblyEntry(Path.GetFileName(assemblyPath))); if (DebugLevel != 0) { @@ -276,7 +276,7 @@ private bool ExecuteInternal () string name = Path.GetFileName(fullPath); string directory = Path.Combine(AppDir, config.AssemblyRootFolder, culture); Directory.CreateDirectory(directory); - if (!DisableWebcil) + if (UseWebcil) { var tmpWebcil = Path.GetTempFileName(); var webcilWriter = Microsoft.WebAssembly.Build.Tasks.WebcilConverter.FromPortableExecutable(inputPath: fullPath, outputPath: tmpWebcil, logger: Log); From f0f63407006ea7ed534f6a2be9a8b021831740ef Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Wed, 11 Jan 2023 10:18:30 -0500 Subject: [PATCH 40/57] Build non-wasm runtimes without .webcil support --- src/mono/cmake/config.h.in | 4 ++-- src/mono/cmake/options.cmake | 2 +- src/mono/mono.proj | 1 + src/mono/mono/metadata/assembly.c | 4 ++-- src/mono/mono/metadata/image.c | 8 ++++---- src/mono/mono/metadata/mono-debug.c | 2 +- src/mono/mono/mini/monovm.c | 2 +- src/mono/wasm/build/WasmApp.targets | 2 +- 8 files changed, 13 insertions(+), 12 deletions(-) diff --git a/src/mono/cmake/config.h.in b/src/mono/cmake/config.h.in index 069b4ba5d9400..d0b5d79ff2097 100644 --- a/src/mono/cmake/config.h.in +++ b/src/mono/cmake/config.h.in @@ -933,8 +933,8 @@ /* Enable System.WeakAttribute support */ #cmakedefine ENABLE_WEAK_ATTR 1 -/* Disable WebCIL image loader */ -#cmakedefine DISABLE_WEBCIL 1 +/* Enable WebCIL image loader */ +#cmakedefine ENABLE_WEBCIL 1 #if defined(ENABLE_LLVM) && defined(HOST_WIN32) && defined(TARGET_WIN32) && (!defined(TARGET_AMD64) || !defined(_MSC_VER)) #error LLVM for host=Windows and target=Windows is only supported on x64 MSVC build. diff --git a/src/mono/cmake/options.cmake b/src/mono/cmake/options.cmake index b7ca5648e2d4d..202a578db4fc4 100644 --- a/src/mono/cmake/options.cmake +++ b/src/mono/cmake/options.cmake @@ -57,7 +57,7 @@ option (ENABLE_SIGALTSTACK "Enable support for using sigaltstack for SIGSEGV and option (USE_MALLOC_FOR_MEMPOOLS "Use malloc for each single mempool allocation, so tools like Valgrind can run better") option (STATIC_COMPONENTS "Compile mono runtime components as static (not dynamic) libraries") option (DISABLE_WASM_USER_THREADS "Disable creation of user managed threads on WebAssembly, only allow runtime internal managed and native threads") -option (DISABLE_WEBCIL "Disable the WebCIL loader") +option (ENABLE_WEBCIL "Enable the WebCIL loader") set (MONO_GC "sgen" CACHE STRING "Garbage collector implementation (sgen or boehm). Default: sgen") set (GC_SUSPEND "default" CACHE STRING "GC suspend method (default, preemptive, coop, hybrid)") diff --git a/src/mono/mono.proj b/src/mono/mono.proj index b4289a0b76f8b..43a084f07a5a0 100644 --- a/src/mono/mono.proj +++ b/src/mono/mono.proj @@ -405,6 +405,7 @@ <_MonoCMakeArgs Include="-DDISABLE_ICALL_TABLES=1"/> <_MonoCMakeArgs Include="-DENABLE_ICALL_EXPORT=1"/> <_MonoCMakeArgs Include="-DENABLE_LAZY_GC_THREAD_CREATION=1"/> + <_MonoCMakeArgs Include="-DENABLE_WEBCIL=1"/> <_MonoCFLAGS Include="-fexceptions"/> <_MonoCFLAGS Condition="'$(MonoWasmThreads)' == 'true'" Include="-pthread"/> <_MonoCFLAGS Condition="'$(MonoWasmThreads)' == 'true'" Include="-D_GNU_SOURCE=1" /> diff --git a/src/mono/mono/metadata/assembly.c b/src/mono/mono/metadata/assembly.c index 274a55d96605a..5b2a80f12d686 100644 --- a/src/mono/mono/metadata/assembly.c +++ b/src/mono/mono/metadata/assembly.c @@ -1453,7 +1453,7 @@ absolute_dir (const gchar *filename) static gboolean bundled_assembly_match (const char *bundled_name, const char *name) { -#ifdef DISABLE_WEBCIL +#ifndef ENABLE_WEBCIL return strcmp (bundled_name, name) == 0; #else if (strcmp (bundled_name, name) == 0) @@ -2729,7 +2729,7 @@ mono_assembly_load_corlib (void) corlib = mono_assembly_request_open (corlib_name, &req, &status); g_free (corlib_name); } -#ifndef DISABLE_WEBCIL +#ifdef ENABLE_WEBCIL if (!corlib) { /* Maybe its in a bundle */ char *corlib_name = g_strdup_printf ("%s.webcil", MONO_ASSEMBLY_CORLIB_NAME); diff --git a/src/mono/mono/metadata/image.c b/src/mono/mono/metadata/image.c index fa5c3b09758ac..2eef0230f2a71 100644 --- a/src/mono/mono/metadata/image.c +++ b/src/mono/mono/metadata/image.c @@ -260,7 +260,7 @@ mono_images_init (void) install_pe_loader (); -#ifndef DISABLE_WEBCIL +#ifdef ENABLE_WEBCIL mono_webcil_loader_install (); #endif @@ -957,7 +957,7 @@ mono_has_pdb_checksum (char *raw_data, uint32_t raw_data_len) int32_t ret = try_load_pe_cli_header (raw_data, raw_data_len, &cli_header); -#ifndef DISABLE_WEBCIL +#ifdef ENABLE_WEBCIL if (ret == -1) { ret = mono_webcil_load_cli_header (raw_data, raw_data_len, 0, &cli_header); is_pe = FALSE; @@ -975,7 +975,7 @@ mono_has_pdb_checksum (char *raw_data, uint32_t raw_data_len) int i = 0; gboolean found = FALSE; for (i = 0; i < top; i++){ - MonoSectionTable t; + MonoSectionTable t = {0,}; if (G_LIKELY (is_pe)) { if (ret + sizeof (MonoSectionTable) > raw_data_len) @@ -990,7 +990,7 @@ mono_has_pdb_checksum (char *raw_data, uint32_t raw_data_len) t.st_raw_data_ptr = GUINT32_FROM_LE (t.st_raw_data_ptr); #endif } -#ifndef DISABLE_WEBCIL +#ifdef ENABLE_WEBCIL else { ret = mono_webcil_load_section_table (raw_data, raw_data_len, ret, &t); if (ret == -1) diff --git a/src/mono/mono/metadata/mono-debug.c b/src/mono/mono/metadata/mono-debug.c index 9e6f97c235599..958b33657d43a 100644 --- a/src/mono/mono/metadata/mono-debug.c +++ b/src/mono/mono/metadata/mono-debug.c @@ -1102,7 +1102,7 @@ bsymfile_match (BundledSymfile *bsymfile, const char *assembly_name) { if (!strcmp (bsymfile->aname, assembly_name)) return TRUE; -#ifndef DISABLE_WEBCIL +#ifdef ENABLE_WEBCIL const char *p = strstr (assembly_name, ".webcil"); /* if assembly_name ends with .webcil, check if aname matches, with a .dll extension instead */ if (p && *(p + 7) == 0) { diff --git a/src/mono/mono/mini/monovm.c b/src/mono/mono/mini/monovm.c index a70f2fd560c41..63e25581ac7a6 100644 --- a/src/mono/mono/mini/monovm.c +++ b/src/mono/mono/mini/monovm.c @@ -131,7 +131,7 @@ mono_core_preload_hook (MonoAssemblyLoadContext *alc, MonoAssemblyName *aname, c if (result) break; } -#ifndef DISABLE_WEBCIL +#ifdef ENABLE_WEBCIL else { /* /path/foo.dll -> /path/foo.webcil */ size_t n = strlen (fullpath) - 4; diff --git a/src/mono/wasm/build/WasmApp.targets b/src/mono/wasm/build/WasmApp.targets index 8d47e512d6ec5..8934610bfa84a 100644 --- a/src/mono/wasm/build/WasmApp.targets +++ b/src/mono/wasm/build/WasmApp.targets @@ -120,7 +120,7 @@ -1 - false From a2d4cbba7ea63e74daaa02d7029666528b9ba634 Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Wed, 11 Jan 2023 10:21:06 -0500 Subject: [PATCH 41/57] XXX HACK - Remove - turn on webcil by default on browser-wasm --- Directory.Build.props | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Directory.Build.props b/Directory.Build.props index 6738ab7a8d26c..38f25f017a896 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -410,4 +410,9 @@ $(RepositoryEngineeringDir)NoTargetsSdk.AfterTargets.targets $(RepositoryEngineeringDir)TraversalSdk.AfterTargets.targets + + + + true + From 388dc2cd980e08d96edc10d15f0c0e8befb0e374 Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Wed, 11 Jan 2023 16:05:30 -0500 Subject: [PATCH 42/57] Run WBT twice: with and without webcil This is a total of 4 runs: with and without workloads x with and without webcil --- src/libraries/sendtohelix-wasm.targets | 5 +++-- src/libraries/sendtohelix.proj | 7 ++++--- src/libraries/sendtohelixhelp.proj | 3 ++- src/mono/wasm/Wasm.Build.Tests/BuildEnvironment.cs | 5 +++++ src/mono/wasm/Wasm.Build.Tests/BuildTestBase.cs | 11 ++++++++--- .../wasm/Wasm.Build.Tests/EnvironmentVariables.cs | 1 + .../wasm/Wasm.Build.Tests/Wasm.Build.Tests.csproj | 3 +++ .../wasm/Wasm.Build.Tests/data/RunScriptTemplate.cmd | 5 +++++ .../wasm/Wasm.Build.Tests/data/RunScriptTemplate.sh | 6 ++++++ 9 files changed, 37 insertions(+), 9 deletions(-) diff --git a/src/libraries/sendtohelix-wasm.targets b/src/libraries/sendtohelix-wasm.targets index e1afc40d2cb1f..b177328a43585 100644 --- a/src/libraries/sendtohelix-wasm.targets +++ b/src/libraries/sendtohelix-wasm.targets @@ -236,6 +236,7 @@ $(RepositoryEngineeringDir)testing\scenarios\BuildWasmAppsJobsList.txt Workloads- NoWorkload- + $(WorkItemPrefix)Webcil- @@ -252,7 +253,7 @@ - + $(_BuildWasmAppsPayloadArchive) set "HELIX_XUNIT_ARGS=-class %(Identity)" export "HELIX_XUNIT_ARGS=-class %(Identity)" @@ -260,7 +261,7 @@ $(_workItemTimeout) - + $(_BuildWasmAppsPayloadArchive) $(HelixCommand) $(_workItemTimeout) diff --git a/src/libraries/sendtohelix.proj b/src/libraries/sendtohelix.proj index 65d5ca651db72..d2de27e65fc30 100644 --- a/src/libraries/sendtohelix.proj +++ b/src/libraries/sendtohelix.proj @@ -78,12 +78,13 @@ - + - <_TestUsingWorkloadsValues Include="true;false" /> + <_TestUsingWorkloadsValues Include="true;false" /> + <_TestUsingWebcilValues Include="true;false" /> <_BuildWasmAppsProjectsToBuild Include="$(PerScenarioProjectFile)"> - $(_PropertiesToPass);Scenario=BuildWasmApps;TestArchiveRuntimeFile=$(TestArchiveRuntimeFile);TestUsingWorkloads=%(_TestUsingWorkloadsValues.Identity) + $(_PropertiesToPass);Scenario=BuildWasmApps;TestArchiveRuntimeFile=$(TestArchiveRuntimeFile);TestUsingWorkloads=%(_TestUsingWorkloadsValues.Identity);TestUsingWebcil=%(_TestUsingWebcilValues.Identity) %(_BuildWasmAppsProjectsToBuild.AdditionalProperties);NeedsToBuildWasmAppsOnHelix=$(NeedsToBuildWasmAppsOnHelix) diff --git a/src/libraries/sendtohelixhelp.proj b/src/libraries/sendtohelixhelp.proj index 6f0b7704b575f..bfb6447e647b6 100644 --- a/src/libraries/sendtohelixhelp.proj +++ b/src/libraries/sendtohelixhelp.proj @@ -136,6 +136,7 @@ + @@ -287,7 +288,7 @@ + Text="Scenario: $(Scenario), TestUsingWorkloads: $(TestUsingWorkloads), TestUsingWebcil: $(TestUsingWebcil)" /> diff --git a/src/mono/wasm/Wasm.Build.Tests/BuildEnvironment.cs b/src/mono/wasm/Wasm.Build.Tests/BuildEnvironment.cs index f40ddec9aa4a8..6d88eb47a2563 100644 --- a/src/mono/wasm/Wasm.Build.Tests/BuildEnvironment.cs +++ b/src/mono/wasm/Wasm.Build.Tests/BuildEnvironment.cs @@ -24,11 +24,14 @@ public class BuildEnvironment public string WorkloadPacksDir { get; init; } public string BuiltNuGetsPath { get; init; } + public bool UseWebcil { get; init; } + public static readonly string RelativeTestAssetsPath = @"..\testassets\"; public static readonly string TestAssetsPath = Path.Combine(AppContext.BaseDirectory, "testassets"); public static readonly string TestDataPath = Path.Combine(AppContext.BaseDirectory, "data"); public static readonly string TmpPath = Path.Combine(AppContext.BaseDirectory, "wbt"); + private static readonly Dictionary s_runtimePackVersions = new(); public BuildEnvironment() @@ -86,6 +89,8 @@ public BuildEnvironment() DirectoryBuildTargetsContents = s_directoryBuildTargetsForLocal; } + UseWebcil = EnvironmentVariables.UseWebcil; + if (EnvironmentVariables.BuiltNuGetsPath is null || !Directory.Exists(EnvironmentVariables.BuiltNuGetsPath)) throw new Exception($"Cannot find 'BUILT_NUGETS_PATH={EnvironmentVariables.BuiltNuGetsPath}'"); diff --git a/src/mono/wasm/Wasm.Build.Tests/BuildTestBase.cs b/src/mono/wasm/Wasm.Build.Tests/BuildTestBase.cs index 23502a8849cb9..171e94d8ba5ea 100644 --- a/src/mono/wasm/Wasm.Build.Tests/BuildTestBase.cs +++ b/src/mono/wasm/Wasm.Build.Tests/BuildTestBase.cs @@ -50,6 +50,7 @@ public abstract class BuildTestBase : IClassFixture s_buildEnv.IsWorkload; public static bool IsNotUsingWorkloads => !s_buildEnv.IsWorkload; + public static bool UseWebcil => s_buildEnv.UseWebcil; public static string GetNuGetConfigPathFor(string targetFramework) => Path.Combine(BuildEnvironment.TestDataPath, "nuget8.config"); // for now - we are still using net7, but with // targetFramework == "net7.0" ? "nuget7.config" : "nuget8.config"); @@ -71,6 +72,8 @@ static BuildTestBase() Console.WriteLine (""); Console.WriteLine ($"=============================================================================================="); Console.WriteLine ($"=============== Running with {(s_buildEnv.IsWorkload ? "Workloads" : "No workloads")} ==============="); + if (UseWebcil) + Console.WriteLine($"=============== Using .webcil ==============="); Console.WriteLine ($"=============================================================================================="); Console.WriteLine (""); } @@ -320,6 +323,7 @@ protected void InitProjectDir(string dir, bool addNuGetSourceForLocalPackages = Exe true test-main.js + ##USE_WEBCIL## ##EXTRA_PROPERTIES## @@ -337,6 +341,7 @@ protected static BuildArgs ExpandBuildArgs(BuildArgs buildArgs, string extraProp } string projectContents = projectTemplate + .Replace("##USE_WEBCIL##", UseWebcil ? "true" : "") .Replace("##EXTRA_PROPERTIES##", extraProperties) .Replace("##EXTRA_ITEMS##", extraItems) .Replace("##INSERT_AT_END##", insertAtEnd); @@ -424,7 +429,7 @@ protected static BuildArgs ExpandBuildArgs(BuildArgs buildArgs, string extraProp options.TargetFramework ?? DefaultTargetFramework, options.HasIcudt, options.DotnetWasmFromRuntimePack ?? !buildArgs.AOT, - options.UseWebcil); + UseWebcil); } if (options.UseCache) @@ -551,6 +556,7 @@ public string CreateBlazorWasmTemplateProject(string id) label, // same as the command name $"-bl:{logPath}", $"-p:Configuration={config}", + UseWebcil ? "-p:WasmEnableWebcil=true" : string.Empty, "-p:BlazorEnableCompression=false", "-nr:false", setWasmDevel ? "-p:_WasmDevel=true" : string.Empty @@ -1098,8 +1104,7 @@ public record BuildProjectOptions bool CreateProject = true, bool Publish = true, bool BuildOnlyAfterPublish = true, - bool HasV8Script = true, - bool UseWebcil = true, + bool HasV8Script = true, string? Verbosity = null, string? Label = null, string? TargetFramework = null, diff --git a/src/mono/wasm/Wasm.Build.Tests/EnvironmentVariables.cs b/src/mono/wasm/Wasm.Build.Tests/EnvironmentVariables.cs index 328006786d09c..13270145d1be4 100644 --- a/src/mono/wasm/Wasm.Build.Tests/EnvironmentVariables.cs +++ b/src/mono/wasm/Wasm.Build.Tests/EnvironmentVariables.cs @@ -18,5 +18,6 @@ internal static class EnvironmentVariables internal static readonly string? BuiltNuGetsPath = Environment.GetEnvironmentVariable("BUILT_NUGETS_PATH"); internal static readonly string? BrowserPathForTests = Environment.GetEnvironmentVariable("BROWSER_PATH_FOR_TESTS"); internal static readonly bool ShowBuildOutput = Environment.GetEnvironmentVariable("SHOW_BUILD_OUTPUT") is not null; + internal static readonly bool UseWebcil = Environment.GetEnvironmentVariable("USE_WEBCIL_FOR_TESTS") is not null; } } diff --git a/src/mono/wasm/Wasm.Build.Tests/Wasm.Build.Tests.csproj b/src/mono/wasm/Wasm.Build.Tests/Wasm.Build.Tests.csproj index cf7a20b562391..133140d696095 100644 --- a/src/mono/wasm/Wasm.Build.Tests/Wasm.Build.Tests.csproj +++ b/src/mono/wasm/Wasm.Build.Tests/Wasm.Build.Tests.csproj @@ -66,6 +66,9 @@ + + + diff --git a/src/mono/wasm/Wasm.Build.Tests/data/RunScriptTemplate.cmd b/src/mono/wasm/Wasm.Build.Tests/data/RunScriptTemplate.cmd index f38746bf13151..23e3dbf4ce470 100644 --- a/src/mono/wasm/Wasm.Build.Tests/data/RunScriptTemplate.cmd +++ b/src/mono/wasm/Wasm.Build.Tests/data/RunScriptTemplate.cmd @@ -104,6 +104,11 @@ if [%TEST_USING_WORKLOADS%] == [true] ( set _DIR_NAME=dotnet-none set SDK_HAS_WORKLOAD_INSTALLED=false ) +if [%TEST_USING_WEBCIL%] == [true] ( + set USE_WEBCIL_FOR_TESTS=true +) else ( + set USE_WEBCIL_FOR_TESTS=false +) if [%HELIX_CORRELATION_PAYLOAD%] NEQ [] ( robocopy /mt /np /nfl /NDL /nc /e %BASE_DIR%\%_DIR_NAME% %EXECUTION_DIR%\%_DIR_NAME% diff --git a/src/mono/wasm/Wasm.Build.Tests/data/RunScriptTemplate.sh b/src/mono/wasm/Wasm.Build.Tests/data/RunScriptTemplate.sh index 1f520aebd65f6..70187b9c5e58e 100644 --- a/src/mono/wasm/Wasm.Build.Tests/data/RunScriptTemplate.sh +++ b/src/mono/wasm/Wasm.Build.Tests/data/RunScriptTemplate.sh @@ -79,6 +79,12 @@ function set_env_vars() export SDK_HAS_WORKLOAD_INSTALLED=false fi + if [ "x$TEST_USING_WEBCIL" = "xtrue" ]; then + export USE_WEBCIL_FOR_TESTS=true + else + export USE_WEBCIL_FOR_TESTS=false + fi + local _SDK_DIR= if [[ -n "$HELIX_WORKITEM_UPLOAD_ROOT" ]]; then cp -r $BASE_DIR/$_DIR_NAME $EXECUTION_DIR From 1abe30a6a79b53d3c72f0718810ab046c8da0f8a Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Fri, 13 Jan 2023 15:40:26 -0500 Subject: [PATCH 43/57] do the cartesian product correctly in msbuild --- src/libraries/sendtohelix.proj | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/libraries/sendtohelix.proj b/src/libraries/sendtohelix.proj index d2de27e65fc30..71619fc40466a 100644 --- a/src/libraries/sendtohelix.proj +++ b/src/libraries/sendtohelix.proj @@ -80,11 +80,19 @@ - <_TestUsingWorkloadsValues Include="true;false" /> - <_TestUsingWebcilValues Include="true;false" /> - + <_TestUsingWorkloadsValues Include="true;false" /> + <_TestUsingWebcilValues Include="true;false" /> + + + <_TestUsingCrossProductValuesTemp Include="@(_TestUsingWorkloadsValues)"> + %(_TestUsingWorkloadsValues.Identity) + + <_TestUsingCrossProductValues Include="@(_TestUsingCrossProductValuesTemp)"> + %(_TestUsingWebcilValues.Identity) + + <_BuildWasmAppsProjectsToBuild Include="$(PerScenarioProjectFile)"> - $(_PropertiesToPass);Scenario=BuildWasmApps;TestArchiveRuntimeFile=$(TestArchiveRuntimeFile);TestUsingWorkloads=%(_TestUsingWorkloadsValues.Identity);TestUsingWebcil=%(_TestUsingWebcilValues.Identity) + $(_PropertiesToPass);Scenario=BuildWasmApps;TestArchiveRuntimeFile=$(TestArchiveRuntimeFile);TestUsingWorkloads=%(_TestUsingCrossProductValues.Workloads);TestUsingWebcil=%(_TestUsingCrossProductValues.Webcil) %(_BuildWasmAppsProjectsToBuild.AdditionalProperties);NeedsToBuildWasmAppsOnHelix=$(NeedsToBuildWasmAppsOnHelix) From 29c52348a18bc610fd8d04c0d7c0c0a0f89ed70d Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Fri, 13 Jan 2023 18:49:14 -0500 Subject: [PATCH 44/57] Revert "XXX HACK - Remove - turn on webcil by default on browser-wasm" This reverts commit 0f14c3b7c613ad5569316a757587b10dbe2aaa02. --- Directory.Build.props | 5 ----- 1 file changed, 5 deletions(-) diff --git a/Directory.Build.props b/Directory.Build.props index 38f25f017a896..6738ab7a8d26c 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -410,9 +410,4 @@ $(RepositoryEngineeringDir)NoTargetsSdk.AfterTargets.targets $(RepositoryEngineeringDir)TraversalSdk.AfterTargets.targets - - - - true - From 3b6bef2fbb93c8164155e809198e3b9f66097a8b Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Tue, 17 Jan 2023 13:09:23 -0500 Subject: [PATCH 45/57] also add webcil to template projects --- src/mono/wasm/Wasm.Build.Tests/BuildTestBase.cs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/mono/wasm/Wasm.Build.Tests/BuildTestBase.cs b/src/mono/wasm/Wasm.Build.Tests/BuildTestBase.cs index 171e94d8ba5ea..9ebc210e803b5 100644 --- a/src/mono/wasm/Wasm.Build.Tests/BuildTestBase.cs +++ b/src/mono/wasm/Wasm.Build.Tests/BuildTestBase.cs @@ -323,7 +323,6 @@ protected void InitProjectDir(string dir, bool addNuGetSourceForLocalPackages = Exe true test-main.js - ##USE_WEBCIL## ##EXTRA_PROPERTIES## @@ -340,8 +339,11 @@ protected static BuildArgs ExpandBuildArgs(BuildArgs buildArgs, string extraProp extraProperties += $"\n{RuntimeInformation.IsOSPlatform(OSPlatform.Windows)}\n"; } + if (UseWebcil) { + extraProperties += "true\n"; + } + string projectContents = projectTemplate - .Replace("##USE_WEBCIL##", UseWebcil ? "true" : "") .Replace("##EXTRA_PROPERTIES##", extraProperties) .Replace("##EXTRA_ITEMS##", extraItems) .Replace("##INSERT_AT_END##", insertAtEnd); @@ -493,6 +495,8 @@ public string CreateWasmTemplateProject(string id, string template = "wasmbrowse string projectfile = Path.Combine(_projectDir!, $"{id}.csproj"); if (runAnalyzers) AddItemsPropertiesToProject(projectfile, "true"); + if (UseWebcil) + AddItemsPropertiesToProject(projectfile, "true"); return projectfile; } @@ -505,7 +509,10 @@ public string CreateBlazorWasmTemplateProject(string id) .ExecuteWithCapturedOutput("new blazorwasm") .EnsureSuccessful(); - return Path.Combine(_projectDir!, $"{id}.csproj"); + string projectFile = Path.Combine(_projectDir!, $"{id}.csproj"); + if (UseWebcil) + AddItemsPropertiesToProject(projectFile, "true"); + return projectFile; } protected (CommandResult, string) BlazorBuild(BlazorBuildOptions options, params string[] extraArgs) From 4ab4f8f49f9f9a27c4c243964c4d0537b95323d2 Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Tue, 17 Jan 2023 13:56:23 -0500 Subject: [PATCH 46/57] environment variable has to be non-null and "true" We set it to "false" sometimes --- src/mono/wasm/Wasm.Build.Tests/EnvironmentVariables.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mono/wasm/Wasm.Build.Tests/EnvironmentVariables.cs b/src/mono/wasm/Wasm.Build.Tests/EnvironmentVariables.cs index 13270145d1be4..91f6fadb51f5b 100644 --- a/src/mono/wasm/Wasm.Build.Tests/EnvironmentVariables.cs +++ b/src/mono/wasm/Wasm.Build.Tests/EnvironmentVariables.cs @@ -18,6 +18,6 @@ internal static class EnvironmentVariables internal static readonly string? BuiltNuGetsPath = Environment.GetEnvironmentVariable("BUILT_NUGETS_PATH"); internal static readonly string? BrowserPathForTests = Environment.GetEnvironmentVariable("BROWSER_PATH_FOR_TESTS"); internal static readonly bool ShowBuildOutput = Environment.GetEnvironmentVariable("SHOW_BUILD_OUTPUT") is not null; - internal static readonly bool UseWebcil = Environment.GetEnvironmentVariable("USE_WEBCIL_FOR_TESTS") is not null; + internal static readonly bool UseWebcil = Environment.GetEnvironmentVariable("USE_WEBCIL_FOR_TESTS") is "true"; } } From 5741cb8fa71f64e83cecd7142b9ede192ab036c0 Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Tue, 17 Jan 2023 15:21:50 -0500 Subject: [PATCH 47/57] Fix wasm work items They shoudl be the same whether or not webcil is used. Just the WorkloadItemPrefix should be used to change the name. Co-Authored-By: Ankit Jain --- src/libraries/sendtohelix-wasm.targets | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libraries/sendtohelix-wasm.targets b/src/libraries/sendtohelix-wasm.targets index b177328a43585..cbee73699ae69 100644 --- a/src/libraries/sendtohelix-wasm.targets +++ b/src/libraries/sendtohelix-wasm.targets @@ -253,7 +253,7 @@ - + $(_BuildWasmAppsPayloadArchive) set "HELIX_XUNIT_ARGS=-class %(Identity)" export "HELIX_XUNIT_ARGS=-class %(Identity)" @@ -261,7 +261,7 @@ $(_workItemTimeout) - + $(_BuildWasmAppsPayloadArchive) $(HelixCommand) $(_workItemTimeout) From 61fef7366b1e132f85164ed0ea3e538d68def709 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleksey=20Kliger=20=28=CE=BBgeek=29?= Date: Tue, 17 Jan 2023 19:17:29 -0500 Subject: [PATCH 48/57] Update src/libraries/sendtohelix-wasm.targets Co-authored-by: Ankit Jain --- src/libraries/sendtohelix-wasm.targets | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libraries/sendtohelix-wasm.targets b/src/libraries/sendtohelix-wasm.targets index cbee73699ae69..0e9db2b6f861d 100644 --- a/src/libraries/sendtohelix-wasm.targets +++ b/src/libraries/sendtohelix-wasm.targets @@ -261,7 +261,7 @@ $(_workItemTimeout) - + $(_BuildWasmAppsPayloadArchive) $(HelixCommand) $(_workItemTimeout) From 81b8bcea1ba6fbf96120214d9bae863da0edc724 Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Wed, 18 Jan 2023 14:51:15 -0500 Subject: [PATCH 49/57] FIXME: why is this crashing in MetadataLoadContext --- src/mono/wasm/Wasm.Build.Tests/PInvokeTableGeneratorTests.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/mono/wasm/Wasm.Build.Tests/PInvokeTableGeneratorTests.cs b/src/mono/wasm/Wasm.Build.Tests/PInvokeTableGeneratorTests.cs index 5473667f294c9..267b39d8f0327 100644 --- a/src/mono/wasm/Wasm.Build.Tests/PInvokeTableGeneratorTests.cs +++ b/src/mono/wasm/Wasm.Build.Tests/PInvokeTableGeneratorTests.cs @@ -376,6 +376,7 @@ public static int Main() } [ConditionalTheory(typeof(BuildTestBase), nameof(IsUsingWorkloads))] + [ActiveIssue("FIXME: MetadataLoadContet dotnet build crash", typeof(BuildTestBase), nameof(UseWebcil))] [BuildAndRun(host: RunHost.None)] public void IcallWithOverloadedParametersAndEnum(BuildArgs buildArgs, string id) { From a795f9e880032bbb5acd8a4625fbad022c317834 Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Wed, 18 Jan 2023 15:42:04 -0500 Subject: [PATCH 50/57] PInvokeTableGeneratorTests: don't try to use the net472 WasmAppBuilder Look for the default target framework subdirectory under the tasks directory in the runtime pack when trying to find the tasks dll. In particular don't try to load the net472 version on modern .NET --- .../wasm/Wasm.Build.Tests/PInvokeTableGeneratorTests.cs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/mono/wasm/Wasm.Build.Tests/PInvokeTableGeneratorTests.cs b/src/mono/wasm/Wasm.Build.Tests/PInvokeTableGeneratorTests.cs index 267b39d8f0327..abf51dd29226f 100644 --- a/src/mono/wasm/Wasm.Build.Tests/PInvokeTableGeneratorTests.cs +++ b/src/mono/wasm/Wasm.Build.Tests/PInvokeTableGeneratorTests.cs @@ -376,7 +376,6 @@ public static int Main() } [ConditionalTheory(typeof(BuildTestBase), nameof(IsUsingWorkloads))] - [ActiveIssue("FIXME: MetadataLoadContet dotnet build crash", typeof(BuildTestBase), nameof(UseWebcil))] [BuildAndRun(host: RunHost.None)] public void IcallWithOverloadedParametersAndEnum(BuildArgs buildArgs, string id) { @@ -467,7 +466,8 @@ public static void Main() string tasksDir = Path.Combine(s_buildEnv.WorkloadPacksDir, "Microsoft.NET.Runtime.WebAssembly.Sdk", s_buildEnv.GetRuntimePackVersion(DefaultTargetFramework), - "tasks"); + "tasks", + BuildTestBase.DefaultTargetFramework); // not net472! if (!Directory.Exists(tasksDir)) throw new DirectoryNotFoundException($"Could not find tasks directory {tasksDir}"); @@ -476,6 +476,8 @@ public static void Main() if (string.IsNullOrEmpty(taskPath)) throw new FileNotFoundException($"Could not find WasmAppBuilder.dll in {tasksDir}"); + _testOutput.WriteLine ("Using WasmAppBuilder.dll from {0}", taskPath); + projectCode = projectCode .Replace("###WasmPInvokeModule###", AddAssembly("System.Private.CoreLib") + AddAssembly("System.Runtime") + AddAssembly(libraryBuildArgs.ProjectName)) .Replace("###WasmAppBuilder###", taskPath); From 0473ec1c4326711665997fdda49267f0083efc47 Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Wed, 18 Jan 2023 15:51:53 -0500 Subject: [PATCH 51/57] PInvokeTableGeneratorTests: Add more diagnostic output if tasksDir is not found --- .../Wasm.Build.Tests/PInvokeTableGeneratorTests.cs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/mono/wasm/Wasm.Build.Tests/PInvokeTableGeneratorTests.cs b/src/mono/wasm/Wasm.Build.Tests/PInvokeTableGeneratorTests.cs index abf51dd29226f..06325a60b7881 100644 --- a/src/mono/wasm/Wasm.Build.Tests/PInvokeTableGeneratorTests.cs +++ b/src/mono/wasm/Wasm.Build.Tests/PInvokeTableGeneratorTests.cs @@ -468,8 +468,17 @@ public static void Main() s_buildEnv.GetRuntimePackVersion(DefaultTargetFramework), "tasks", BuildTestBase.DefaultTargetFramework); // not net472! - if (!Directory.Exists(tasksDir)) + if (!Directory.Exists(tasksDir)) { + string tasksDirParent = Path.GetDirectoryName (tasksDir); + if (!Directory.Exists(tasksDirParent)) { + _testOutput.WriteLine($"Expected {tasksDirParent} to exist and contain TFM subdirectories"); + } + _testOutput.WriteLine($"runtime pack tasks dir {tasksDir} contains subdirectories:"); + foreach (string subdir in Directory.EnumerateDirectories(tasksDirParent)) { + _testOutput.WriteLine($" - {subdir}"); + } throw new DirectoryNotFoundException($"Could not find tasks directory {tasksDir}"); + } string? taskPath = Directory.EnumerateFiles(tasksDir, "WasmAppBuilder.dll", SearchOption.AllDirectories) .FirstOrDefault(); From 56addec8eaf6e3e049f68fb1b4b8343b67754a0f Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Wed, 18 Jan 2023 16:31:04 -0500 Subject: [PATCH 52/57] nullability --- .../PInvokeTableGeneratorTests.cs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/mono/wasm/Wasm.Build.Tests/PInvokeTableGeneratorTests.cs b/src/mono/wasm/Wasm.Build.Tests/PInvokeTableGeneratorTests.cs index 06325a60b7881..0168840ad99bf 100644 --- a/src/mono/wasm/Wasm.Build.Tests/PInvokeTableGeneratorTests.cs +++ b/src/mono/wasm/Wasm.Build.Tests/PInvokeTableGeneratorTests.cs @@ -469,13 +469,15 @@ public static void Main() "tasks", BuildTestBase.DefaultTargetFramework); // not net472! if (!Directory.Exists(tasksDir)) { - string tasksDirParent = Path.GetDirectoryName (tasksDir); - if (!Directory.Exists(tasksDirParent)) { - _testOutput.WriteLine($"Expected {tasksDirParent} to exist and contain TFM subdirectories"); - } - _testOutput.WriteLine($"runtime pack tasks dir {tasksDir} contains subdirectories:"); - foreach (string subdir in Directory.EnumerateDirectories(tasksDirParent)) { - _testOutput.WriteLine($" - {subdir}"); + string? tasksDirParent = Path.GetDirectoryName (tasksDir); + if (!string.IsNullOrEmpty (tasksDirParent)) { + if (!Directory.Exists(tasksDirParent)) { + _testOutput.WriteLine($"Expected {tasksDirParent} to exist and contain TFM subdirectories"); + } + _testOutput.WriteLine($"runtime pack tasks dir {tasksDir} contains subdirectories:"); + foreach (string subdir in Directory.EnumerateDirectories(tasksDirParent)) { + _testOutput.WriteLine($" - {subdir}"); + } } throw new DirectoryNotFoundException($"Could not find tasks directory {tasksDir}"); } From 52b195a7df4e7ce1488df6d336fd40226837f1f2 Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Thu, 19 Jan 2023 15:01:13 -0500 Subject: [PATCH 53/57] simplify prefix comparison in bundled_assembly_match --- src/mono/mono/metadata/assembly.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/mono/mono/metadata/assembly.c b/src/mono/mono/metadata/assembly.c index 5b2a80f12d686..2538329ad7ac9 100644 --- a/src/mono/mono/metadata/assembly.c +++ b/src/mono/mono/metadata/assembly.c @@ -1462,8 +1462,7 @@ bundled_assembly_match (const char *bundled_name, const char *name) if (g_str_has_suffix (bundled_name, ".webcil") && g_str_has_suffix (name, ".dll")) { size_t bprefix = strlen (bundled_name) - 7; size_t nprefix = strlen (name) - 4; - size_t longer = MAX(bprefix, nprefix); - if (strncmp (bundled_name, name, longer) == 0) + if (bprefix == nprefix && strncmp (bundled_name, name, bprefix) == 0) return TRUE; } return FALSE; From a176400db42ae78b06bbf53f1a47092b11872b8a Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Thu, 19 Jan 2023 15:05:29 -0500 Subject: [PATCH 54/57] WasmAppBuilder improve logging Just emit a single Normal importance message about webcil; details as Low importance. --- src/tasks/WasmAppBuilder/WasmAppBuilder.cs | 2 ++ src/tasks/WasmAppBuilder/WebcilConverter.cs | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/tasks/WasmAppBuilder/WasmAppBuilder.cs b/src/tasks/WasmAppBuilder/WasmAppBuilder.cs index 9d60875c8e05d..8dd76eeab7952 100644 --- a/src/tasks/WasmAppBuilder/WasmAppBuilder.cs +++ b/src/tasks/WasmAppBuilder/WasmAppBuilder.cs @@ -188,6 +188,8 @@ private bool ExecuteInternal () var asmRootPath = Path.Combine(AppDir, config.AssemblyRootFolder); Directory.CreateDirectory(AppDir!); Directory.CreateDirectory(asmRootPath); + if (UseWebcil) + Log.LogMessage (MessageImportance.Normal, "Converting assemblies to Webcil"); foreach (var assembly in _assemblies) { if (UseWebcil) diff --git a/src/tasks/WasmAppBuilder/WebcilConverter.cs b/src/tasks/WasmAppBuilder/WebcilConverter.cs index 3cd8b1fc74514..694b698a7ecf4 100644 --- a/src/tasks/WasmAppBuilder/WebcilConverter.cs +++ b/src/tasks/WasmAppBuilder/WebcilConverter.cs @@ -37,7 +37,7 @@ public static WebcilConverter FromPortableExecutable(string inputPath, string ou public void ConvertToWebcil() { - Log.LogMessage($"Converting to Webcil: input {_inputPath} output: {_outputPath}"); + Log.LogMessage(MessageImportance.Low, $"Converting to Webcil: input {_inputPath} output: {_outputPath}"); _converter.ConvertToWebcil(); } From 67d64e3aacd899a60f7ed1e82cf5d6ab6066c321 Mon Sep 17 00:00:00 2001 From: Larry Ewing Date: Thu, 19 Jan 2023 16:50:10 -0600 Subject: [PATCH 55/57] Add missing using --- src/tasks/WasmAppBuilder/WebcilConverter.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/tasks/WasmAppBuilder/WebcilConverter.cs b/src/tasks/WasmAppBuilder/WebcilConverter.cs index 694b698a7ecf4..51add150a953c 100644 --- a/src/tasks/WasmAppBuilder/WebcilConverter.cs +++ b/src/tasks/WasmAppBuilder/WebcilConverter.cs @@ -6,6 +6,7 @@ using System.Collections.Immutable; using System.Reflection.PortableExecutable; +using Microsoft.Build.Framework; using Microsoft.Build.Utilities; namespace Microsoft.WebAssembly.Build.Tasks; From 5983e65d3ba9b638a0aeb08cff810d406d9c3d69 Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Fri, 20 Jan 2023 15:02:25 -0500 Subject: [PATCH 56/57] XXX REVERT ME - temporarily disable dedup --- src/mono/wasm/build/WasmApp.targets | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mono/wasm/build/WasmApp.targets b/src/mono/wasm/build/WasmApp.targets index 8934610bfa84a..1b59cd23c6bdb 100644 --- a/src/mono/wasm/build/WasmApp.targets +++ b/src/mono/wasm/build/WasmApp.targets @@ -88,7 +88,7 @@ --> - true + false false false From d7b364086d2253e66065079e321a6459c9a7209a Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Fri, 20 Jan 2023 16:44:36 -0500 Subject: [PATCH 57/57] Revert "XXX REVERT ME - temporarily disable dedup" This reverts commit 5983e65d3ba9b638a0aeb08cff810d406d9c3d69. --- src/mono/wasm/build/WasmApp.targets | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mono/wasm/build/WasmApp.targets b/src/mono/wasm/build/WasmApp.targets index 1b59cd23c6bdb..8934610bfa84a 100644 --- a/src/mono/wasm/build/WasmApp.targets +++ b/src/mono/wasm/build/WasmApp.targets @@ -88,7 +88,7 @@ --> - false + true false false