R2RDump support for MVIDs; R2R format update; various minor fixes

(*) Renamed 'ComponentAssemblyMvids' to 'ManifestAssemblyMvids' to
better express the fact that the MVID table is parallel to the
manifest metadata.

(*) Added basic R2RDump support for dumping the manifest assembly
MVID table.

(*) Fixed R2R format version number in Crossgen2.

Thanks

Tomas

Author: trylek

docs/design/coreclr/botr/readytorun-format.md

@@ -5,6 +5,7 @@ Revisions:
 * 1.1 - [Jan Kotas](https://github.com/jkotas) - 2015
 * 3.1 - [Tomas Rylek](https://github.com/trylek) - 2019
 * 4.1 - [Tomas Rylek](https://github.com/trylek) - 2020
+* 5.3 - [Tomas Rylek](https://github.com/trylek) - 2021
 
 # Introduction
 
@@ -161,6 +162,8 @@ The following section types are defined and described later in this document:
 | InliningInfo2             |   114 | Image (added in V4.1)
 | ComponentAssemblies       |   115 | Image (added in V4.1)
 | OwnerCompositeExecutable  |   116 | Image (added in V4.1)
+| PgoInstrumentationData    |   117 | Image (added in V5.2)
+| ManifestAssemblyMvids     |   118 | Image (added in V5.3)
 
 ## ReadyToRunSectionType.CompilerIdentifier
 
@@ -540,6 +543,17 @@ the `OwnerCompositeExecutable` section that contains a UTF-8 string encoding the
 composite R2R executable this MSIL belongs to with extension (without path). Runtime uses this
 information to locate the composite R2R executable with the compiled native code when loading the MSIL.
 
+## ReadyToRunSectionType.PgoInstrumentationData (v5.2+)
+
+**TODO**: document PGO instrumentation data
+
+## ReadyToRunSectionType.ManifestAssemblyMvids (v5.3+)
+
+This section is a binary array of 16-byte MVID records, one for each assembly in the manifest metadata.
+Number of assemblies stored in the manifest metadata is equal to the number of MVID records in the array.
+MVID records are used at runtime to verify that the assemblies loaded match those referenced by the
+manifest metadata representing the versioning bubble.
+
 # Native Format
 
 Native format is set of encoding patterns that allow persisting type system data in a binary format that is

src/coreclr/inc/readytorun.h

@@ -80,7 +80,7 @@ enum class ReadyToRunSectionType : uint32_t
     ComponentAssemblies         = 115, // Added in V4.1
     OwnerCompositeExecutable    = 116, // Added in V4.1
     PgoInstrumentationData      = 117, // Added in 5.2
-    ComponentAssemblyMvids      = 118, // Added in V5.3
+    ManifestAssemblyMvids       = 118, // Added in V5.3
 
     // If you add a new section consider whether it is a breaking or non-breaking change.
     // Usually it is non-breaking, but if it is preferable to have older runtimes fail

src/coreclr/tools/Common/Internal/Runtime/ModuleHeaders.cs

@@ -15,7 +15,7 @@ internal struct ReadyToRunHeaderConstants
         public const uint Signature = 0x00525452; // 'RTR'
 
         public const ushort CurrentMajorVersion = 5;
-        public const ushort CurrentMinorVersion = 4;
+        public const ushort CurrentMinorVersion = 3;
     }
 
 #pragma warning disable 0169
@@ -66,6 +66,7 @@ public enum ReadyToRunSectionType
         ComponentAssemblies = 115, // Added in 4.1
         OwnerCompositeExecutable = 116, // Added in 4.1
         PgoInstrumentationData = 117, // Added in 5.2
+        ManifestAssemblyMvids = 118, // Added in 5.3
 
         //
         // CoreRT ReadyToRun sections

src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/ComponentAssemblyMvidHeaderNode.cs renamed to src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/ManifestAssemblyMvidHeaderNode.cs

@@ -1,6 +1,7 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
+using System;
 using System.Reflection.Metadata;
 using System.Reflection.PortableExecutable;
 
@@ -11,11 +12,11 @@
 
 namespace ILCompiler.DependencyAnalysis.ReadyToRun
 {
-    public class ComponentMetadataMvidHeaderNode : ObjectNode, ISymbolDefinitionNode
+    public class ManifestAssemblyMvidHeaderNode : ObjectNode, ISymbolDefinitionNode
     {
         private ManifestMetadataTableNode _manifestNode;
 
-        public ComponentMetadataMvidHeaderNode(ManifestMetadataTableNode manifestNode)
+        public ManifestAssemblyMvidHeaderNode(ManifestMetadataTableNode manifestNode)
         {
             _manifestNode = manifestNode;
         }
@@ -26,18 +27,18 @@ public ComponentMetadataMvidHeaderNode(ManifestMetadataTableNode manifestNode)
 
         protected internal override int Phase => (int)ObjectNodePhase.Ordered;
 
-        public override int ClassCode => (int)ObjectNodeOrder.CorHeaderNode;
+        public override int ClassCode => 735231445;
 
         public override bool StaticDependenciesAreComputed => true;
 
         public int Offset => 0;
 
-        public int Size => _manifestNode.ComponentAssemblyMvidTableSize;
+        public int Size => _manifestNode.ManifestAssemblyMvidTableSize;
 
         public void AppendMangledName(NameMangler nameMangler, Utf8StringBuilder sb)
         {
             sb.Append(nameMangler.CompilationUnitPrefix);
-            sb.Append("__ComponentAssemblyMvids");
+            sb.Append("__ManifestAssemblyMvids");
         }
 
         protected override string GetName(NodeFactory nodeFactory)
@@ -47,6 +48,15 @@ protected override string GetName(NodeFactory nodeFactory)
             return sb.ToString();
         }
 
-        public override ObjectData GetData(NodeFactory factory, bool relocsOnly = false) => _manifestNode.GetComponentAssemblyMvidTableData(factory, relocsOnly);
+        public override ObjectData GetData(NodeFactory factory, bool relocsOnly = false)
+        {
+            if (relocsOnly)
+            {
+                return new ObjectData(Array.Empty<byte>(), null, 1, null);
+            }
+
+            byte[] manifestAssemblyMvidTable = _manifestNode.GetManifestAssemblyMvidTableData();
+            return new ObjectData(manifestAssemblyMvidTable, Array.Empty<Relocation>(), alignment: 0, new ISymbolDefinitionNode[] { this });
+        }
     }
 }

src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/ManifestMetadataTableNode.cs

@@ -43,10 +43,10 @@ public class ManifestMetadataTableNode : HeaderTableNode
 
         /// <summary>
         /// MVID's of the assemblies included in manifest metadata to be emitted as the
-        /// ComponentAssemblyMvid R2R header table used by the runtime to check loaded assemblies
+        /// ManifestAssemblyMvid R2R header table used by the runtime to check loaded assemblies
         /// and fail fast in case of mismatch.
         /// </summary>
-        private readonly List<Guid> _componentAssemblyMvids;
+        private readonly List<Guid> _manifestAssemblyMvids;
 
         /// <summary>
         /// Registered signature emitters.
@@ -78,7 +78,7 @@ public ManifestMetadataTableNode(NodeFactory nodeFactory)
         {
             _assemblyRefToModuleIdMap = new Dictionary<string, int>();
             _moduleIdToAssemblyNameMap = new Dictionary<int, AssemblyName>();
-            _componentAssemblyMvids = new List<Guid>();
+            _manifestAssemblyMvids = new List<Guid>();
             _signatureEmitters = new List<ISignatureEmitter>();
             _nodeFactory = nodeFactory;
             _nextModuleId = 1;
@@ -158,7 +158,7 @@ private int ModuleToIndexInternal(EcmaModule module)
                 Debug.Assert(_nodeFactory.CompilationModuleGroup.VersionsWithModule(module));
 
                 _moduleIdToAssemblyNameMap.Add(assemblyRefIndex, assemblyName);
-                _componentAssemblyMvids.Add(module.MetadataReader.GetGuid(module.MetadataReader.GetModuleDefinition().Mvid));
+                _manifestAssemblyMvids.Add(module.MetadataReader.GetGuid(module.MetadataReader.GetModuleDefinition().Mvid));
             }
             return assemblyRefIndex;
         }
@@ -253,26 +253,21 @@ public override ObjectData GetData(NodeFactory factory, bool relocsOnly = false)
 
         private const int GuidByteSize = 16;
 
-        public int ComponentAssemblyMvidTableSize => GuidByteSize * _componentAssemblyMvids.Count;
+        public int ManifestAssemblyMvidTableSize => GuidByteSize * _manifestAssemblyMvids.Count;
 
-        public ObjectData GetComponentAssemblyMvidTableData(NodeFactory factory, bool relocsOnly = false)
+        internal byte[] GetManifestAssemblyMvidTableData()
         {
-            if (relocsOnly)
-            {
-                return new ObjectData(Array.Empty<byte>(), null, 1, null);
-            }
-
-            byte[] componentAssemblyMvidTable = new byte[ComponentAssemblyMvidTableSize];
-            for (int i = 0; i < _componentAssemblyMvids.Count; i++)
+            byte[] manifestAssemblyMvidTable = new byte[ManifestAssemblyMvidTableSize];
+            for (int i = 0; i < _manifestAssemblyMvids.Count; i++)
             {
                 Array.Copy(
-                    sourceArray: _componentAssemblyMvids[i].ToByteArray(),
+                    sourceArray: _manifestAssemblyMvids[i].ToByteArray(),
                     sourceIndex: 0,
-                    destinationArray: componentAssemblyMvidTable,
+                    destinationArray: manifestAssemblyMvidTable,
                     destinationIndex: GuidByteSize * i,
                     length: GuidByteSize);
             }
-            return new ObjectData(componentAssemblyMvidTable, Array.Empty<Relocation>(), alignment: 0, new ISymbolDefinitionNode[] { this });
+            return manifestAssemblyMvidTable;
         }
     }
 }

src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRunCodegenNodeFactory.cs

@@ -570,6 +570,9 @@ public void AttachToDependencyGraph(DependencyAnalyzerBase<NodeFactory> graph)
             Header.Add(Internal.Runtime.ReadyToRunSectionType.ManifestMetadata, ManifestMetadataTable, ManifestMetadataTable);
             Resolver.SetModuleIndexLookup(ManifestMetadataTable.ModuleToIndex);
 
+            ManifestAssemblyMvidHeaderNode mvidTableNode = new ManifestAssemblyMvidHeaderNode(ManifestMetadataTable);
+            Header.Add(Internal.Runtime.ReadyToRunSectionType.ManifestAssemblyMvids, mvidTableNode, mvidTableNode);
+
             AssemblyTableNode assemblyTable = null;
 
             if (CompilationModuleGroup.IsCompositeBuildMode)

src/coreclr/tools/aot/ILCompiler.ReadyToRun/ILCompiler.ReadyToRun.csproj

@@ -103,7 +103,7 @@
     <Compile Include="..\..\Common\TypeSystem\Interop\InteropTypes.cs" Link="Interop\InteropTypes.cs" />
     <Compile Include="Compiler\AssemblyExtensions.cs" />
     <Compile Include="Compiler\DependencyAnalysis\ReadyToRun\DeferredTillPhaseNode.cs" />
-    <Compile Include="Compiler\DependencyAnalysis\ReadyToRun\ComponentAssemblyMvidHeaderNode.cs" />
+    <Compile Include="Compiler\DependencyAnalysis\ReadyToRun\ManifestAssemblyMvidHeaderNode.cs" />
     <Compile Include="Compiler\DependencyAnalysis\ReadyToRun\DelayLoadMethodCallThunkNodeRange.cs" />
     <Compile Include="Compiler\CallChainProfile.cs" />
     <Compile Include="ObjectWriter\MapFileBuilder.cs" />

src/coreclr/tools/r2rdump/TextDumper.cs

@@ -18,6 +18,8 @@ namespace R2RDump
 {
     class TextDumper : Dumper
     {
+        private const int GuidByteSize = 16;
+
         public TextDumper(ReadyToRunReader r2r, TextWriter writer, Disassembler disassembler, DumpOptions options)
             : base(r2r, writer, disassembler, options)
         {
@@ -88,7 +90,14 @@ internal override void DumpHeader(bool dumpSections)
                     int assemblyIndex = 0;
                     foreach (string assemblyName in _r2r.ManifestReferenceAssemblies.OrderBy(kvp => kvp.Value).Select(kvp => kvp.Key))
                     {
-                        WriteDivider($@"Component Assembly [{assemblyIndex}]: {assemblyName}");
+                        string dividerName = $@"Component Assembly [{assemblyIndex}]: {assemblyName}";
+                        if (_r2r.ReadyToRunHeader.Sections.TryGetValue(ReadyToRunSectionType.ManifestAssemblyMvids, out ReadyToRunSection mvidSection))
+                        {
+                            int mvidOffset = _r2r.GetOffset(mvidSection.RelativeVirtualAddress) + GuidByteSize * assemblyIndex;
+                            Guid mvid = new Guid(new ReadOnlySpan<byte>(_r2r.Image, mvidOffset, GuidByteSize));
+                            dividerName += $@" - MVID {mvid}";
+                        }
+                        WriteDivider(dividerName);
                         ReadyToRunCoreHeader assemblyHeader = _r2r.ReadyToRunAssemblyHeaders[assemblyIndex];
                         foreach (ReadyToRunSection section in NormalizedSections(assemblyHeader))
                         {
@@ -500,6 +509,18 @@ internal override void DumpSectionContents(ReadyToRunSection section)
                     string ownerCompositeExecutable = Encoding.UTF8.GetString(_r2r.Image, oceOffset, section.Size - 1); // exclude the zero terminator
                     _writer.WriteLine("Composite executable: {0}", ownerCompositeExecutable.ToEscapedString());
                     break;
+                case ReadyToRunSectionType.ManifestAssemblyMvids:
+                    int mvidOffset = _r2r.GetOffset(section.RelativeVirtualAddress);
+                    int mvidCount = section.Size / GuidByteSize;
+                    for (int mvidIndex = 0; mvidIndex < mvidCount; mvidIndex++)
+                    {
+                        Guid mvid = new Guid(new Span<byte>(_r2r.Image, mvidOffset + GuidByteSize * mvidIndex, GuidByteSize));
+                        _writer.WriteLine("MVID[{0}] = {1}", mvidIndex, mvid);
+                    }
+                    break;
+                default:
+                    _writer.WriteLine("Unsupported section type {0}", section.Type);
+                    break;
             }
         }
 

src/coreclr/vm/appdomain.cpp

@@ -3043,13 +3043,15 @@ DomainAssembly *AppDomain::LoadDomainAssemblyInternal(AssemblySpec* pIdentity,
 
         // Find the list lock entry
         FileLoadLock * fileLock = (FileLoadLock *)lock->FindFileLock(pFile);
+        bool registerNewAssembly = false;
         if (fileLock == NULL)
         {
             // Check again in case we were racing
             result = FindAssembly(pFile, FindAssemblyOptions_IncludeFailedToLoad);
             if (result == NULL)
             {
                 // We are the first one in - create the DomainAssembly
+                registerNewAssembly = true;
                 fileLock = FileLoadLock::Create(lock, pFile, pDomainAssembly);
                 pDomainAssembly.SuppressRelease();
 #ifndef CROSSGEN_COMPILE
@@ -3082,6 +3084,11 @@ DomainAssembly *AppDomain::LoadDomainAssemblyInternal(AssemblySpec* pIdentity,
         {
             result->EnsureLoadLevel(targetLevel);
         }
+
+        if (registerNewAssembly)
+        {
+            pFile->GetAssemblyLoadContext()->AddLoadedAssembly(pDomainAssembly->GetLoadedAssembly());
+        }
     }
     else
         result->EnsureLoadLevel(targetLevel);

src/coreclr/vm/nativeimage.cpp

@@ -67,7 +67,7 @@ void NativeImage::Initialize(READYTORUN_HEADER *pHeader, LoaderAllocator *pLoade
 
     m_pReadyToRunInfo = new ReadyToRunInfo(/*pModule*/ NULL, pLoaderAllocator, m_pImageLayout, pHeader, /*compositeImage*/ NULL, pamTracker);
     m_pComponentAssemblies = m_pReadyToRunInfo->FindSection(ReadyToRunSectionType::ComponentAssemblies);
-    m_pComponentAssemblyMvids = m_pReadyToRunInfo->FindSection(ReadyToRunSectionType::ComponentAssemblyMvids);
+    m_pComponentAssemblyMvids = m_pReadyToRunInfo->FindSection(ReadyToRunSectionType::ManifestAssemblyMvids);
     m_componentAssemblyCount = m_pComponentAssemblies->Size / sizeof(READYTORUN_COMPONENT_ASSEMBLIES_ENTRY);
 
     // Check if the current module's image has native manifest metadata, otherwise the current->GetNativeAssemblyImport() asserts.