[cDAC] IXCLRDataMethodInstance::GetILOffsetsByAddress

docs/design/datacontracts/DebugInfo.md

@@ -0,0 +1,219 @@
+# Contract DebugInfo
+
+This contract is for fetching information related to DebugInfo associated with native code.
+
+## APIs of contract
+
+```csharp
+[Flags]
+public enum SourceTypes : uint
+{
+    SourceTypeInvalid = 0x00, // To indicate that nothing else applies
+    StackEmpty = 0x01, // The stack is empty here
+    CallInstruction = 0x02  // The actual instruction of a call.
+}
+```
+
+```csharp
+public readonly struct OffsetMapping
+{
+    public uint NativeOffset { get; init; }
+    public uint ILOffset { get; init; }
+    public SourceTypes SourceType { get; init; }
+}
+```
+
+```csharp
+// Given a code pointer, return the associated native/IL offset mapping and codeOffset.
+// If preferUninstrumented, will always read the uninstrumented bounds.
+// Otherwise will read the instrumented bounds and fallback to the uninstrumented bounds.
+IEnumerable<OffsetMapping> GetMethodNativeMap(TargetCodePointer pCode, bool preferUninstrumented, out uint codeOffset);
+```
+
+## Version 1
+
+Data descriptors used:
+| Data Descriptor Name | Field | Meaning |
+| --- | --- | --- |
+| `PatchpointInfo` | `LocalCount` | Number of locals in the method associated with the patchpoint. |
+
+Contracts used:
+| Contract Name |
+| --- |
+| `CodeVersions` |
+| `ExecutionManager` |
+
+Constants:
+| Constant Name | Meaning | Value |
+| --- | --- | --- |
+| IL_OFFSET_BIAS | IL offsets are encoded in the DebugInfo with this bias. | `0xfffffffd` (-3) |
+| DEBUG_INFO_BOUNDS_HAS_INSTRUMENTED_BOUNDS | Indicates bounds data contains instrumented bounds | `0xFFFFFFFF` |
+| EXTRA_DEBUG_INFO_PATCHPOINT | Indicates debug info contains patchpoint information | 0x1 |
+| EXTRA_DEBUG_INFO_RICH | Indicates debug info contains rich information | 0x2 |
+
+### DebugInfo Stream Encoding
+
+The DebugInfo stream is encoded using variable length 32-bit values with the following scheme:
+
+A value can be stored using one or more nibbles (a nibble is a 4-bit value). 3 bits of a nibble are used to store 3 bits of the value, and the top bit indicates if  the following nibble contains rest of the value. If the top bit is not set, then this nibble is the last part of the value. The higher bits of the value are written out first, and the lowest 3 bits are written out last.
+
+In the encoded stream of bytes, the lower nibble of a byte is used before the high nibble.
+
+A binary value ABCDEFGHI (where A is the highest bit) is encoded as
+the follow two bytes : 1DEF1ABC XXXX0GHI
+
+Examples:
+| Decimal Value | Hex Value | Encoded Result |
+| --- | --- | --- |
+| 0 | 0x0 | X0 |
+| 1 | 0x1 | X1 |
+| 7 | 0x7 | X7 |
+| 8 | 0x8 | 09 |
+| 9 | 0x9 | 19 |
+| 63 | 0x3F | 7F |
+| 64 | 0x40 | F9 X0 |
+| 65 | 0x41 | F9 X1 |
+| 511 | 0x1FF | FF X7 |
+| 512 | 0x200 | 89 08 |
+| 513 | 0x201 | 89 18 |
+
+Based on the encoding specification, we use a decoder defined originally for r2r dump `NibbleReader.cs`
+
+### Bounds Data Encoding (R2R Major Version 16+)
+
+For R2R major version 16 and above, the bounds data uses a bit-packed encoding algorithm:
+
+1. The bounds entry count, bits needed for native deltas, and bits needed for IL offsets are encoded using the nibble scheme above
+2. Each bounds entry is then bit-packed with:
+   - 2 bits for source type (SourceTypeInvalid=0, CallInstruction=1, StackEmpty=2, StackEmpty|CallInstruction=3)
+   - Variable bits for native offset delta (accumulated from previous offset)
+   - Variable bits for IL offset (with IL_OFFSET_BIAS applied)
+
+The bit-packed data is read byte by byte, collecting bits until enough are available for each entry.
+
+### Implementation
+
+``` csharp
+IEnumerable<OffsetMapping> IDebugInfo.GetMethodNativeMap(TargetCodePointer pCode, bool preferUninstrumented, out uint codeOffset)
+{
+    // Get the method's DebugInfo
+    if (_eman.GetCodeBlockHandle(pCode) is not CodeBlockHandle cbh)
+        throw new InvalidOperationException($"No CodeBlockHandle found for native code {pCode}.");
+    TargetPointer debugInfo = _eman.GetDebugInfo(cbh, out bool hasFlagByte);
+
+    TargetCodePointer nativeCodeStart = _eman.GetStartAddress(cbh);
+    codeOffset = (uint)(CodePointerUtils.AddressFromCodePointer(pCode, _target) - CodePointerUtils.AddressFromCodePointer(nativeCodeStart, _target));
+
+    return RestoreBoundaries(debugInfo, hasFlagByte, preferUninstrumented);
+}
+
+private IEnumerable<OffsetMapping> RestoreBoundaries(TargetPointer debugInfo, bool hasFlagByte, bool preferUninstrumented)
+{
+    if (hasFlagByte)
+    {
+        // Check flag byte and skip over any patchpoint info
+        byte flagByte = _target.Read<byte>(debugInfo++);
+
+        if ((flagByte & EXTRA_DEBUG_INFO_PATCHPOINT) != 0)
+        {
+            uint localCount = _target.Read<uint>(debugInfo + /*PatchpointInfo::LocalCount offset*/)
+            debugInfo += /*size of PatchpointInfo*/ + (localCount * 4);
+        }
+
+        if ((flagByte & EXTRA_DEBUG_INFO_RICH) != 0)
+        {
+            uint richDebugInfoSize = _target.Read<uint>(debugInfo);
+            debugInfo += 4;
+            debugInfo += richDebugInfoSize;
+        }
+    }
+
+    NativeReader nibbleNativeReader = new(new TargetStream(_target, debugInfo, 24 /*maximum size of 4 32bit ints compressed*/), _target.IsLittleEndian);
+    NibbleReader nibbleReader = new(nibbleNativeReader, 0);
+
+    uint cbBounds = nibbleReader.ReadUInt();
+    uint cbUninstrumentedBounds = 0;
+    if (cbBounds == DEBUG_INFO_BOUNDS_HAS_INSTRUMENTED_BOUNDS)
+    {
+        // This means we have instrumented bounds.
+        cbBounds = nibbleReader.ReadUInt();
+        cbUninstrumentedBounds = nibbleReader.ReadUInt();
+    }
+    uint _ /*cbVars*/ = nibbleReader.ReadUInt();
+
+    TargetPointer addrBounds = debugInfo + (uint)nibbleReader.GetNextByteOffset();
+    // TargetPointer addrVars = addrBounds + cbBounds + cbUninstrumentedBounds;
+
+    if (preferUninstrumented && cbUninstrumentedBounds != 0)
+    {
+        // If we have uninstrumented bounds, we will use them instead of the regular bounds.
+        addrBounds += cbBounds;
+        cbBounds = cbUninstrumentedBounds;
+    }
+
+    if (cbBounds > 0)
+    {
+        NativeReader boundsNativeReader = new(new TargetStream(_target, addrBounds, cbBounds), _target.IsLittleEndian);
+        return DoBounds(boundsNativeReader);
+    }
+
+    return Enumerable.Empty<OffsetMapping>();
+}
+
+private static IEnumerable<OffsetMapping> DoBounds(NativeReader nativeReader)
+{
+    NibbleReader reader = new(nativeReader, 0);
+
+    uint boundsEntryCount = reader.ReadUInt();
+
+    uint bitsForNativeDelta = reader.ReadUInt() + 1; // Number of bits needed for native deltas
+    uint bitsForILOffsets = reader.ReadUInt() + 1; // Number of bits needed for IL offsets
+
+    uint bitsPerEntry = bitsForNativeDelta + bitsForILOffsets + 2; // 2 bits for source type
+    ulong bitsMeaningfulMask = (1UL << ((int)bitsPerEntry)) - 1;
+    int offsetOfActualBoundsData = reader.GetNextByteOffset();
+
+    uint bitsCollected = 0;
+    ulong bitTemp = 0;
+    uint curBoundsProcessed = 0;
+
+    uint previousNativeOffset = 0;
+
+    while (curBoundsProcessed < boundsEntryCount)
+    {
+        bitTemp |= ((uint)nativeReader[offsetOfActualBoundsData++]) << (int)bitsCollected;
+        bitsCollected += 8;
+        while (bitsCollected >= bitsPerEntry)
+        {
+            ulong mappingDataEncoded = bitsMeaningfulMask & bitTemp;
+            bitTemp >>= (int)bitsPerEntry;
+            bitsCollected -= bitsPerEntry;
+
+            SourceTypes sourceType = (mappingDataEncoded & 0x3) switch
+            {
+                0 => SourceTypes.SourceTypeInvalid,
+                1 => SourceTypes.CallInstruction,
+                2 => SourceTypes.StackEmpty,
+                3 => SourceTypes.StackEmpty | SourceTypes.CallInstruction,
+                _ => throw new InvalidOperationException($"Unknown source type encoding: {mappingDataEncoded & 0x3}")
+            };
+
+            mappingDataEncoded >>= 2;
+            uint nativeOffsetDelta = (uint)(mappingDataEncoded & ((1UL << (int)bitsForNativeDelta) - 1));
+            previousNativeOffset += nativeOffsetDelta;
+            uint nativeOffset = previousNativeOffset;
+
+            mappingDataEncoded >>= (int)bitsForNativeDelta;
+            uint ilOffset = (uint)mappingDataEncoded + IL_OFFSET_BIAS;
+
+            yield return new OffsetMapping()
+            {
+                NativeOffset = nativeOffset,
+                ILOffset = ilOffset,
+                SourceType = sourceType
+            };
+            curBoundsProcessed++;
+        }
+    }
+}
+```

docs/design/datacontracts/ExecutionManager.md

@@ -29,6 +29,9 @@ struct CodeBlockHandle
     TargetPointer GetUnwindInfo(CodeBlockHandle codeInfoHandle);
     // Gets the base address the UnwindInfo of codeInfoHandle is relative to
     TargetPointer GetUnwindInfoBaseAddress(CodeBlockHandle codeInfoHandle);
+    // Gets the DebugInfo associated with the code block and specifies if the DebugInfo contains
+    // the flag byte which modifies how DebugInfo is parsed.
+    TargetPointer GetDebugInfo(CodeBlockHandle codeInfoHandle, out bool hasFlagByte);
     // Gets the GCInfo associated with the code block and its version
     // **Currently GetGCInfo only supports X86**
     void GetGCInfo(CodeBlockHandle codeInfoHandle, out TargetPointer gcInfo, out uint gcVersion);
@@ -66,9 +69,11 @@ Data descriptors used:
 | `CodeHeapListNode` | `EndAddress` | End address of the used portion of the code heap |
 | `CodeHeapListNode` | `MapBase` | Start of the map - start address rounded down based on OS page size |
 | `CodeHeapListNode` | `HeaderMap` | Bit array used to find the start of methods - relative to `MapBase` |
+| `EEJitManager` | `StoreRichDebugInfo` | Boolean value determining if debug info associated with the JitManager contains rich info. |
 | `RealCodeHeader` | `MethodDesc` | Pointer to the corresponding `MethodDesc` |
 | `RealCodeHeader` | `NumUnwindInfos` | Number of Unwind Infos |
 | `RealCodeHeader` | `UnwindInfos` | Start address of Unwind Infos |
+| `RealCodeHeader` | `DebugInfo` | Pointer to the DebugInfo |
 | `RealCodeHeader` | `GCInfo` | Pointer to the GCInfo encoding |
 | `Module` | `ReadyToRunInfo` | Pointer to the `ReadyToRunInfo` for the module |
 | `ReadyToRunInfo` | `ReadyToRunHeader` | Pointer to the ReadyToRunHeader |
@@ -78,6 +83,7 @@ Data descriptors used:
 | `ReadyToRunInfo` | `NumHotColdMap` | Number of entries in the `HotColdMap` |
 | `ReadyToRunInfo` | `HotColdMap` | Pointer to an array of 32-bit integers - [see R2R format](../coreclr/botr/readytorun-format.md#readytorunsectiontypehotcoldmap-v80) |
 | `ReadyToRunInfo` | `DelayLoadMethodCallThunks` | Pointer to an `ImageDataDirectory` for the delay load method call thunks |
+| `ReadyToRunInf` | `DebugInfo` | Pointer to an `ImageDataDirectory` for the debug info |
 | `ReadyToRunInfo` | `EntryPointToMethodDescMap` | `HashMap` of entry point addresses to `MethodDesc` pointers |
 | `ReadyToRunHeader` | `MajorVersion` | ReadyToRun major version |
 | `ReadyToRunHeader` | `MinorVersion` | ReadyToRun minor version |
@@ -100,6 +106,7 @@ Global variables used:
 | `HashMapValueMask` | uint64 | Bitmask used when storing values in a `HashMap` |
 | `FeatureEHFunclets` | uint8 | 1 if EH funclets are enabled, 0 otherwise |
 | `GCInfoVersion` | uint32 | JITted code GCInfo version |
+| `FeatureOnStackReplacement` | uint8 | 1 if FEATURE_ON_STACK_REPLACEMENT is enabled, 0 otherwise |
 
 Contracts used:
 | Contract Name |
@@ -266,9 +273,16 @@ The `GetMethodDesc`, `GetStartAddress`, and `GetRelativeOffset` APIs extract fie
 Unwind info (`RUNTIME_FUNCTION`) use relative addressing. For managed code, these values are relative to the start of the code's containing range in the RangeSectionMap (described below). This could be the beginning of a `CodeHeap` for jitted code or the base address of the loaded image for ReadyToRun code.
 `GetUnwindInfoBaseAddress` finds this base address for a given `CodeBlockHandle`.
 
+`IExecutionManager.GetDebugInfo` gets a pointer to the relevant DebugInfo for a `CodeBlockHandle`. The ExecutionManager delegates to the JitManager implementations as the DebugInfo is stored in different ways on jitted and R2R code.
+
+* For Jitted code (`EEJitManager`) a pointer to the `DebugInfo` is stored on the `RealCodeHeader` which is accessed in the same way as `GetMethodInfo` described above. `hasFlagByte` is `true` if either the global `FeatureOnStackReplacement` is `true` or `StoreRichDebugInfo` is `true` on the `EEJitManager`.
+
+* For R2R code (`ReadyToRunJitManager`) the `DebugInfo` is stored as part of the R2R image. The relevant `ReadyToRunInfo` stores a pointer to the an `ImageDataDirectory` representing the `DebugInfo` directory. Read the `VirtualAddress` of this data directory as a `NativeArray` containing the `DebugInfos`. To find the specific `DebugInfo`, index into the array using the `index` of the beginning of the R2R function as found like in `GetMethodInfo` above. This yields an offset `offset` value relative to the image base. Read the first variable length uint at `imageBase + offset`, `lookBack`. If `lookBack != 0`, return `imageBase + offset - lookback`. Otherwise return `offset + size of reading lookback`.
+For R2R images, `hasFlagByte` is always `false`.
+
 `IExecutionManager.GetGCInfo` gets a pointer to the relevant GCInfo for a `CodeBlockHandle`. The ExecutionManager delegates to the JitManager implementations as the GCInfo is stored differently on jitted and R2R code.
 
-* For jitted code (`EEJitManager`) a pointer to the `GCInfo` is stored on the `RealCodeHeader` which is accessed in the same was as `GetMethodInfo` described above. This can simply be returned as is. The `GCInfoVersion` is defined by the runtime global `GCInfoVersion`.
+* For jitted code (`EEJitManager`) a pointer to the `GCInfo` is stored on the `RealCodeHeader` which is accessed in the same way as `GetMethodInfo` described above. This can simply be returned as is. The `GCInfoVersion` is defined by the runtime global `GCInfoVersion`.
 
 * For R2R code (`ReadyToRunJitManager`), the `GCInfo` is stored directly after the `UnwindData`. This in turn is found by looking up the `UnwindInfo` (`RUNTIME_FUNCTION`) and reading the `UnwindData` offset. We find the `UnwindInfo` as described above in `IExecutionManager.GetUnwindInfo`. Once we have the relevant unwind data, we calculate the size of the unwind data and return a pointer to the following byte (first byte of the GCInfo). The size of the unwind data is a platform specific. Currently only X86 is supported with a constant unwind data size of 32-bits.
     * The `GCInfoVersion` of R2R code is mapped from the R2R MajorVersion and MinorVersion which is read from the ReadyToRunHeader which itself is read from the ReadyToRunInfo (can be found as in GetMethodInfo). The current GCInfoVersion mapping is:
@@ -301,6 +315,10 @@ On 64-bit targets, we take advantage of the fact that most architectures don't s
 That is, level 5 has 256 entires pointing to level 4 maps (or nothing if there's no
 code allocated in that address range), level 4 entires point to level 3 maps and so on.  Each level 1 map has 256 entries covering a 128 KiB chunk and pointing to a linked list of range section fragments that fall within that 128 KiB chunk.
 
+### Native Format
+
+The ReadyToRun image stores data in a compressed native foramt defined in [nativeformatreader.h](../../../src/coreclr/vm/nativeformatreader.h).
+
 ### NibbleMap
 
 The ExecutionManager contract depends on a "nibble map" data structure

docs/design/datacontracts/Loader.md

@@ -146,7 +146,6 @@ TargetPointer GetStubHeap(TargetPointer loaderAllocatorPointer);
 | `InstMethodHashTable` | `VolatileEntryNextEntry` | Next pointer in the hash table entry |
 
 
-
 ### Global variables used:
 | Global Name | Type | Purpose |
 | --- | --- | --- |

src/coreclr/inc/patchpointinfo.h

@@ -7,6 +7,10 @@
 
 #include <clrtypes.h>
 
+#ifndef JIT_BUILD
+#include "cdacdata.h"
+#endif // JIT_BUILD
+
 #ifndef _PATCHPOINTINFO_H_
 #define _PATCHPOINTINFO_H_
 
@@ -201,7 +205,19 @@ struct PatchpointInfo
     int32_t      m_securityCookieOffset;
     int32_t      m_monitorAcquiredOffset;
     int32_t      m_offsetAndExposureData[];
+
+#ifndef JIT_BUILD
+    friend struct ::cdac_data<PatchpointInfo>;
+#endif // JIT_BUILD
+};
+
+#ifndef JIT_BUILD
+template<>
+struct cdac_data<PatchpointInfo>
+{
+    static constexpr size_t LocalCount = offsetof(PatchpointInfo, m_numberOfLocals);
 };
+#endif // JIT_BUILD
 
 typedef DPTR(struct PatchpointInfo) PTR_PatchpointInfo;
 

src/coreclr/tools/aot/ILCompiler.Reflection.ReadyToRun/DebugInfo.cs

@@ -158,7 +158,7 @@ private void ParseBounds(NativeReader imageReader, int offset)
                 uint bitsCollected = 0;
                 ulong bitTemp = 0;
                 uint curBoundsProcessed = 0;
-                
+
                 uint previousNativeOffset = 0;
 
                 while (curBoundsProcessed < boundsEntryCount)

src/coreclr/tools/aot/ILCompiler.Reflection.ReadyToRun/NativeArray.cs

@@ -56,7 +56,7 @@ public bool TryGetAt(uint index, ref int pOffset)
             if (index >= _nElements)
                 return false;
 
-            uint offset = 0;
+            uint offset;
             if (_entryIndexSize == 0)
             {
                 int i = (int)(_baseOffset + (index / _blockSize));
@@ -82,7 +82,7 @@ public bool TryGetAt(uint index, ref int pOffset)
                 {
                     if ((val & 2) != 0)
                     {
-                        offset = offset + (val >> 2);
+                        offset += val >> 2;
                         continue;
                     }
                 }

src/coreclr/tools/aot/ILCompiler.Reflection.ReadyToRun/NativeReader.cs

@@ -39,7 +39,7 @@ public void ReadSpanAt(ref int start, Span<byte> buffer)
                 throw new ArgumentOutOfRangeException(nameof(start), "Start index is out of bounds");
 
             _backingStream.Seek(start, SeekOrigin.Begin);
-            _backingStream.Read(buffer);
+            _backingStream.ReadExactly(buffer);
             start += buffer.Length;
         }
 
@@ -382,4 +382,4 @@ public uint DecodeUDelta(ref int start, uint lastValue)
             return lastValue + delta;
         }
     }
-}
+}

src/coreclr/tools/aot/ILCompiler.Reflection.ReadyToRun/NibbleReader.cs

@@ -7,7 +7,7 @@ namespace ILCompiler.Reflection.ReadyToRun
     /// Helper to read memory by 4-bit (half-byte) nibbles as is used for encoding
     /// method fixups. More or less ported over from CoreCLR src\inc\nibblestream.h.
     /// </summary>
-    class NibbleReader
+    internal class NibbleReader
     {
         /// <summary>
         /// Special value in _nextNibble saying there's no next nibble and the next byte

src/coreclr/vm/codeman.h

@@ -2232,8 +2232,17 @@ private :
     HINSTANCE           m_AltJITCompiler;
     bool                m_AltJITRequired;
 #endif //ALLOW_SXS_JIT
+
+    friend struct ::cdac_data<EEJitManager>;
+};
+
+template<>
+struct cdac_data<EEJitManager>
+{
+    static constexpr size_t StoreRichDebugInfo = offsetof(EEJitManager, m_storeRichDebugInfo);
 };
 
+
 //*****************************************************************************
 //
 // This class manages IJitManagers and ICorJitCompilers.  It has only static