[clr-interp] Runtime async support

src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs

@@ -6,6 +6,7 @@
 using System.Diagnostics.CodeAnalysis;
 using System.Numerics;
 using System.Reflection;
+using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
 using System.Runtime.Serialization;
 using System.Runtime.Versioning;
@@ -144,6 +145,18 @@ public ref byte GetResultStorageOrNull()
 
     public static partial class AsyncHelpers
     {
+#if FEATURE_INTERPRETER
+        [LibraryImport(RuntimeHelpers.QCall, EntryPoint = "AsyncHelpers_ResumeInterpreterContinuation")]
+        private static partial void AsyncHelpers_ResumeInterpreterContinuation(ObjectHandleOnStack cont, ref byte resultStorage);
+
+        internal static Continuation? ResumeInterpreterContinuation(Continuation cont, ref byte resultStorage)
+        {
+            ObjectHandleOnStack contHandle = ObjectHandleOnStack.Create(ref cont);
+            AsyncHelpers_ResumeInterpreterContinuation(contHandle, ref resultStorage);
+            return cont;
+        }
+#endif
+
         // This is the "magic" method on which other "Await" methods are built.
         // Calling this from an Async method returns the continuation to the caller thus
         // explicitly initiates suspension.

src/coreclr/inc/clrconfigvalues.h

@@ -714,7 +714,7 @@ RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableRiscV64Zbb,             W("EnableRiscV64
 #endif
 
 // Runtime-async
-RETAIL_CONFIG_DWORD_INFO(UNSUPPORTED_RuntimeAsync, W("RuntimeAsync"), 0, "Enables runtime async method support")
+RETAIL_CONFIG_DWORD_INFO(UNSUPPORTED_RuntimeAsync, W("RuntimeAsync"), 1, "Enables runtime async method support")
 
 ///
 /// Uncategorized

src/coreclr/interpreter/compiler.h

@@ -342,6 +342,9 @@ struct InterpBasicBlock
     // Number of catch, filter or finally clauses that overlap with this basic block.
     int32_t overlappingEHClauseCount;
 
+    // Number of try blocks that enclose this basic block.
+    int32_t enclosingTryBlockCount;
+
     InterpBasicBlock(int32_t index) : InterpBasicBlock(index, 0) { }
 
     InterpBasicBlock(int32_t index, int32_t ilOffset)
@@ -367,6 +370,7 @@ struct InterpBasicBlock
         isFinallyCallIsland = false;
         clauseVarIndex = -1;
         overlappingEHClauseCount = 0;
+        enclosingTryBlockCount = -1;
     }
 };
 
@@ -541,6 +545,7 @@ class InterpCompiler
     friend class InterpIAllocator;
     friend class InterpGcSlotAllocator;
     friend class InterpILOpcodePeeps;
+    friend class InterpAsyncCallPeeps;
 
 private:
     CORINFO_METHOD_HANDLE m_methodHnd;
@@ -609,6 +614,8 @@ class InterpCompiler
     // from the interpreter code header during execution.
     TArray<void*, MemPoolAllocator> m_dataItems;
 
+    TArray<InterpAsyncSuspendData*, MemPoolAllocator> m_asyncSuspendDataItems;
+
     InterpDataItemIndexMap m_genericLookupToDataItemIndex;
     int32_t GetDataItemIndex(void* data)
     {
@@ -759,13 +766,22 @@ class InterpCompiler
     int32_t m_paramArgIndex = -1; // Index of the type parameter argument in the m_pVars array.
     // For each catch or filter clause, we create a variable that holds the exception object.
     // This is the index of the first such variable.
+    int32_t m_continuationArgIndex = -1; // Index of the continuation argument in the m_pVars array for async methods.
     int32_t m_clauseVarsIndex = 0;
+
+    int32_t m_synchronizedOrAsyncPostFinallyOffset = -1; // If the method is synchronized/async, this is the offset of the instruction after the finally which does the actual return
 
     bool m_isSynchronized = false;
     int32_t m_synchronizedFlagVarIndex = -1; // If the method is synchronized, this is the index of the argument that flag indicating if the lock was taken
-    int32_t m_synchronizedRetValVarIndex = -1; // If the method is synchronized, ret instructions are replaced with a store to this var and a leave to an epilog instruction.
+    int32_t m_synchronizedOrAsyncRetValVarIndex = -1; // If the method is synchronized, ret instructions are replaced with a store to this var and a leave to an epilog instruction.
     int32_t m_synchronizedFinallyStartOffset = -1; // If the method is synchronized, this is the offset of the start of the finally epilog
-    int32_t m_synchronizedPostFinallyOffset = -1; // If the method is synchronized, this is the offset of the instruction after the finally which does the actual return
+
+    int32_t m_execContextVarIndex = -1; // If the method is async, this is the var index of the ExecutionContext local
+    int32_t m_syncContextVarIndex = -1; // If the method is async, this is the var index of the SynchronizationContext local
+
+    void *m_asyncResumeFuncPtr = NULL;
+    bool m_isAsyncMethodWithContextSaveRestore = false;
+    int32_t m_asyncFinallyStartOffset = -1; // If the method is async, this is the offset of the start of the fault handler
 
     bool m_shadowCopyOfThisPointerActuallyNeeded = false;
     bool m_shadowCopyOfThisPointerHasVar = false;
@@ -802,6 +818,8 @@ class InterpCompiler
     void ConvertFloatingPointStackEntryToStackType(StackInfo* entry, StackType type);
 
     // Opcode peeps
+    bool    FindAndApplyPeep(OpcodePeep* Peeps[]);
+
     bool    IsStoreLoadPeep(const uint8_t* ip, OpcodePeepElement* peep, void** computedInfo);
     void    ApplyStoreLoadPeep(const uint8_t* ip, OpcodePeepElement* peep, void* computedInfo);
 
@@ -814,6 +832,21 @@ class InterpCompiler
     bool    IsTypeValueTypePeep(const uint8_t* ip, OpcodePeepElement* peep, void** outComputedInfo);
     void    ApplyTypeValueTypePeep(const uint8_t* ip, OpcodePeepElement* peep, void* computedInfo);
 
+    enum class ContinuationContextHandling : uint8_t
+    {
+        ContinueOnCapturedContext,
+        ContinueOnThreadPool,
+        None
+    };
+    bool    IsRuntimeAsyncCall(const uint8_t* ip, OpcodePeepElement* peep, void** computedInfo);
+    bool    IsRuntimeAsyncCallConfigureAwaitTask(const uint8_t* ip, OpcodePeepElement* peep, void** computedInfo);
+    bool    IsRuntimeAsyncCallConfigureAwaitValueTask(const uint8_t* ip, OpcodePeepElement* peep, void** computedInfo);
+    bool    IsRuntimeAsyncCallConfigureAwaitValueTaskExactStLoc(const uint8_t* ip, OpcodePeepElement* peep, void** computedInfo);
+
+    void    ApplyRuntimeAsyncCall(const uint8_t* ip, OpcodePeepElement* peep, void* computedInfo) {}
+    ContinuationContextHandling m_currentContinuationContextHandling = ContinuationContextHandling::None;
+    CORINFO_RESOLVED_TOKEN m_resolvedAsyncCallToken;
+
     // Code emit
     void    EmitConv(StackInfo *sp, StackType type, InterpOpcode convOp);
     void    EmitLoadVar(int var);
@@ -823,6 +856,7 @@ class InterpCompiler
     void    EmitShiftOp(int32_t opBase);
     void    EmitCompareOp(int32_t opBase);
     void    EmitCall(CORINFO_RESOLVED_TOKEN* pConstrainedToken, bool readonly, bool tailcall, bool newObj, bool isCalli);
+    void    EmitSuspend(const CORINFO_CALL_INFO &callInfo, ContinuationContextHandling ContinuationContextHandling, InterpBasicBlock* pBB);
     void    EmitCalli(bool isTailCall, void* calliCookie, int callIFunctionPointerVar, CORINFO_SIG_INFO* callSiteSig);
     bool    EmitNamedIntrinsicCall(NamedIntrinsic ni, bool nonVirtualCall, CORINFO_CLASS_HANDLE clsHnd, CORINFO_METHOD_HANDLE method, CORINFO_SIG_INFO sig);
     void    EmitLdind(InterpType type, CORINFO_CLASS_HANDLE clsHnd, int32_t offset);
@@ -852,6 +886,15 @@ class InterpCompiler
     void    EndActiveCall(InterpInst *call);
     void    CompactActiveVars(int32_t *current_offset);
 
+    TArray<InterpIntervalMapEntry**, MemPoolAllocator> m_varIntervalMaps;
+    InterpIntervalMapEntry ComputeNextIntervalMapEntry_ForVars(const TArray<int32_t, MemPoolAllocator> &vars, int32_t *pNextIndex);
+    void AllocateIntervalMapData_ForVars(InterpIntervalMapEntry** ppIntervalMap, const TArray<int32_t, MemPoolAllocator> &vars);
+
+    void GetVarSizeAndOffset(const InterpIntervalMapEntry* pVarIntervalMap, int32_t entryIndex, int32_t internalIndex, uint32_t* pVarSize, uint32_t* pVarOffset);
+    InterpIntervalMapEntry ComputeNextIntervalMapEntry_ForOffsets(const InterpIntervalMapEntry* pVarIntervalMap, int32_t *pNextIndex, int32_t *pInternalIndex);
+    void ConvertToIntervalMapData_ForOffsets(InterpIntervalMapEntry** ppIntervalMap);
+    void UpdateLocalIntervalMaps();
+
     // Passes
     int32_t* m_pMethodCode;
     int32_t m_methodCodeSize; // code size measured in int32_t slots, instead of bytes
@@ -884,6 +927,7 @@ class InterpCompiler
     void PrintInsData(InterpInst *ins, int32_t offset, const int32_t *pData, int32_t opcode);
     void PrintCompiledCode();
     void PrintCompiledIns(const int32_t *ip, const int32_t *start);
+    void PrintInterpAsyncSuspendData(InterpAsyncSuspendData* pSuspendInfo);
 #ifdef DEBUG
     InterpDumpScope m_dumpScope;
     TArray<char, MallocAllocator> m_methodName;
@@ -910,6 +954,7 @@ class InterpCompiler
     InterpMethod* CompileMethod();
     void BuildGCInfo(InterpMethod *pInterpMethod);
     void BuildEHInfo();
+    void UpdateWithFinalMethodByteCodeAddress(InterpByteCodeStart *pByteCodeStart);
 
     int32_t* GetCode(int32_t *pCodeSize);
 };

src/coreclr/interpreter/compileropt.cpp

@@ -453,4 +453,6 @@ void InterpCompiler::AllocOffsets()
 
     m_globalVarsWithRefsStackTop = globalVarsWithRefsStackTop;
     m_totalVarsStackSize = ALIGN_UP_TO(finalVarsStackSize, INTERP_STACK_ALIGNMENT);
+
+    UpdateLocalIntervalMaps();
 }

src/coreclr/interpreter/eeinterp.cpp

@@ -126,6 +126,7 @@ CorJitResult CILInterp::compileMethod(ICorJitInfo*         compHnd,
         *(InterpMethod**)args.hotCodeBlockRW = pMethod;
         memcpy ((uint8_t*)args.hotCodeBlockRW + sizeof(InterpMethod*), pIRCode, IRCodeSize * sizeof(int32_t));
 
+        compiler.UpdateWithFinalMethodByteCodeAddress((InterpByteCodeStart*)args.hotCodeBlock);
         *entryAddress = (uint8_t*)args.hotCodeBlock;
         *nativeSizeOfCode = sizeOfCode;
 

src/coreclr/interpreter/inc/interpretershared.h

@@ -24,6 +24,7 @@ struct InterpHelperData {
 
 #ifndef INTERPRETER_COMPILER_INTERNAL
 class MethodDesc;
+class MethodTable;
 #endif
 
 struct CallStubHeader;
@@ -188,4 +189,53 @@ enum class CalliFlags : int32_t
     PInvoke = 1 << 2, // The call is a PInvoke call
 };
 
+struct InterpIntervalMapEntry
+{
+    uint32_t startOffset;
+    uint32_t countBytes; // If count is 0 then this is the end marker.
+};
+
+struct InterpAsyncSuspendData
+{
+// ResumeInfo . Keep in sync with dataAsyncResumeInfo in the JIT and System.Runtime.CompilerServices.ResumeInfo
+    void* resumeFuncPtr; // Pointer to the resume function
+    void* DiagnosticIP; // IP to report in diagnostic scenarios
+
+#ifdef INTERPRETER_COMPILER_INTERNAL
+    CORINFO_CLASS_HANDLE ContinuationTypeHnd;
+#else
+    DPTR(MethodTable) ContinuationTypeHnd;
+#endif
+
+    InterpIntervalMapEntry* zeroedLocalsIntervals; // This will be used for the locals we need to keep live.
+    InterpIntervalMapEntry* liveLocalsIntervals; // Following the end of this struct is the array of InterpIntervalMapEntry for live locals
+    CorInfoContinuationFlags flags;
+    int32_t offsetIntoContinuationTypeForExecutionContext;
+    int32_t keepAliveOffset; // Only needed if we have a generic context to keep alive
+    InterpByteCodeStart* methodStartIP;
+#ifdef INTERPRETER_COMPILER_INTERNAL
+    CORINFO_CLASS_HANDLE asyncMethodReturnType;
+#else
+    DPTR(MethodTable) asyncMethodReturnType;
+#endif
+    int32_t asyncMethodReturnTypePrimitiveSize; // 0 if not primitive, otherwise size in bytes
+    int32_t continuationArgOffset;
+
+#ifdef INTERPRETER_COMPILER_INTERNAL
+    CORINFO_METHOD_HANDLE pCaptureSyncContextMethod;
+#else
+    DPTR(MethodDesc) pCaptureSyncContextMethod;
+#endif
+#ifdef INTERPRETER_COMPILER_INTERNAL
+    CORINFO_METHOD_HANDLE pRestoreExecutionContextMethod;
+#else
+    DPTR(MethodDesc) pRestoreExecutionContextMethod;
+#endif
+#ifdef INTERPRETER_COMPILER_INTERNAL
+    CORINFO_METHOD_HANDLE pRestoreContextsMethod;
+#else
+    DPTR(MethodDesc) pRestoreContextsMethod;
+#endif
+};
+
 #endif

src/coreclr/interpreter/inc/intops.def

@@ -309,6 +309,8 @@ OPDEF(INTOP_SHL_I8, "shl.i8", 4, 1, 2, InterpOpNoArgs)
 OPDEF(INTOP_SHR_I4, "shr.i4", 4, 1, 2, InterpOpNoArgs)
 OPDEF(INTOP_SHR_I8, "shr.i8", 4, 1, 2, InterpOpNoArgs)
 
+OPDEF(INTOP_CZERO_I, "czero.i", 4, 1, 1, InterpOpNoArgs) // Set dvar to 1 if operand is non-zero otherwise 0 (produces a 32bit int)
+
 OPDEF(INTOP_CEQ_I4, "ceq.i4", 4, 1, 2, InterpOpNoArgs)
 OPDEF(INTOP_CEQ_I8, "ceq.i8", 4, 1, 2, InterpOpNoArgs)
 OPDEF(INTOP_CEQ_R4, "ceq.r4", 4, 1, 2, InterpOpNoArgs)
@@ -427,6 +429,17 @@ OPDEF(INTOP_THROW_PNSE, "throw.pnse", 1, 0, 0, InterpOpNoArgs)
 
 OPDEF(INTOP_LOAD_FRAMEVAR, "load.framevar", 2, 1, 0, InterpOpNoArgs)
 
+OPDEF(INTOP_SET_CONTINUATION_NULL, "set.continuation.null", 1, 0, 0, InterpOpNoArgs)
+OPDEF(INTOP_SET_CONTINUATION, "set.continuation", 2, 0, 1, InterpOpNoArgs)
+OPDEF(INTOP_GET_CONTINUATION, "get.continuation", 2, 1, 0, InterpOpNoArgs)
+OPDEF(INTOP_HANDLE_CONTINUATION, "handle.continuation", 4, 1, 0, InterpOpHandleContinuation)
+OPDEF(INTOP_HANDLE_CONTINUATION_GENERIC, "handle.continuation.generic", 5, 1, 1, InterpOpHandleContinuation)
+OPDEF(INTOP_HANDLE_CONTINUATION_SUSPEND, "handle.continuation.suspend", 3, 0, 1, InterpOpHandleContinuationPt2)
+OPDEF(INTOP_HANDLE_CONTINUATION_RESUME, "handle.continuation.resume", 2, 0, 0, InterpOpHandleContinuationPt2)
+OPDEF(INTOP_CHECK_FOR_CONTINUATION, "check.for.continuation", 3, 0, 1, InterpOpNoArgs)
+OPDEF(INTOP_CAPTURE_CONTEXT_ON_SUSPEND, "capture.context.on.suspend", 4, 1, 1, InterpOpHandleContinuationPt2)
+OPDEF(INTOP_RESTORE_CONTEXTS_ON_SUSPEND, "restore.contexts.on.suspend", 6, 1, 3, InterpOpHandleContinuationPt2)
+
 // Intrinsics
 OPDEF(INTOP_COMPARE_EXCHANGE_U1, "compare.exchange.u1", 5, 1, 3, InterpOpNoArgs)
 OPDEF(INTOP_COMPARE_EXCHANGE_U2, "compare.exchange.u2", 5, 1, 3, InterpOpNoArgs)

src/coreclr/interpreter/intops.h

@@ -30,6 +30,8 @@ typedef enum
     InterpOpPointerHelperFtn,
     InterpOpPointerInt,
     InterpOpGenericLookupInt,
+    InterpOpHandleContinuation,
+    InterpOpHandleContinuationPt2,
 } InterpOpArgType;
 
 extern const uint8_t g_interpOpLen[];
@@ -127,6 +129,7 @@ inline double getR8LittleEndian(const uint8_t* ptr)
 // We use a couple of special "intrinsic" tokens to represent these operations.
 // These are recognized by our implementation of the CALL opcode.
 const uint32_t INTERP_CALL_SYNCHRONIZED_MONITOR_EXIT = 0xFFFFFFFE;
-const uint32_t INTERP_LOAD_RETURN_VALUE_FOR_SYNCHRONIZED = 0xFFFFFFFF;
+const uint32_t INTERP_LOAD_RETURN_VALUE_FOR_SYNCHRONIZED_OR_ASYNC = 0xFFFFFFFF;
+const uint32_t INTERP_RESTORE_CONTEXTS_FOR_ASYNC_METHOD = 0xFFFFFFFD;
 
 #endif

src/coreclr/interpreter/intrinsics.cpp

@@ -111,6 +111,15 @@ NamedIntrinsic GetNamedIntrinsic(COMP_HANDLE compHnd, CORINFO_METHOD_HANDLE comp
                 else if (!strcmp(methodName, "GetMethodTable"))
                     return NI_System_Runtime_CompilerServices_RuntimeHelpers_GetMethodTable;
             }
+            else if (!strcmp(className, "AsyncHelpers"))
+            {
+                if (!strcmp(methodName, "AsyncSuspend"))
+                    return NI_System_Runtime_CompilerServices_AsyncHelpers_AsyncSuspend;
+                else if (!strcmp(methodName, "AsyncCallContinuation"))
+                    return NI_System_Runtime_CompilerServices_AsyncHelpers_AsyncCallContinuation;
+                else if (!strcmp(methodName, "Await"))
+                    return NI_System_Runtime_CompilerServices_AsyncHelpers_Await;
+            }
         }
         else if (!strcmp(namespaceName, "System.Runtime.InteropServices"))
         {
@@ -147,6 +156,15 @@ NamedIntrinsic GetNamedIntrinsic(COMP_HANDLE compHnd, CORINFO_METHOD_HANDLE comp
                 return NI_System_Threading_Volatile_WriteBarrier;
         }
     }
+    else if (!strcmp(namespaceName, "System.Threading.Tasks"))
+    {
+        if (!strcmp(methodName, "ConfigureAwait"))
+        {
+            if (!strcmp(className, "Task`1") || !strcmp(className, "Task") ||
+                !strcmp(className, "ValueTask`1") || !strcmp(className, "ValueTask"))
+                return NI_System_Threading_Tasks_Task_ConfigureAwait;
+        }
+    }
 
     return NI_Illegal;
 }

src/coreclr/vm/amd64/AsmHelpers.asm

@@ -587,6 +587,8 @@ HaveInterpThreadContext:
         lea             rax, [rsp + __PWTB_TransitionBlock]
         ; Copy the arguments to the interpreter stack, invoke the InterpExecMethod and load the return value
         call            qword ptr [r11]
+        ; Fill in the ContinuationContext register
+        mov             rcx, [rsp + __PWTB_ArgumentRegisters]
 
         EPILOG_WITH_TRANSITION_BLOCK_RETURN
 
@@ -1114,6 +1116,8 @@ END_PROLOGUE
         mov r11, rcx ; The routines list
         mov r10, rdx ; interpreter stack args
         call qword ptr [r11]
+        mov rdx, [rbp + 48]
+        mov [rdx], rcx
         mov rsp, rbp
         pop rbp
         ret
@@ -1129,6 +1133,8 @@ END_PROLOGUE
         mov r10, rdx ; interpreter stack args
         mov rcx, r8  ; return buffer
         call qword ptr [r11]
+        mov rdx, [rbp + 48]
+        mov [rdx], rcx
         mov rsp, rbp
         pop rbp
         ret
@@ -1144,6 +1150,8 @@ END_PROLOGUE
         mov r10, rdx ; interpreter stack args
         mov rdx, r8  ; return buffer
         call qword ptr [r11]
+        mov rdx, [rbp + 48]
+        mov [rdx], rcx
         mov rsp, rbp
         pop rbp
         ret
@@ -1161,6 +1169,8 @@ END_PROLOGUE
         mov r11, rcx ; The routines list
         mov r10, rdx ; interpreter stack args
         call qword ptr [r11]
+        mov rdx, [rbp + 48]
+        mov [rdx], rcx
         mov r8, [rbp - 8]
         movsd real8 ptr [r8], xmm0
         mov rsp, rbp
@@ -1179,6 +1189,8 @@ END_PROLOGUE
         mov r11, rcx ; The routines list
         mov r10, rdx ; interpreter stack args
         call qword ptr [r11]
+        mov rdx, [rbp + 48]
+        mov [rdx], rcx
         mov r8, [rbp - 8]
         mov qword ptr [r8], rax
         mov rsp, rbp

src/coreclr/vm/amd64/asmconstants.h

@@ -582,7 +582,7 @@ ASMCONSTANTS_C_ASSERT(OFFSETOF__Thread__m_pInterpThreadContext == offsetof(Threa
 #define OFFSETOF__InterpThreadContext__pStackPointer 0x10
 ASMCONSTANTS_C_ASSERT(OFFSETOF__InterpThreadContext__pStackPointer == offsetof(InterpThreadContext, pStackPointer))
 
-#define OFFSETOF__CallStubHeader__Routines 0x10
+#define OFFSETOF__CallStubHeader__Routines 0x18
 ASMCONSTANTS_C_ASSERT(OFFSETOF__CallStubHeader__Routines == offsetof(CallStubHeader, Routines))
 
 #ifdef TARGET_UNIX