Produce crashreport.json and use llvm-symbolizer to create stack trace (

#77578)

* add DOTNET_EnableCrashReport

* Enable JitStress and JitStressRegs by default to see some test failures

* Enable stress switches only for JIT

* fix the build errors

* Revert "Set DNER for all local fields (#77341)"

This reverts commit 4e85471.

* Set EnableCrashReport in testenvironment.proj

* Pass --crashReport to capture for hangs

* Comment bunch of jobs to trigger

* Comment all runs except Linux_x64

* Stop running libraries job

* Fix the tag in testenvironment.proj

* Use keepnativesymbols

* Update JitStress/JitStressRegs

* Embed native_image_offset field in crashreport.json

* Add TryPrintStackTraceFromCrashReport()

* Call TryPrintStackTraceFromCrashReport() even for crashes

* Add newtonsoft in CoreCLRTestLibrary.csproj

* Fix the crash dump folder

* Add random failure in jit

* Add some logging around TryPrintStackTraceFromCrashReport

* Move the error randomization to GetLayout() that has compiler object access

* Seach for crashreport.json files

* Switch to System.Text.Json

* fix the json path

* Add some logging

* Fix the clrjit symbols

* Add a comment

* Add few more commnets

* chown

* ls -l, chmod

* Change to chown

* USER environment variable

* update runtime.yml

* fix runtime.yml

* add logging

* Add llvm-symbolizer -h

* Skip superpmi pipelines

* read contents before process exit

* Add error output also

* fix some parameters for llvm-symbolizer

* Also enable OSX_x64 runs

* Remove the debugging code:

* Revert "Revert "Set DNER for all local fields (#77341)""

This reverts commit 5cfff11.

* Undo JIT changes to trigger assert

* Undo some yml changes

* Review feedback

* Undo runtime.yml changes

* Add some comments

* Address review feedback

eng/pipelines/coreclr/templates/build-job.yml

@@ -189,8 +189,9 @@ jobs:
       displayName: Build and generate native prerequisites
 
     # Build CoreCLR Runtime
+    # TODO: Use --keepnativesymbols only for PRs.
     - ${{ if ne(parameters.osGroup, 'windows') }}:
-      - script: $(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) $(crossArg) $(osArg) -ci $(compilerArg) $(clrRuntimeComponentsBuildArg) $(pgoInstrumentArg) $(officialBuildIdArg) $(clrInterpreterBuildArg) $(clrRuntimePortableBuildArg) $(CoreClrPgoDataArg)
+      - script: $(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) $(crossArg) $(osArg) -ci $(compilerArg) $(clrRuntimeComponentsBuildArg) $(pgoInstrumentArg) $(officialBuildIdArg) $(clrInterpreterBuildArg) $(clrRuntimePortableBuildArg) $(CoreClrPgoDataArg) --keepnativesymbols
         displayName: Build CoreCLR Runtime
     - ${{ if eq(parameters.osGroup, 'windows') }}:
       - script: $(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) -ci $(enforcePgoArg) $(pgoInstrumentArg) $(officialBuildIdArg) $(clrInterpreterBuildArg) $(CoreClrPgoDataArg)

src/coreclr/debug/createdump/crashreportwriter.cpp

@@ -223,6 +223,7 @@ CrashReportWriter::WriteStackFrame(const StackFrame& frame)
     WriteValue64("stack_pointer", frame.StackPointer());
     WriteValue64("native_address", frame.InstructionPointer());
     WriteValue64("native_offset", frame.NativeOffset());
+    WriteValue64("native_image_offset", (frame.InstructionPointer() - frame.ModuleAddress()));
     if (frame.IsManaged())
     {
         WriteValue32("token", frame.Token());

src/tests/Common/Coreclr.TestWrapper/CoreclrTestWrapperLib.cs

@@ -10,6 +10,9 @@
 using System.Linq;
 using System.Runtime.InteropServices;
 using System.Text;
+using System.Text.Json;
+using System.Text.Json.Nodes;
+using System.Text.Json.Serialization;
 using System.Text.RegularExpressions;
 using System.Threading;
 using System.Threading.Tasks;
@@ -204,12 +207,13 @@ public class CoreclrTestWrapperLib
         public const string COLLECT_DUMPS_ENVIRONMENT_VAR = "__CollectDumps";
         public const string CRASH_DUMP_FOLDER_ENVIRONMENT_VAR = "__CrashDumpFolder";
 
-        static bool CollectCrashDump(Process process, string path)
+        static bool CollectCrashDump(Process process, string crashDumpPath, StreamWriter outputWriter)
         {
             string coreRoot = Environment.GetEnvironmentVariable("CORE_ROOT");
             string createdumpPath = Path.Combine(coreRoot, "createdump");
-            string arguments = $"--name \"{path}\" {process.Id} --withheap";
+            string arguments = $"--name \"{crashDumpPath}\" {process.Id} --withheap";
             Process createdump = new Process();
+            bool crashReportPresent = false;
 
             if (OperatingSystem.IsWindows())
             {
@@ -219,8 +223,8 @@ static bool CollectCrashDump(Process process, string path)
             else if (OperatingSystem.IsLinux() || OperatingSystem.IsMacOS())
             {
                 createdump.StartInfo.FileName = "sudo";
-                createdump.StartInfo.Arguments = $"{createdumpPath} " + arguments;
-                createdump.StartInfo.EnvironmentVariables.Add("DOTNET_DbgEnableElfDumpOnMacOS", "1");
+                createdump.StartInfo.Arguments = $"{createdumpPath} --crashreport {arguments}";
+                crashReportPresent = true;
             }
 
             createdump.StartInfo.UseShellExecute = false;
@@ -244,6 +248,11 @@ static bool CollectCrashDump(Process process, string path)
                 Console.WriteLine(output);
                 Console.WriteLine("createdump stderr:");
                 Console.WriteLine(error);
+
+                if (crashReportPresent)
+                {
+                    TryPrintStackTraceFromCrashReport(crashDumpPath + ".crashreport.json", outputWriter);
+                }
             }
             else
             {
@@ -253,6 +262,271 @@ static bool CollectCrashDump(Process process, string path)
             return fSuccess && createdump.ExitCode == 0;
         }
 
+        private static List<string> knownNativeModules = new List<string>() { "libcoreclr.so", "libclrjit.so" };
+        private static string TO_BE_CONTINUE_TAG = "<TO_BE_CONTINUE>";
+        private static string SKIP_LINE_TAG = "# <SKIP_LINE>";
+
+
+        static bool RunProcess(string fileName, string arguments)
+        {
+            Process proc = new Process()
+            {
+                StartInfo = new ProcessStartInfo()
+                {
+                    FileName = fileName,
+                    Arguments = arguments,
+                    UseShellExecute = false,
+                    RedirectStandardOutput = true,
+                    RedirectStandardError = true,
+                }
+            };
+
+            Console.WriteLine($"Invoking: {proc.StartInfo.FileName} {proc.StartInfo.Arguments}");
+            proc.Start();
+
+            Task<string> stdOut = proc.StandardOutput.ReadToEndAsync();
+            Task<string> stdErr = proc.StandardError.ReadToEndAsync();
+            if(!proc.WaitForExit(DEFAULT_TIMEOUT_MS))
+            {
+                proc.Kill(true);
+                Console.WriteLine($"Timedout: '{fileName} {arguments}");
+                return false;
+            }
+
+            Task.WaitAll(stdOut, stdErr);
+            string output = stdOut.Result;
+            string error = stdErr.Result;
+            if (!string.IsNullOrWhiteSpace(output))
+            {
+                Console.WriteLine($"stdout: {output}");
+            }
+            if (!string.IsNullOrWhiteSpace(error))
+            {
+                Console.WriteLine($"stderr: {error}");
+            }
+            return true;
+        }
+
+        /// <summary>
+        ///     Parse crashreport.json file, use llvm-symbolizer to extract symbols
+        ///     and recreate the stacktrace that is printed on the console.
+        /// </summary>
+        /// <param name="crashReportJsonFile">crash dump path</param>
+        /// <param name="outputWriter">Stream for writing logs</param>
+        /// <returns>true, if we can print the stack trace, otherwise false.</returns>
+        static bool TryPrintStackTraceFromCrashReport(string crashReportJsonFile, StreamWriter outputWriter)
+        {
+            if (OperatingSystem.IsLinux() || OperatingSystem.IsMacOS())
+            {
+                if (!RunProcess("sudo", $"ls -l {crashReportJsonFile}"))
+                {
+                    return false;
+                }
+
+                Console.WriteLine("=========================================");
+                string userName = Environment.GetEnvironmentVariable("USER");
+                if (!string.IsNullOrEmpty(userName))
+                {
+                    if (!RunProcess("sudo", $"chown {userName} {crashReportJsonFile}"))
+                    {
+                        return false;
+                    }
+
+                    Console.WriteLine("=========================================");
+                    if (!RunProcess("sudo", $"ls -l {crashReportJsonFile}"))
+                    {
+                        return false;
+                    }
+
+                    Console.WriteLine("=========================================");
+                    if (!RunProcess("ls", $"-l {crashReportJsonFile}"))
+                    {
+                        return false;
+                    }
+                }
+            }
+
+            if (!File.Exists(crashReportJsonFile))
+            {
+                return false;
+            }
+            outputWriter.WriteLine($"Printing stacktrace from '{crashReportJsonFile}'");
+
+            string contents = File.ReadAllText(crashReportJsonFile);
+            dynamic crashReport = JsonSerializer.Deserialize<JsonObject>(contents);
+            var threads = crashReport["payload"]["threads"];
+
+            // The logic happens in 3 steps:
+            // 1. Read the crashReport.json file, locate all the addresses of interest and then build
+            //    a string that will be passed to llvm-symbolizer. It is populated so that each address
+            //    is in its separate line along with the file name, etc. Some TAGS are added in the
+            //    string that is used in step 2.
+            // 2. llvm-symbolizer is ran and above string is passed as input.
+            // 3. After llvm-symbolizer completes, TAGS are used to format its output to print it in
+            //    the way it will be printed by sos.
+
+            StringBuilder addrBuilder = new StringBuilder();
+            string coreRoot = Environment.GetEnvironmentVariable("CORE_ROOT");
+            foreach (var thread in threads)
+            {
+
+                if (thread["native_thread_id"] == null)
+                {
+                    continue;
+                }
+
+                addrBuilder.AppendLine();
+                addrBuilder.AppendLine("----------------------------------");
+                addrBuilder.AppendLine($"Thread Id: {thread["native_thread_id"]}");
+                addrBuilder.AppendLine("      Child SP               IP Call Site");
+                var stack_frames = thread["stack_frames"];
+                foreach (var frame in stack_frames)
+                {
+                    addrBuilder.Append($"{SKIP_LINE_TAG} {frame["stack_pointer"]} {frame["native_address"]} ");
+                    bool isNative = (string)frame["is_managed"] == "false";
+
+                    if (isNative)
+                    {
+                        string nativeModuleName = (string)frame["native_module"];
+                        string unmanagedName = (string)frame["unmanaged_name"];
+
+                        if ((nativeModuleName != null) && (knownNativeModules.Contains(nativeModuleName)))
+                        {
+                            // Need to use llvm-symbolizer (only if module_address != 0)
+                            AppendAddress(addrBuilder, coreRoot, nativeModuleName, (string)frame["native_address"], (string)frame["module_address"]);
+                        }
+                        else if ((nativeModuleName != null) || (unmanagedName != null))
+                        {
+                            if (nativeModuleName != null)
+                            {
+                                addrBuilder.Append($"{nativeModuleName}!");
+                            }
+                            if (unmanagedName != null)
+                            {
+                                addrBuilder.Append($"{unmanagedName}");
+                            }
+                        }
+                    }
+                    else
+                    {
+                        string fileName = (string)frame["filename"];
+                        string methodName = (string)frame["method_name"];
+
+                        if ((fileName != null) || (methodName != null))
+                        {
+                            // found the managed method name
+                            if (fileName != null)
+                            {
+                                addrBuilder.Append($"{fileName}!");
+                            }
+                            if (methodName != null)
+                            {
+                                addrBuilder.Append($"{methodName}");
+                            }
+                        }
+                        else
+                        {
+                            addrBuilder.Append($"{frame["native_address"]}");
+                        }
+                    }
+                    addrBuilder.AppendLine();
+
+                }
+            }
+
+            string symbolizerOutput = null;
+
+            Process llvmSymbolizer = new Process()
+            {
+                StartInfo = {
+                    FileName = "llvm-symbolizer",
+                    Arguments = $"--pretty-print",
+                    UseShellExecute = false,
+                    RedirectStandardOutput = true,
+                    RedirectStandardError = true,
+                    RedirectStandardInput = true,
+                }
+            };
+
+            outputWriter.WriteLine($"Invoking {llvmSymbolizer.StartInfo.FileName} {llvmSymbolizer.StartInfo.Arguments}");
+
+            try
+            {
+                if (!llvmSymbolizer.Start())
+                {
+                    outputWriter.WriteLine($"Unable to start {llvmSymbolizer.StartInfo.FileName}");
+                }
+
+                using (var symbolizerWriter = llvmSymbolizer.StandardInput)
+                {
+                    symbolizerWriter.WriteLine(addrBuilder.ToString());
+                }
+
+                Task<string> stdout = llvmSymbolizer.StandardOutput.ReadToEndAsync();
+                Task<string> stderr = llvmSymbolizer.StandardError.ReadToEndAsync();
+                bool fSuccess = llvmSymbolizer.WaitForExit(DEFAULT_TIMEOUT_MS);
+
+                Task.WaitAll(stdout, stderr);
+
+                if (!fSuccess)
+                {
+                    outputWriter.WriteLine("Errors while running llvm-symbolizer --pretty-print");
+                    string output = stdout.Result;
+                    string error = stderr.Result;
+
+                    Console.WriteLine("llvm-symbolizer stdout:");
+                    Console.WriteLine(output);
+                    Console.WriteLine("llvm-symbolizer stderr:");
+                    Console.WriteLine(error);
+
+                    llvmSymbolizer.Kill(true);
+
+                    return false;
+                }
+
+                symbolizerOutput = stdout.Result;
+
+            } catch (Exception e) {
+                outputWriter.WriteLine("Errors while running llvm-symbolizer --pretty-print");
+                outputWriter.WriteLine(e.ToString());
+                return false;
+            }
+
+            // Go through the output of llvm-symbolizer and strip all the markers we added initially.
+            string[] contentsToSantize = symbolizerOutput.Split(Environment.NewLine);
+            StringBuilder finalBuilder = new StringBuilder();
+            for (int lineNum = 0; lineNum < contentsToSantize.Length; lineNum++)
+            {
+                string line = contentsToSantize[lineNum].Replace(SKIP_LINE_TAG, string.Empty);
+                if (string.IsNullOrWhiteSpace(line)) continue;
+
+                if (line.EndsWith(TO_BE_CONTINUE_TAG))
+                {
+                    finalBuilder.Append(line.Replace(TO_BE_CONTINUE_TAG, string.Empty));
+                    continue;
+                }
+                finalBuilder.AppendLine(line);
+            }
+            outputWriter.WriteLine("Stack trace:");
+            outputWriter.WriteLine(finalBuilder.ToString());
+            return true;
+        }
+
+        private static void AppendAddress(StringBuilder sb, string coreRoot, string nativeModuleName, string native_address, string module_address)
+        {
+            if (module_address != "0x0")
+            {
+                sb.Append($"{nativeModuleName}!");
+                sb.Append(TO_BE_CONTINUE_TAG);
+                sb.AppendLine();
+                //addrBuilder.AppendLine(frame.native_image_offset);
+                ulong nativeAddress = ulong.Parse(native_address.Substring(2), System.Globalization.NumberStyles.HexNumber);
+                ulong moduleAddress = ulong.Parse(module_address.Substring(2), System.Globalization.NumberStyles.HexNumber);
+                string fullPathToModule = Path.Combine(coreRoot, nativeModuleName);
+                sb.AppendFormat("{0} 0x{1:x}", fullPathToModule, nativeAddress - moduleAddress);
+            }
+        }
+
         // Finds all children processes starting with a process named childName
         // The children are sorted in the order they should be dumped
         static unsafe IEnumerable<Process> FindChildProcessesByName(Process process, string childName)
@@ -343,6 +617,32 @@ public int RunTest(string executable, string outputFile, string errorFile, strin
                         exitCode = process.ExitCode;
                         MobileAppHandler.CheckExitCode(exitCode, testBinaryBase, category, outputWriter);
                         Task.WaitAll(copyOutput, copyError);
+
+                        if (!OperatingSystem.IsWindows())
+                        {
+                            // crashreport is only for non-windows.
+                            if (exitCode != 0)
+                            {
+                                // Search for dump, if created.
+                                if (Directory.Exists(crashDumpFolder))
+                                {
+                                    outputWriter.WriteLine($"Test failed. Trying to see if dump file was created in {crashDumpFolder} since {startTime}");
+                                    DirectoryInfo crashDumpFolderInfo = new DirectoryInfo(crashDumpFolder);
+                                    var dmpFilesInfo = crashDumpFolderInfo.GetFiles("*.crashreport.json").OrderByDescending(f => f.CreationTime);
+                                    foreach (var dmpFile in dmpFilesInfo)
+                                    {
+                                        if (dmpFile.CreationTime < startTime)
+                                        {
+                                            // No new files since test started.
+                                            outputWriter.WriteLine("Finish looking for *.crashreport.json. No new files created.");
+                                            break;
+                                        }
+                                        outputWriter.WriteLine($"Processing {dmpFile.FullName}");
+                                        TryPrintStackTraceFromCrashReport(dmpFile.FullName, outputWriter);
+                                    }
+                                }
+                            }
+                        }
                     }
                     else
                     {
@@ -370,7 +670,7 @@ public int RunTest(string executable, string outputFile, string errorFile, strin
                                 {
                                     string crashDumpPath = Path.Combine(Path.GetFullPath(crashDumpFolder), string.Format("crashdump_{0}.dmp", child.Id));
                                     Console.WriteLine($"Attempting to collect crash dump: {crashDumpPath}");
-                                    if (CollectCrashDump(child, crashDumpPath))
+                                    if (CollectCrashDump(child, crashDumpPath, outputWriter))
                                     {
                                         Console.WriteLine("Collected crash dump: {0}", crashDumpPath);
                                     }

src/tests/Common/testenvironment.proj

@@ -15,6 +15,7 @@
     <DOTNETVariables>
       DOTNET_TieredCompilation;
       DOTNET_DbgEnableMiniDump;
+      DOTNET_EnableCrashReport;
       DOTNET_DbgEnableElfDumpOnMacOS;
       DOTNET_DbgMiniDumpName;
       DOTNET_EnableAES;
@@ -83,8 +84,8 @@
     <TestEnvironment>
       <TieredCompilation>0</TieredCompilation>
       <DbgEnableMiniDump Condition="'$(TargetsWindows)' != 'true'">1</DbgEnableMiniDump> <!-- Enable minidumps for all scenarios -->
-      <DbgEnableElfDumpOnMacOS Condition="'$(TargetsOSX)' == 'true'">1</DbgEnableElfDumpOnMacOS> <!-- Enable minidumps for OSX -->
       <DbgMiniDumpName Condition="'$(TargetsWindows)' != 'true'">$HELIX_DUMP_FOLDER/coredump.%d.dmp</DbgMiniDumpName>
+      <EnableCrashReport Condition="'$(TargetsWindows)' != 'true'">1</EnableCrashReport>
     </TestEnvironment>
   </ItemDefinitionGroup>