Skip to content

Commit

Permalink
Produce crashreport.json and use llvm-symbolizer to create stack trace (
Browse files Browse the repository at this point in the history
#77578)

* add DOTNET_EnableCrashReport

* Enable JitStress and JitStressRegs by default to see some test failures

* Enable stress switches only for JIT

* fix the build errors

* Revert "Set DNER for all local fields (#77341)"

This reverts commit 4e85471.

* Set EnableCrashReport in testenvironment.proj

* Pass --crashReport to capture for hangs

* Comment bunch of jobs to trigger

* Comment all runs except Linux_x64

* Stop running libraries job

* Fix the tag in testenvironment.proj

* Use keepnativesymbols

* Update JitStress/JitStressRegs

* Embed native_image_offset field in crashreport.json

* Add TryPrintStackTraceFromCrashReport()

* Call TryPrintStackTraceFromCrashReport() even for crashes

* Add newtonsoft in CoreCLRTestLibrary.csproj

* Fix the crash dump folder

* Add random failure in jit

* Add some logging around TryPrintStackTraceFromCrashReport

* Move the error randomization to GetLayout() that has compiler object access

* Seach for crashreport.json files

* Switch to System.Text.Json

* fix the json path

* Add some logging

* Fix the clrjit symbols

* Add a comment

* Add few more commnets

* chown

* ls -l, chmod

* Change to chown

* USER environment variable

* update runtime.yml

* fix runtime.yml

* add logging

* Add llvm-symbolizer -h

* Skip superpmi pipelines

* read contents before process exit

* Add error output also

* fix some parameters for llvm-symbolizer

* Also enable OSX_x64 runs

* Remove the debugging code:

* Revert "Revert "Set DNER for all local fields (#77341)""

This reverts commit 5cfff11.

* Undo JIT changes to trigger assert

* Undo some yml changes

* Review feedback

* Undo runtime.yml changes

* Add some comments

* Address review feedback
  • Loading branch information
kunalspathak authored Jan 27, 2023
1 parent 89c68dc commit ff987dc
Show file tree
Hide file tree
Showing 4 changed files with 310 additions and 7 deletions.
3 changes: 2 additions & 1 deletion eng/pipelines/coreclr/templates/build-job.yml
Original file line number Diff line number Diff line change
Expand Up @@ -189,8 +189,9 @@ jobs:
displayName: Build and generate native prerequisites

# Build CoreCLR Runtime
# TODO: Use --keepnativesymbols only for PRs.
- ${{ if ne(parameters.osGroup, 'windows') }}:
- script: $(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) $(crossArg) $(osArg) -ci $(compilerArg) $(clrRuntimeComponentsBuildArg) $(pgoInstrumentArg) $(officialBuildIdArg) $(clrInterpreterBuildArg) $(clrRuntimePortableBuildArg) $(CoreClrPgoDataArg)
- script: $(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) $(crossArg) $(osArg) -ci $(compilerArg) $(clrRuntimeComponentsBuildArg) $(pgoInstrumentArg) $(officialBuildIdArg) $(clrInterpreterBuildArg) $(clrRuntimePortableBuildArg) $(CoreClrPgoDataArg) --keepnativesymbols
displayName: Build CoreCLR Runtime
- ${{ if eq(parameters.osGroup, 'windows') }}:
- script: $(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) -ci $(enforcePgoArg) $(pgoInstrumentArg) $(officialBuildIdArg) $(clrInterpreterBuildArg) $(CoreClrPgoDataArg)
Expand Down
1 change: 1 addition & 0 deletions src/coreclr/debug/createdump/crashreportwriter.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -223,6 +223,7 @@ CrashReportWriter::WriteStackFrame(const StackFrame& frame)
WriteValue64("stack_pointer", frame.StackPointer());
WriteValue64("native_address", frame.InstructionPointer());
WriteValue64("native_offset", frame.NativeOffset());
WriteValue64("native_image_offset", (frame.InstructionPointer() - frame.ModuleAddress()));
if (frame.IsManaged())
{
WriteValue32("token", frame.Token());
Expand Down
310 changes: 305 additions & 5 deletions src/tests/Common/Coreclr.TestWrapper/CoreclrTestWrapperLib.cs
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,9 @@
using System.Linq;
using System.Runtime.InteropServices;
using System.Text;
using System.Text.Json;
using System.Text.Json.Nodes;
using System.Text.Json.Serialization;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
Expand Down Expand Up @@ -204,12 +207,13 @@ public class CoreclrTestWrapperLib
public const string COLLECT_DUMPS_ENVIRONMENT_VAR = "__CollectDumps";
public const string CRASH_DUMP_FOLDER_ENVIRONMENT_VAR = "__CrashDumpFolder";

static bool CollectCrashDump(Process process, string path)
static bool CollectCrashDump(Process process, string crashDumpPath, StreamWriter outputWriter)
{
string coreRoot = Environment.GetEnvironmentVariable("CORE_ROOT");
string createdumpPath = Path.Combine(coreRoot, "createdump");
string arguments = $"--name \"{path}\" {process.Id} --withheap";
string arguments = $"--name \"{crashDumpPath}\" {process.Id} --withheap";
Process createdump = new Process();
bool crashReportPresent = false;

if (OperatingSystem.IsWindows())
{
Expand All @@ -219,8 +223,8 @@ static bool CollectCrashDump(Process process, string path)
else if (OperatingSystem.IsLinux() || OperatingSystem.IsMacOS())
{
createdump.StartInfo.FileName = "sudo";
createdump.StartInfo.Arguments = $"{createdumpPath} " + arguments;
createdump.StartInfo.EnvironmentVariables.Add("DOTNET_DbgEnableElfDumpOnMacOS", "1");
createdump.StartInfo.Arguments = $"{createdumpPath} --crashreport {arguments}";
crashReportPresent = true;
}

createdump.StartInfo.UseShellExecute = false;
Expand All @@ -244,6 +248,11 @@ static bool CollectCrashDump(Process process, string path)
Console.WriteLine(output);
Console.WriteLine("createdump stderr:");
Console.WriteLine(error);

if (crashReportPresent)
{
TryPrintStackTraceFromCrashReport(crashDumpPath + ".crashreport.json", outputWriter);
}
}
else
{
Expand All @@ -253,6 +262,271 @@ static bool CollectCrashDump(Process process, string path)
return fSuccess && createdump.ExitCode == 0;
}

private static List<string> knownNativeModules = new List<string>() { "libcoreclr.so", "libclrjit.so" };
private static string TO_BE_CONTINUE_TAG = "<TO_BE_CONTINUE>";
private static string SKIP_LINE_TAG = "# <SKIP_LINE>";


static bool RunProcess(string fileName, string arguments)
{
Process proc = new Process()
{
StartInfo = new ProcessStartInfo()
{
FileName = fileName,
Arguments = arguments,
UseShellExecute = false,
RedirectStandardOutput = true,
RedirectStandardError = true,
}
};

Console.WriteLine($"Invoking: {proc.StartInfo.FileName} {proc.StartInfo.Arguments}");
proc.Start();

Task<string> stdOut = proc.StandardOutput.ReadToEndAsync();
Task<string> stdErr = proc.StandardError.ReadToEndAsync();
if(!proc.WaitForExit(DEFAULT_TIMEOUT_MS))
{
proc.Kill(true);
Console.WriteLine($"Timedout: '{fileName} {arguments}");
return false;
}

Task.WaitAll(stdOut, stdErr);
string output = stdOut.Result;
string error = stdErr.Result;
if (!string.IsNullOrWhiteSpace(output))
{
Console.WriteLine($"stdout: {output}");
}
if (!string.IsNullOrWhiteSpace(error))
{
Console.WriteLine($"stderr: {error}");
}
return true;
}

/// <summary>
/// Parse crashreport.json file, use llvm-symbolizer to extract symbols
/// and recreate the stacktrace that is printed on the console.
/// </summary>
/// <param name="crashReportJsonFile">crash dump path</param>
/// <param name="outputWriter">Stream for writing logs</param>
/// <returns>true, if we can print the stack trace, otherwise false.</returns>
static bool TryPrintStackTraceFromCrashReport(string crashReportJsonFile, StreamWriter outputWriter)
{
if (OperatingSystem.IsLinux() || OperatingSystem.IsMacOS())
{
if (!RunProcess("sudo", $"ls -l {crashReportJsonFile}"))
{
return false;
}

Console.WriteLine("=========================================");
string userName = Environment.GetEnvironmentVariable("USER");
if (!string.IsNullOrEmpty(userName))
{
if (!RunProcess("sudo", $"chown {userName} {crashReportJsonFile}"))
{
return false;
}

Console.WriteLine("=========================================");
if (!RunProcess("sudo", $"ls -l {crashReportJsonFile}"))
{
return false;
}

Console.WriteLine("=========================================");
if (!RunProcess("ls", $"-l {crashReportJsonFile}"))
{
return false;
}
}
}

if (!File.Exists(crashReportJsonFile))
{
return false;
}
outputWriter.WriteLine($"Printing stacktrace from '{crashReportJsonFile}'");

string contents = File.ReadAllText(crashReportJsonFile);
dynamic crashReport = JsonSerializer.Deserialize<JsonObject>(contents);
var threads = crashReport["payload"]["threads"];

// The logic happens in 3 steps:
// 1. Read the crashReport.json file, locate all the addresses of interest and then build
// a string that will be passed to llvm-symbolizer. It is populated so that each address
// is in its separate line along with the file name, etc. Some TAGS are added in the
// string that is used in step 2.
// 2. llvm-symbolizer is ran and above string is passed as input.
// 3. After llvm-symbolizer completes, TAGS are used to format its output to print it in
// the way it will be printed by sos.

StringBuilder addrBuilder = new StringBuilder();
string coreRoot = Environment.GetEnvironmentVariable("CORE_ROOT");
foreach (var thread in threads)
{

if (thread["native_thread_id"] == null)
{
continue;
}

addrBuilder.AppendLine();
addrBuilder.AppendLine("----------------------------------");
addrBuilder.AppendLine($"Thread Id: {thread["native_thread_id"]}");
addrBuilder.AppendLine(" Child SP IP Call Site");
var stack_frames = thread["stack_frames"];
foreach (var frame in stack_frames)
{
addrBuilder.Append($"{SKIP_LINE_TAG} {frame["stack_pointer"]} {frame["native_address"]} ");
bool isNative = (string)frame["is_managed"] == "false";

if (isNative)
{
string nativeModuleName = (string)frame["native_module"];
string unmanagedName = (string)frame["unmanaged_name"];

if ((nativeModuleName != null) && (knownNativeModules.Contains(nativeModuleName)))
{
// Need to use llvm-symbolizer (only if module_address != 0)
AppendAddress(addrBuilder, coreRoot, nativeModuleName, (string)frame["native_address"], (string)frame["module_address"]);
}
else if ((nativeModuleName != null) || (unmanagedName != null))
{
if (nativeModuleName != null)
{
addrBuilder.Append($"{nativeModuleName}!");
}
if (unmanagedName != null)
{
addrBuilder.Append($"{unmanagedName}");
}
}
}
else
{
string fileName = (string)frame["filename"];
string methodName = (string)frame["method_name"];

if ((fileName != null) || (methodName != null))
{
// found the managed method name
if (fileName != null)
{
addrBuilder.Append($"{fileName}!");
}
if (methodName != null)
{
addrBuilder.Append($"{methodName}");
}
}
else
{
addrBuilder.Append($"{frame["native_address"]}");
}
}
addrBuilder.AppendLine();

}
}

string symbolizerOutput = null;

Process llvmSymbolizer = new Process()
{
StartInfo = {
FileName = "llvm-symbolizer",
Arguments = $"--pretty-print",
UseShellExecute = false,
RedirectStandardOutput = true,
RedirectStandardError = true,
RedirectStandardInput = true,
}
};

outputWriter.WriteLine($"Invoking {llvmSymbolizer.StartInfo.FileName} {llvmSymbolizer.StartInfo.Arguments}");

try
{
if (!llvmSymbolizer.Start())
{
outputWriter.WriteLine($"Unable to start {llvmSymbolizer.StartInfo.FileName}");
}

using (var symbolizerWriter = llvmSymbolizer.StandardInput)
{
symbolizerWriter.WriteLine(addrBuilder.ToString());
}

Task<string> stdout = llvmSymbolizer.StandardOutput.ReadToEndAsync();
Task<string> stderr = llvmSymbolizer.StandardError.ReadToEndAsync();
bool fSuccess = llvmSymbolizer.WaitForExit(DEFAULT_TIMEOUT_MS);

Task.WaitAll(stdout, stderr);

if (!fSuccess)
{
outputWriter.WriteLine("Errors while running llvm-symbolizer --pretty-print");
string output = stdout.Result;
string error = stderr.Result;

Console.WriteLine("llvm-symbolizer stdout:");
Console.WriteLine(output);
Console.WriteLine("llvm-symbolizer stderr:");
Console.WriteLine(error);

llvmSymbolizer.Kill(true);

return false;
}

symbolizerOutput = stdout.Result;

} catch (Exception e) {
outputWriter.WriteLine("Errors while running llvm-symbolizer --pretty-print");
outputWriter.WriteLine(e.ToString());
return false;
}

// Go through the output of llvm-symbolizer and strip all the markers we added initially.
string[] contentsToSantize = symbolizerOutput.Split(Environment.NewLine);
StringBuilder finalBuilder = new StringBuilder();
for (int lineNum = 0; lineNum < contentsToSantize.Length; lineNum++)
{
string line = contentsToSantize[lineNum].Replace(SKIP_LINE_TAG, string.Empty);
if (string.IsNullOrWhiteSpace(line)) continue;

if (line.EndsWith(TO_BE_CONTINUE_TAG))
{
finalBuilder.Append(line.Replace(TO_BE_CONTINUE_TAG, string.Empty));
continue;
}
finalBuilder.AppendLine(line);
}
outputWriter.WriteLine("Stack trace:");
outputWriter.WriteLine(finalBuilder.ToString());
return true;
}

private static void AppendAddress(StringBuilder sb, string coreRoot, string nativeModuleName, string native_address, string module_address)
{
if (module_address != "0x0")
{
sb.Append($"{nativeModuleName}!");
sb.Append(TO_BE_CONTINUE_TAG);
sb.AppendLine();
//addrBuilder.AppendLine(frame.native_image_offset);
ulong nativeAddress = ulong.Parse(native_address.Substring(2), System.Globalization.NumberStyles.HexNumber);
ulong moduleAddress = ulong.Parse(module_address.Substring(2), System.Globalization.NumberStyles.HexNumber);
string fullPathToModule = Path.Combine(coreRoot, nativeModuleName);
sb.AppendFormat("{0} 0x{1:x}", fullPathToModule, nativeAddress - moduleAddress);
}
}

// Finds all children processes starting with a process named childName
// The children are sorted in the order they should be dumped
static unsafe IEnumerable<Process> FindChildProcessesByName(Process process, string childName)
Expand Down Expand Up @@ -343,6 +617,32 @@ public int RunTest(string executable, string outputFile, string errorFile, strin
exitCode = process.ExitCode;
MobileAppHandler.CheckExitCode(exitCode, testBinaryBase, category, outputWriter);
Task.WaitAll(copyOutput, copyError);

if (!OperatingSystem.IsWindows())
{
// crashreport is only for non-windows.
if (exitCode != 0)
{
// Search for dump, if created.
if (Directory.Exists(crashDumpFolder))
{
outputWriter.WriteLine($"Test failed. Trying to see if dump file was created in {crashDumpFolder} since {startTime}");
DirectoryInfo crashDumpFolderInfo = new DirectoryInfo(crashDumpFolder);
var dmpFilesInfo = crashDumpFolderInfo.GetFiles("*.crashreport.json").OrderByDescending(f => f.CreationTime);
foreach (var dmpFile in dmpFilesInfo)
{
if (dmpFile.CreationTime < startTime)
{
// No new files since test started.
outputWriter.WriteLine("Finish looking for *.crashreport.json. No new files created.");
break;
}
outputWriter.WriteLine($"Processing {dmpFile.FullName}");
TryPrintStackTraceFromCrashReport(dmpFile.FullName, outputWriter);
}
}
}
}
}
else
{
Expand Down Expand Up @@ -370,7 +670,7 @@ public int RunTest(string executable, string outputFile, string errorFile, strin
{
string crashDumpPath = Path.Combine(Path.GetFullPath(crashDumpFolder), string.Format("crashdump_{0}.dmp", child.Id));
Console.WriteLine($"Attempting to collect crash dump: {crashDumpPath}");
if (CollectCrashDump(child, crashDumpPath))
if (CollectCrashDump(child, crashDumpPath, outputWriter))
{
Console.WriteLine("Collected crash dump: {0}", crashDumpPath);
}
Expand Down
3 changes: 2 additions & 1 deletion src/tests/Common/testenvironment.proj
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
<DOTNETVariables>
DOTNET_TieredCompilation;
DOTNET_DbgEnableMiniDump;
DOTNET_EnableCrashReport;
DOTNET_DbgEnableElfDumpOnMacOS;
DOTNET_DbgMiniDumpName;
DOTNET_EnableAES;
Expand Down Expand Up @@ -83,8 +84,8 @@
<TestEnvironment>
<TieredCompilation>0</TieredCompilation>
<DbgEnableMiniDump Condition="'$(TargetsWindows)' != 'true'">1</DbgEnableMiniDump> <!-- Enable minidumps for all scenarios -->
<DbgEnableElfDumpOnMacOS Condition="'$(TargetsOSX)' == 'true'">1</DbgEnableElfDumpOnMacOS> <!-- Enable minidumps for OSX -->
<DbgMiniDumpName Condition="'$(TargetsWindows)' != 'true'">$HELIX_DUMP_FOLDER/coredump.%d.dmp</DbgMiniDumpName>
<EnableCrashReport Condition="'$(TargetsWindows)' != 'true'">1</EnableCrashReport>
</TestEnvironment>
</ItemDefinitionGroup>

Expand Down

0 comments on commit ff987dc

Please sign in to comment.