execution traces (eg for code coverage, debugging, introspection, profiling)

compiler/ccgexprs.nim

@@ -2535,9 +2535,12 @@ template genStmtListExprImpl(exprOrStmt) {.dirty.} =
   let hasNimFrame = p.prc != nil and
       sfSystemModule notin p.module.module.flags and
       optStackTrace in p.prc.options
+  # let hasNimExecTrace = p.prc != nil and optExecTrace in p.prc.options
   var frameName: Rope = nil
+  # var frameNameExecTrace: Rope = nil
   for i in 0..<n.len - 1:
     let it = n[i]
+    # dbg it.kind, i, n.len
     if it.kind == nkComesFrom:
       if hasNimFrame and frameName == nil:
         inc p.labels
@@ -2546,9 +2549,23 @@ template genStmtListExprImpl(exprOrStmt) {.dirty.} =
         add p.s(cpsStmts), initFrameNoDebug(p, frameName,
            makeCString theMacro.name.s,
            quotedFilename(p.config, theMacro.info), it.info.line.int)
+
+      when false:
+        # TODO: is that for exceptions?
+        if hasNimExecTrace and frameNameExecTrace == nil:
+          inc p.labels
+          frameNameExecTrace = "ExecTraceFr_" & rope(p.labels) & "_"
+          let theMacro = it[0].sym
+          dbg "D20200619T215212", theMacro.name.s # PRTEMP
+          add p.s(cpsStmts), initExecTrace(p, frameNameExecTrace,
+             makeCString theMacro.name.s,
+             quotedFilename(p.config, theMacro.info), it.info.line.int)
     else:
       genStmts(p, it)
   if n.len > 0: exprOrStmt
+  when false:
+    if frameNameExecTrace != nil:
+      p.s(cpsStmts).add deinitExecTrace(p, frameNameExecTrace)
   if frameName != nil:
     p.s(cpsStmts).add deinitFrameNoDebug(p, frameName)
 

compiler/ccgmerge_unused.nim

@@ -22,6 +22,7 @@ const
     cfsMergeInfo: "",
     cfsHeaders: "NIM_merge_HEADERS",
     cfsFrameDefines: "NIM_merge_FRAME_DEFINES",
+    cfsExecTraceDefines: "NIM_merge_EXECTRACE_DEFINES",
     cfsForwardTypes: "NIM_merge_FORWARD_TYPES",
     cfsTypes: "NIM_merge_TYPES",
     cfsSeqTypes: "NIM_merge_SEQ_TYPES",

compiler/cgen.nim

@@ -26,6 +26,15 @@ from ic / ic import ModuleBackendFlag
 from modulegraphs import ModuleGraph, PPassContext
 import dynlib
 
+when true: # exectrace
+  proc execTracingEnabled*(conf: ConfigRef, prc: PSym | BProc): bool =
+    result = optExecTraceScope in prc.options
+  const nimExecTraceDefine = "nimExecTraceDefine"
+  const nimExecTraceEnter = "nimExecTraceEnter"
+  const nimExecTraceExit = "nimExecTraceExit"
+  const nimExecTraceLineDefine = "nimExecTraceLineDefine"
+  const nimExecTraceLine = "nimExecTraceLine"
+
 when not declared(dynlib.libCandidates):
   proc libCandidates(s: string, dest: var seq[string]) =
     ## given a library name pattern `s` write possible library names to `dest`.
@@ -266,11 +275,15 @@ proc genLineDir(p: BProc, t: PNode) =
   if optEmbedOrigSrc in p.config.globalOptions:
     p.s(cpsStmts).add(~"//" & sourceLine(p.config, t.info) & "\L")
   genCLineDir(p.s(cpsStmts), toFullPath(p.config, t.info), line, p.config)
-  if ({optLineTrace, optStackTrace} * p.options == {optLineTrace, optStackTrace}) and
-      (p.prc == nil or sfPure notin p.prc.flags) and t.info.fileIndex != InvalidFileIdx:
-    if freshLineInfo(p, t.info):
-      linefmt(p, cpsStmts, "nimln_($1, $2);$n",
-              [line, quotedFilename(p.config, t.info)])
+
+  template isTrace(opt): bool =
+    ({opt, optLineTrace} * p.options == {opt, optLineTrace}) and
+      (p.prc == nil or sfPure notin p.prc.flags) and t.info.fileIndex != InvalidFileIdx
+  if isTrace(optStackTrace) and freshLineInfo(p, t.info):
+    linefmt(p, cpsStmts, "nimln_($1, $2);$n", [line, quotedFilename(p.config, t.info)])
+  # IMPROVE
+  if isTrace(optExecTrace) and execTracingEnabled(p.config, p) and freshLineInfo(p, t.info):
+    linefmt(p, cpsStmts, "$3($1, $2);$n", [line, quotedFilename(p.config, t.info), nimExecTraceLineDefine])
 
 proc postStmtActions(p: BProc) {.inline.} =
   p.s(cpsStmts).add(p.module.injectStmt)
@@ -653,12 +666,36 @@ proc initFrame(p: BProc, procname, filename: Rope): Rope =
   $1  define nimln_(n, file) \
       FR_.line = n; FR_.filename = file;
   """
+  # TODO: does `nimln_` really need to reset filename? or does my stacktrace speedup PR make this irrelevant anyway ?
   if p.module.s[cfsFrameDefines].len == 0:
     appcg(p.module, p.module.s[cfsFrameDefines], frameDefines, ["#"])
 
   discard cgsym(p.module, "nimFrame")
   result = ropecg(p.module, "\tnimfr_($1, $2);$n", [procname, filename])
 
+when true:
+  proc initExecTrace(p: BProc, procname, filename: Rope, line: int = 0): Rope =
+    const frameDefines = """
+    #define $1(proc, file, line2) \
+        TFrame FR2_; \
+        FR2_.procname = proc; FR2_.filename = file; FR2_.line = line2; FR2_.len = 0; \
+        $2(&FR2_);
+
+    #define $3(n, file) \
+      FR2_.line = n; FR2_.filename = file; \
+      $4(&FR2_, n); // PRTEMP
+    """.format [nimExecTraceDefine, nimExecTraceEnter, nimExecTraceLineDefine, nimExecTraceLine]
+    if p.module.s[cfsExecTraceDefines].len == 0:
+      p.module.s[cfsExecTraceDefines].add frameDefines
+    discard cgsym(p.module, nimExecTraceEnter)
+    discard cgsym(p.module, nimExecTraceExit)
+    discard cgsym(p.module, nimExecTraceLine)
+    # let line = 0
+    result = ropecg(p.module, "\t$1($2, $3, $4);$n", [nimExecTraceDefine, procname, filename, line.`$`.rope])
+
+  proc deinitExecTrace(p: BProc): Rope =
+    result = ropecg(p.module, "\t#$1();$n", [nimExecTraceExit])
+
 proc initFrameNoDebug(p: BProc; frame, procname, filename: Rope; line: int): Rope =
   discard cgsym(p.module, "nimFrame")
   p.blocks[0].sections[cpsLocals].addf("TFrame $1;$n", [frame])
@@ -1081,6 +1118,9 @@ proc genProcAux(m: BModule, prc: PSym) =
       generatedProc.add(initFrame(p, procname, quotedFilename(p.config, prc.info)))
     else:
       generatedProc.add(p.s(cpsLocals))
+    if execTracingEnabled(m.config, prc):
+      var procname = makeCString(prc.name.s)
+      generatedProc.add(initExecTrace(p, procname, quotedFilename(p.config, prc.info), prc.info.line.int))
     if optProfiler in prc.options:
       # invoke at proc entry for recursion:
       appcg(p, cpsInit, "\t#nimProfile();$n", [])
@@ -1090,6 +1130,7 @@ proc genProcAux(m: BModule, prc: PSym) =
     generatedProc.add(p.s(cpsInit))
     generatedProc.add(p.s(cpsStmts))
     if beforeRetNeeded in p.flags: generatedProc.add(~"\t}BeforeRet_: ;$n")
+    if execTracingEnabled(p.config, prc): generatedProc.add(deinitExecTrace(p))
     if optStackTrace in prc.options: generatedProc.add(deinitFrame(p))
     generatedProc.add(returnStmt)
     generatedProc.add(~"}$N")
@@ -1709,6 +1750,12 @@ proc genInitCode(m: BModule) =
       else:
         prc.add(~"\tTFrame FR_; FR_.len = 0;$N")
 
+    # PRTEMP FACTOR
+    if optExecTrace in m.initProc.options and frameDeclared2 notin m.flags:
+      incl m.flags, frameDeclared2
+      var procname = makeCString(m.module.name.s)
+      prc.add(initExecTrace(m.initProc, procname, quotedFilename(m.config, m.module.info)))
+
     writeSection(initProc, cpsInit, m.hcrOn)
     writeSection(initProc, cpsStmts)
 
@@ -1773,10 +1820,17 @@ proc genModule(m: BModule, cfile: Cfile): Rope =
   result.add(m.s[cfsHeaders])
   if m.config.cppCustomNamespace.len > 0:
     result.add openNamespaceNim(m.config.cppCustomNamespace)
-  if m.s[cfsFrameDefines].len > 0:
-    result.add(m.s[cfsFrameDefines])
-  else:
-    result.add("#define nimfr_(x, y)\n#define nimln_(x, y)\n")
+
+  template gensec(cfs, body) =
+    # result.add(genSectionStart(cfs, m.config))
+    if m.s[cfs].len > 0:
+      result.add(m.s[cfs])
+    else:
+      body
+    # result.add(genSectionEnd(cfs, m.config))
+  gensec(cfsFrameDefines):
+    result.add("#define nimfr_(x, y)\n#define nimln_(x, y)\n") # CHECKME: needed?
+  gensec(cfsExecTraceDefines): discard
 
   for i in cfsForwardTypes..cfsProcs:
     if m.s[i].len > 0:

compiler/cgendata.nim

@@ -19,6 +19,7 @@ type
     cfsMergeInfo,             # section containing merge information
     cfsHeaders,               # section for C include file headers
     cfsFrameDefines           # section for nim frame macros
+    cfsExecTraceDefines       # section for execTrace macros
     cfsForwardTypes,          # section for C forward typedefs
     cfsTypes,                 # section for C typedefs
     cfsSeqTypes,              # section for sequence types only
@@ -109,6 +110,7 @@ type
     usesThreadVars,     # true if the module uses a thread var
     frameDeclared,      # hack for ROD support so that we don't declare
                         # a frame var twice in an init proc
+    frameDeclared2,     # ditto for exectrace
     isHeaderFile,       # C source file is the header file
     includesStringh,    # C source file already includes ``<string.h>``
     objHasKidsValid     # whether we can rely on tfObjHasKids

compiler/commands.nim

@@ -301,6 +301,7 @@ proc testCompileOption*(conf: ConfigRef; switch: string, info: TLineInfo): bool
   of "threadanalysis": result = contains(conf.globalOptions, optThreadAnalysis)
   of "stacktrace": result = contains(conf.options, optStackTrace)
   of "stacktracemsgs": result = contains(conf.options, optStackTraceMsgs)
+  of "exectrace": result = contains(conf.options, optExecTrace)
   of "linetrace": result = contains(conf.options, optLineTrace)
   of "debugger": result = contains(conf.globalOptions, optCDebug)
   of "profiler": result = contains(conf.options, optProfiler)
@@ -657,6 +658,12 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     if conf.backend == backendJs: discard
     else: processOnOffSwitchG(conf, {optThreadAnalysis}, arg, pass, info)
   of "stacktrace": processOnOffSwitch(conf, {optStackTrace}, arg, pass, info)
+  of "exectrace":
+    # TODO: optExecTrace should be processOnOffSwitchG ?
+    processOnOffSwitch(conf, {optExecTrace, optExecTraceScope}, arg, pass, info)
+  of "exectraceskip":
+    expectArg(conf, switch, arg, pass, info)
+    conf.exectraceSkip.add arg
   of "stacktracemsgs": processOnOffSwitch(conf, {optStackTraceMsgs}, arg, pass, info)
   of "excessivestacktrace": processOnOffSwitchG(conf, {optExcessiveStackTrace}, arg, pass, info)
   of "linetrace": processOnOffSwitch(conf, {optLineTrace}, arg, pass, info)

compiler/condsyms.nim

@@ -137,3 +137,4 @@ proc initDefines*(symbols: StringTableRef) =
   defineSymbol("nimHasDragonBox")
   defineSymbol("nimHasHintAll")
   defineSymbol("nimHasTrace")
+  defineSymbol("nimHasExecTrace")

compiler/nim.nim

@@ -26,10 +26,12 @@ when defined(windows) and not defined(nimKochBootstrap):
 import
   commands, options, msgs, extccomp, main, idents, lineinfos, cmdlinehelper,
   pathutils, modulegraphs
-
-from browsers import openDefaultBrowser
+from std/browsers import openDefaultBrowser
 from nodejs import findNodeJs
 
+when defined(nimHasExecTrace):
+  import system/exectrace_aux # PRTEMP
+
 when hasTinyCBackend:
   import tccgen
 
@@ -123,9 +125,14 @@ when compileOption("gc", "refc"):
   # the new correct mark&sweet collector is too slow :-/
   GC_disableMarkAndSweep()
 
+# enableRuntimeTracing(true)
+
 when not defined(selftest):
+  # when defined(nimHasExecTrace):
+  # enableRuntimeTracing(true)
   let conf = newConfigRef()
   handleCmdLine(newIdentCache(), conf)
+  # enableRuntimeTracing(false)
   when declared(GC_setMaxPause):
     echo GC_getStatistics()
   msgQuit(int8(conf.errorCounter > 0))

compiler/options.nim

@@ -34,6 +34,8 @@ type                          # please make sure we have under 32 options
     optStackTrace, # stack tracing support
     optStackTraceMsgs, # enable custom runtime msgs via `setFrameMsg`
     optLineTrace,             # line tracing support (includes stack tracing)
+    optExecTrace,             # execution trace
+    optExecTraceScope,        # execution trace in scope
     optByRef,                 # use pass by ref for objects
                               # (for interfacing with C)
     optProfiler,              # profiler turned on
@@ -386,6 +388,7 @@ type
                                 severity: Severity) {.closure, gcsafe.}
     cppCustomNamespace*: string
     vmProfileData*: ProfileData
+    exectraceSkip*: seq[string]
 
 proc assignIfDefault*[T](result: var T, val: T, def = default(T)) =
   ## if `result` was already assigned to a value (that wasn't `def`), this is a noop.

compiler/pragmas.nim

@@ -30,7 +30,7 @@ const
     wBorrow, wImportCompilerProc, wThread,
     wAsmNoStackFrame, wDiscardable, wNoInit, wCodegenDecl,
     wGensym, wInject, wRaises, wTags, wLocks, wDelegator, wGcSafe,
-    wConstructor, wLiftLocals, wStackTrace, wLineTrace, wNoDestroy,
+    wConstructor, wLiftLocals, wStackTrace, wExecTrace, wLineTrace, wNoDestroy,
     wRequires, wEnsures}
   converterPragmas* = procPragmas
   methodPragmas* = procPragmas+{wBase}-{wImportCpp}
@@ -48,7 +48,7 @@ const
     wBoundChecks, wOverflowChecks, wNilChecks, wStaticBoundchecks,
     wStyleChecks, wAssertions,
     wWarnings, wHints,
-    wLineDir, wStackTrace, wLineTrace, wOptimization, wHint, wWarning, wError,
+    wLineDir, wStackTrace, wExecTrace, wLineTrace, wOptimization, wHint, wWarning, wError,
     wFatal, wDefine, wUndef, wCompile, wLink, wLinksys, wPure, wPush, wPop,
     wPassl, wPassc, wLocalPassc,
     wDeadCodeElimUnused,  # deprecated, always on
@@ -375,6 +375,7 @@ proc pragmaToOptions(w: TSpecialWord): TOptions {.inline.} =
   of wHints: {optHints}
   of wLineDir: {optLineDir}
   of wStackTrace: {optStackTrace}
+  of wExecTrace: {optExecTraceScope}
   of wLineTrace: {optLineTrace}
   of wDebugger: {optNone}
   of wProfiler: {optProfiler, optMemTracker}

compiler/wordrecg.nim

@@ -55,7 +55,8 @@ type
     wHintAsError = "hintAsError",
     wLine = "line", wPush = "push",
     wPop = "pop", wDefine = "define", wUndef = "undef", wLineDir = "lineDir",
-    wStackTrace = "stackTrace", wLineTrace = "lineTrace", wLink = "link", wCompile = "compile",
+    wStackTrace = "stackTrace", wExecTrace = "exectrace",
+    wLineTrace = "lineTrace", wLink = "link", wCompile = "compile",
     wLinksys = "linksys", wDeprecated = "deprecated", wVarargs = "varargs", wCallconv = "callconv",
     wDebugger = "debugger", wNimcall = "nimcall", wStdcall = "stdcall", wCdecl = "cdecl",
     wSafecall = "safecall", wSyscall = "syscall", wInline = "inline", wNoInline = "noinline",
@@ -144,4 +145,4 @@ else:
     for i in a..b:
       if cmpIgnoreStyle($i, s) == 0:
         return i
-    result = default
+    result = default

lib/system.nim

@@ -1898,6 +1898,7 @@ include "system/gc_interface"
 
 # we have to compute this here before turning it off in except.nim anyway ...
 const NimStackTrace = compileOption("stacktrace")
+const NimExecTrace = when defined(nimHasExecTrace): compileOption("exectrace") else: false
 
 import system/coro_detection
 
@@ -2471,6 +2472,14 @@ when defined(js) or defined(nimscript):
   proc addInt*(result: var string; x: int64) =
     result.add $x
 
+when notJSnotNims and NimExecTrace:
+  # move somewhere?
+  # {.compilerRtl, inl, raises: [], importc.}
+  proc nimExecTraceEnter(s: PFrame) {.compilerproc, importc.}
+  proc nimExecTraceExit {.compilerproc, importc.}
+  # TODO: allow enable/disable
+  proc nimExecTraceLine(s: PFrame, line: int16) {.compilerproc, importc.}
+
 proc quit*(errormsg: string, errorcode = QuitFailure) {.noreturn.} =
   ## A shorthand for `echo(errormsg); quit(errorcode)`.
   when defined(nimscript) or defined(js) or (hostOS == "standalone"):

lib/system/excpt.nim

@@ -99,6 +99,8 @@ proc getFrame*(): PFrame {.compilerRtl, inl.} = framePtr
 proc popFrame {.compilerRtl, inl.} =
   framePtr = framePtr.prev
 
+template nimGetFramePtrInternal*(): untyped = framePtr
+
 when false:
   proc popFrameOfAddr(s: PFrame) {.compilerRtl.} =
     var it = framePtr