Skip to content

Commit

Permalink
nim doc --backend:js, nim doc --doccmd:-d:foo, `nim r --backend:j…
Browse files Browse the repository at this point in the history 
…s`, `--doccmd:skip` + other improvements (#14278)

* `nim doc --backend:js|cpp...`
`nim doc --doccmd:'-d:foo --threads:on'`
`nim r --backend:cpp...` (implies --run --usenimcache)
* --usenimcache works with all targets
* --docCmd:skip now skips compiling snippets; 50X speedup for doc/manual.rst
  • Loading branch information
@timotheecour
timotheecour authored May 11, 2020
1 parent d11cb9d commit 9502e39
Show file tree
Hide file tree
Showing 22 changed files with 213 additions and 99 deletions.
2 changes: 1 addition & 1 deletion compiler/ccgthreadvars.nim
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ proc generateThreadLocalStorage(m: BModule) =

proc generateThreadVarsSize(m: BModule) =
if m.g.nimtv != nil:
let externc = if m.config.cmd == cmdCompileToCpp or
let externc = if m.config.backend == backendCpp or
sfCompileToCpp in m.module.flags: "extern \"C\" "
else: ""
m.s[cfsProcs].addf(
Expand Down
8 changes: 4 additions & 4 deletions compiler/cgen.nim
View file
Original file line number Diff line number Diff line change
Expand Up @@ -279,7 +279,7 @@ proc genProc(m: BModule, prc: PSym)
proc raiseInstr(p: BProc): Rope

template compileToCpp(m: BModule): untyped =
m.config.cmd == cmdCompileToCpp or sfCompileToCpp in m.module.flags
m.config.backend == backendCpp or sfCompileToCpp in m.module.flags

proc getTempName(m: BModule): Rope =
result = m.tmpBase & rope(m.labels)
Expand Down Expand Up @@ -1066,11 +1066,11 @@ proc genProcAux(m: BModule, prc: PSym) =
proc requiresExternC(m: BModule; sym: PSym): bool {.inline.} =
result = (sfCompileToCpp in m.module.flags and
sfCompileToCpp notin sym.getModule().flags and
m.config.cmd != cmdCompileToCpp) or (
m.config.backend != backendCpp) or (
sym.flags * {sfInfixCall, sfCompilerProc, sfMangleCpp} == {} and
sym.flags * {sfImportc, sfExportc} != {} and
sym.magic == mNone and
m.config.cmd == cmdCompileToCpp)
m.config.backend == backendCpp)

proc genProcPrototype(m: BModule, sym: PSym) =
useHeader(m, sym)
Expand Down Expand Up @@ -1867,7 +1867,7 @@ proc writeHeader(m: BModule) =
proc getCFile(m: BModule): AbsoluteFile =
let ext =
if m.compileToCpp: ".nim.cpp"
elif m.config.cmd == cmdCompileToOC or sfCompileToObjc in m.module.flags: ".nim.m"
elif m.config.backend == backendObjc or sfCompileToObjc in m.module.flags: ".nim.m"
else: ".nim.c"
result = changeFileExt(completeCfilePath(m.config, withPackageName(m.config, m.cfilename)), ext)

Expand Down
3 changes: 2 additions & 1 deletion compiler/cmdlinehelper.nim
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,8 @@ proc loadConfigsAndRunMainCommand*(self: NimProg, cache: IdentCache; conf: Confi
# XXX This is hacky. We need to find a better way.
case conf.command
of "cpp", "compiletocpp":
conf.cmd = cmdCompileToCpp
conf.backend = backendCpp
conf.cmd = cmdCompileToBackend
else:
discard

Expand Down
11 changes: 9 additions & 2 deletions compiler/commands.nim
View file
Original file line number Diff line number Diff line change
Expand Up @@ -434,11 +434,18 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
of "outdir":
expectArg(conf, switch, arg, pass, info)
conf.outDir = processPath(conf, arg, info, notRelativeToProj=true)
of "usenimcache":
processOnOffSwitchG(conf, {optUseNimcache}, arg, pass, info)
of "docseesrcurl":
expectArg(conf, switch, arg, pass, info)
conf.docSeeSrcUrl = arg
of "docroot":
conf.docRoot = if arg.len == 0: "@default" else: arg
of "backend", "b":
let backend = parseEnum(arg.normalize, TBackend.default)
if backend == TBackend.default: localError(conf, info, "invalid backend: '$1'" % arg)
conf.backend = backend
of "doccmd": conf.docCmd = arg
of "mainmodule", "m":
discard "allow for backwards compatibility, but don't do anything"
of "define", "d":
Expand Down Expand Up @@ -495,7 +502,7 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
defineSymbol(conf.symbols, "gcmarkandsweep")
of "destructors", "arc":
conf.selectedGC = gcArc
if conf.cmd != cmdCompileToCpp:
if conf.backend != backendCpp:
conf.exc = excGoto
defineSymbol(conf.symbols, "gcdestructors")
defineSymbol(conf.symbols, "gcarc")
Expand All @@ -506,7 +513,7 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
defineSymbol(conf.symbols, "nimV2")
of "orc":
conf.selectedGC = gcOrc
if conf.cmd != cmdCompileToCpp:
if conf.backend != backendCpp:
conf.exc = excGoto
defineSymbol(conf.symbols, "gcdestructors")
defineSymbol(conf.symbols, "gcorc")
Expand Down
29 changes: 18 additions & 11 deletions compiler/docgen.nim
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import
const
exportSection = skField
htmldocsDir = RelativeDir"htmldocs"
docCmdSkip = "skip"

type
TSections = array[TSymKind, Rope]
Expand Down Expand Up @@ -196,6 +197,7 @@ proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef,
initStrTable result.types
result.onTestSnippet =
proc (gen: var RstGenerator; filename, cmd: string; status: int; content: string) =
if conf.docCmd == docCmdSkip: return
inc(gen.id)
var d = TDocumentor(gen)
var outp: AbsoluteFile
Expand Down Expand Up @@ -444,21 +446,26 @@ proc testExample(d: PDoc; ex: PNode) =
d.examples.add "import r\"" & outp.string & "\"\n"

proc runAllExamples(d: PDoc) =
if d.examples.len == 0: return
let docCmd = d.conf.docCmd
let backend = d.conf.backend
# This used to be: `let backend = if isDefined(d.conf, "js"): "js"` (etc), however
# using `-d:js` (etc) cannot work properly, eg would fail with `importjs`
# since semantics are affected by `config.backend`, not by isDefined(d.conf, "js")
if d.examples.len == 0 or docCmd == docCmdSkip: return
let outputDir = d.conf.getNimcacheDir / RelativeDir"runnableExamples"
let outp = outputDir / RelativeFile(extractFilename(d.filename.changeFileExt"" &
"_examples.nim"))
writeFile(outp, d.examples)
let backend = if isDefined(d.conf, "js"): "js"
elif isDefined(d.conf, "cpp"): "cpp"
elif isDefined(d.conf, "objc"): "objc"
else: "c"
if os.execShellCmd(os.getAppFilename() & " " & backend &
" --warning[UnusedImport]:off" &
" --path:" & quoteShell(d.conf.projectPath) &
" --nimcache:" & quoteShell(outputDir) &
" -r " & quoteShell(outp)) != 0:
quit "[Examples] failed: see " & outp.string
let cmd = "$nim $backend -r --warning:UnusedImport:off --path:$path --nimcache:$nimcache $docCmd $file" % [
"nim", os.getAppFilename(),
"backend", $d.conf.backend,
"path", quoteShell(d.conf.projectPath),
"nimcache", quoteShell(outputDir),
"file", quoteShell(outp),
"docCmd", docCmd,
]
if os.execShellCmd(cmd) != 0:
quit "[runnableExamples] failed: generated file: '$1' cmd: $2" % [outp.string, cmd]
else:
# keep generated source file `outp` to allow inspection.
rawMessage(d.conf, hintSuccess, ["runnableExamples: " & outp.string])
Expand Down
24 changes: 12 additions & 12 deletions compiler/extccomp.nim
View file
Original file line number Diff line number Diff line change
Expand Up @@ -307,14 +307,12 @@ proc getConfigVar(conf: ConfigRef; c: TSystemCC, suffix: string): string =
# use ``cpu.os.cc`` for cross compilation, unless ``--compileOnly`` is given
# for niminst support
let fullSuffix =
if conf.cmd == cmdCompileToCpp:
".cpp" & suffix
elif conf.cmd == cmdCompileToOC:
".objc" & suffix
elif conf.cmd == cmdCompileToJS:
".js" & suffix
case conf.backend
of backendCpp, backendJs, backendObjc: "." & $conf.backend & suffix
of backendC: suffix
else:
suffix
doAssert false
""

if (conf.target.hostOS != conf.target.targetOS or conf.target.hostCPU != conf.target.targetCPU) and
optCompileOnly notin conf.globalOptions:
Expand Down Expand Up @@ -481,10 +479,10 @@ proc needsExeExt(conf: ConfigRef): bool {.inline.} =
(conf.target.hostOS == osWindows)

proc useCpp(conf: ConfigRef; cfile: AbsoluteFile): bool =
conf.cmd == cmdCompileToCpp and not cfile.string.endsWith(".c")
conf.backend == backendCpp and not cfile.string.endsWith(".c")

proc envFlags(conf: ConfigRef): string =
result = if conf.cmd == cmdCompileToCpp:
result = if conf.backend == backendCpp:
getEnv("CXXFLAGS")
else:
getEnv("CFLAGS")
Expand Down Expand Up @@ -535,7 +533,7 @@ proc ccHasSaneOverflow*(conf: ConfigRef): bool =

proc getLinkerExe(conf: ConfigRef; compiler: TSystemCC): string =
result = if CC[compiler].linkerExe.len > 0: CC[compiler].linkerExe
elif optMixedMode in conf.globalOptions and conf.cmd != cmdCompileToCpp: CC[compiler].cppCompiler
elif optMixedMode in conf.globalOptions and conf.backend != backendCpp: CC[compiler].cppCompiler
else: getCompilerExe(conf, compiler, AbsoluteFile"")

proc getCompileCFileCmd*(conf: ConfigRef; cfile: Cfile,
Expand Down Expand Up @@ -626,8 +624,10 @@ proc footprint(conf: ConfigRef; cfile: Cfile): SecureHash =
getCompileCFileCmd(conf, cfile))

proc externalFileChanged(conf: ConfigRef; cfile: Cfile): bool =
if conf.cmd notin {cmdCompileToC, cmdCompileToCpp, cmdCompileToOC, cmdCompileToLLVM, cmdNone}:
return false
case conf.backend
of backendInvalid: doAssert false
of backendJs: return false # pre-existing behavior, but not sure it's good
else: discard

var hashFile = toGeneratedFile(conf, conf.withPackageName(cfile.cname), "sha1")
var currentHash = footprint(conf, cfile)
Expand Down
6 changes: 3 additions & 3 deletions compiler/lambdalifting.nim
View file
Original file line number Diff line number Diff line change
Expand Up @@ -235,7 +235,7 @@ template isIterator*(owner: PSym): bool =
proc liftingHarmful(conf: ConfigRef; owner: PSym): bool {.inline.} =
## lambda lifting can be harmful for JS-like code generators.
let isCompileTime = sfCompileTime in owner.flags or owner.kind == skMacro
result = conf.cmd == cmdCompileToJS and not isCompileTime
result = conf.backend == backendJs and not isCompileTime

proc createTypeBoundOpsLL(g: ModuleGraph; refType: PType; info: TLineInfo; owner: PSym) =
createTypeBoundOps(g, nil, refType.lastSon, info)
Expand Down Expand Up @@ -846,14 +846,14 @@ proc liftIterToProc*(g: ModuleGraph; fn: PSym; body: PNode; ptrType: PType): PNo
fn.typ.callConv = oldCC

proc liftLambdas*(g: ModuleGraph; fn: PSym, body: PNode; tooEarly: var bool): PNode =
# XXX gCmd == cmdCompileToJS does not suffice! The compiletime stuff needs
# XXX backend == backendJs does not suffice! The compiletime stuff needs
# the transformation even when compiling to JS ...

# However we can do lifting for the stuff which is *only* compiletime.
let isCompileTime = sfCompileTime in fn.flags or fn.kind == skMacro

if body.kind == nkEmpty or (
g.config.cmd == cmdCompileToJS and not isCompileTime) or
g.config.backend == backendJs and not isCompileTime) or
fn.skipGenericOwner.kind != skModule:

# ignore forward declaration:
Expand Down
69 changes: 38 additions & 31 deletions compiler/main.nim
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,16 +119,14 @@ when not defined(leanCompiler):
let conf = graph.config
conf.exc = excCpp

if conf.outDir.isEmpty:
conf.outDir = conf.projectPath
setOutDir(conf)
if conf.outFile.isEmpty:
conf.outFile = RelativeFile(conf.projectName & ".js")

#incl(gGlobalOptions, optSafeCode)
setTarget(graph.config.target, osJS, cpuJS)
#initDefines()
defineSymbol(graph.config.symbols, "ecmascript") # For backward compatibility
defineSymbol(graph.config.symbols, "js")
semanticPasses(graph)
registerPass(graph, JSgenPass)
compileProject(graph)
Expand Down Expand Up @@ -190,44 +188,53 @@ proc mainCommand*(graph: ModuleGraph) =
conf.lastCmdTime = epochTime()
conf.searchPaths.add(conf.libpath)
setId(100)
template handleC() =
conf.cmd = cmdCompileToC
if conf.exc == excNone: conf.exc = excSetjmp
defineSymbol(graph.config.symbols, "c")
commandCompileToC(graph)

## Calling `setOutDir(conf)` unconditionally would fix regression
## https://github.com/nim-lang/Nim/issues/6583#issuecomment-625711125
when false: setOutDir(conf)
if optUseNimcache in conf.globalOptions: setOutDir(conf)

template handleBackend(backend2: TBackend) =
conf.backend = backend2
conf.cmd = cmdCompileToBackend
defineSymbol(graph.config.symbols, $backend2)
case backend2
of backendC:
if conf.exc == excNone: conf.exc = excSetjmp
commandCompileToC(graph)
of backendCpp:
if conf.exc == excNone: conf.exc = excCpp
commandCompileToC(graph)
of backendObjc:
commandCompileToC(graph)
of backendJs:
when defined(leanCompiler):
globalError(conf, unknownLineInfo, "compiler wasn't built with JS code generator")
else:
if conf.hcrOn:
# XXX: At the moment, system.nim cannot be compiled in JS mode
# with "-d:useNimRtl". The HCR option has been processed earlier
# and it has added this define implictly, so we must undo that here.
# A better solution might be to fix system.nim
undefSymbol(conf.symbols, "useNimRtl")
commandCompileToJS(graph)
of backendInvalid: doAssert false

case conf.command.normalize
of "c", "cc", "compile", "compiletoc": handleC() # compile means compileToC currently
of "cpp", "compiletocpp":
conf.cmd = cmdCompileToCpp
if conf.exc == excNone: conf.exc = excCpp
defineSymbol(graph.config.symbols, "cpp")
commandCompileToC(graph)
of "objc", "compiletooc":
conf.cmd = cmdCompileToOC
defineSymbol(graph.config.symbols, "objc")
commandCompileToC(graph)
of "c", "cc", "compile", "compiletoc": handleBackend(backendC) # compile means compileToC currently
of "cpp", "compiletocpp": handleBackend(backendCpp)
of "objc", "compiletooc": handleBackend(backendObjc)
of "js", "compiletojs": handleBackend(backendJs)
of "r": # different from `"run"`!
conf.globalOptions.incl {optRun, optUseNimcache}
handleC()
handleBackend(conf.backend)
of "run":
conf.cmd = cmdRun
when hasTinyCBackend:
extccomp.setCC(conf, "tcc", unknownLineInfo)
commandCompileToC(graph)
else:
rawMessage(conf, errGenerated, "'run' command not available; rebuild with -d:tinyc")
of "js", "compiletojs":
when defined(leanCompiler):
quit "compiler wasn't built with JS code generator"
else:
conf.cmd = cmdCompileToJS
if conf.hcrOn:
# XXX: At the moment, system.nim cannot be compiled in JS mode
# with "-d:useNimRtl". The HCR option has been processed earlier
# and it has added this define implictly, so we must undo that here.
# A better solution might be to fix system.nim
undefSymbol(conf.symbols, "useNimRtl")
commandCompileToJS(graph)
of "doc0":
when defined(leanCompiler):
quit "compiler wasn't built with documentation generator"
Expand Down
12 changes: 7 additions & 5 deletions compiler/nim.nim
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,17 +88,19 @@ proc handleCmdLine(cache: IdentCache; conf: ConfigRef) =
tccgen.run(conf, conf.arguments)
if optRun in conf.globalOptions:
let output = conf.absOutFile
let ex = quoteShell output
case conf.cmd
of cmdCompileToJS:
execExternalProgram(conf, findNodeJs() & " " & ex & ' ' & conf.arguments)
of cmdCompileToBackend:
var cmdPrefix = ""
case conf.backend
of backendC, backendCpp, backendObjc: discard
of backendJs: cmdPrefix = findNodeJs() & " "
else: doAssert false, $conf.backend
execExternalProgram(conf, cmdPrefix & output.quoteShell & ' ' & conf.arguments)
of cmdDoc, cmdRst2html:
if conf.arguments.len > 0:
# reserved for future use
rawMessage(conf, errGenerated, "'$1 cannot handle arguments" % [$conf.cmd])
openDefaultBrowser($output)
of cmdCompileToC, cmdCompileToCpp, cmdCompileToOC:
execExternalProgram(conf, ex & ' ' & conf.arguments)
else:
# support as needed
rawMessage(conf, errGenerated, "'$1 cannot handle --run" % [$conf.cmd])
Expand Down
Loading

3 comments on commit 9502e39

@kaushalmodi
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@timotheecour I have a Nim wrapper for a C++ header file in which I have this:

when defined(c) or defined(js) or defined(objc):
  {.error: "This library needs to be compiled with the cpp backend.".}

..

proc empty*(v: Vector): bool {.importcpp: "empty".}
  ## Check if the Vector is empty i.e. has zero elements.
  ##
  ## .. code-block::
  ##    :test:
  ##    var
  ##      v = newVector[int]()
  ##    doAssert v.empty()
  ##
  ##    v.add(100)
  ##    doAssert not v.empty()

I fear that the --backend:cpp switch does not work for such test code-blocks? For such importcpp-only symbols the runnableExamples do not work.

@timotheecour
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@kaushalmodi I guess this was resolved in kaushalmodi/std_vector#1 ? (also there is --doccmd )

@kaushalmodi
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@timotheecour yes of course, that comment is before I made the following PR :)

Please sign in to comment.