From c129cfff1cef8fb2dc1d873c83dd0a56489257eb Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Wed, 4 Aug 2021 01:05:07 +0300 Subject: [PATCH 1/4] docgen: implement doc link resolution in current module no special handling of routineKinds here fix segfault fix references in the Manual a supplemental test generics: rm type parameters in normalized names allow underscore in names and export (*) many fixes of warnings: - always resolve all the 3 cases: manual references, RST anchors, Nim anchors and print warnings in case of ambiguity - preserve anchor/substitution definition places - correctly handle `include` in ``.nim`` files for doc comment warnings/errors - make warning messages more detailed change wording & add test regarding brackets in input parameter types some progress: - fixes for handling symbols with backticks - now reference points to concrete function (not group) if there is no other overloads - link text is the full name (with function parameters) if there is only 1 overload. If there are > 1 then it's like "proc binarySearch (2 overloads)" add more testcases, fix func add specification of the feature [skip ci] update the spec reworked the implementation + more tests fix ambiguity handling and reporting, including adding RST explicit hyperlinks to ambiguity printing Default priorities now conform to the `docgen.rst` spec also. enable `doc2tex` fix generics parameters for types some cleanups and clarifications tests on inclusion of .rst & .nim into .nim update forgotten files for tests --- compiler/docgen.nim | 154 ++++++++- compiler/lineinfos.nim | 2 + compiler/typesrenderer.nim | 66 +++- config/nimdoc.cfg | 11 + config/nimdoc.tex.cfg | 10 +- doc/docgen.rst | 162 +++++++++ doc/nimdoc.css | 4 + lib/packages/docutils/dochelpers.nim | 262 ++++++++++++++ lib/packages/docutils/rst.nim | 321 ++++++++++++++---- lib/packages/docutils/rstast.nim | 3 + lib/packages/docutils/rstgen.nim | 27 +- .../expected/index.html | 4 + nimdoc/testproject/expected/nimdoc.out.css | 4 + .../expected/subdir/subdir_b/utils.html | 295 +++++++++++++++- .../expected/subdir/subdir_b/utils.idx | 14 + nimdoc/testproject/expected/testproject.html | 208 +++++++++++- nimdoc/testproject/expected/theindex.html | 52 ++- nimdoc/testproject/subdir/subdir_b/utils.nim | 63 ++++ .../subdir/subdir_b/utils_helpers.nim | 1 + .../subdir/subdir_b/utils_overview.rst | 8 + tests/stdlib/trst.nim | 233 ++++++++++++- tests/stdlib/trstgen.nim | 19 +- 22 files changed, 1782 insertions(+), 141 deletions(-) create mode 100644 lib/packages/docutils/dochelpers.nim create mode 100644 nimdoc/testproject/subdir/subdir_b/utils_helpers.nim create mode 100644 nimdoc/testproject/subdir/subdir_b/utils_overview.rst diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 1acfc7489e11..2639840d1004 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -13,7 +13,7 @@ import ast, strutils, strtabs, algorithm, sequtils, options, msgs, os, idents, wordrecg, syntaxes, renderer, lexer, - packages/docutils/rst, packages/docutils/rstgen, + packages/docutils/[rst, rstgen, dochelpers], json, xmltree, trees, types, typesrenderer, astalgo, lineinfos, intsets, pathutils, tables, nimpaths, renderverbatim, osproc @@ -44,8 +44,14 @@ type ## runnableExamples). substitutions: seq[string] ## Variable names in `doc.item`... sortName: string ## The string used for sorting in output + info: rstast.TLineInfo ## place where symbol was defined (for messages) + anchor: string ## e.g. HTML anchor + name: string ## short name of the symbol, not unique + ## (includes backticks ` if present) + detailedName: string ## longer name like `proc search(x: int): int` ModSection = object ## Section like Procs, Types, etc. - secItems: seq[Item] ## Pre-processed items. + secItems: Table[string, seq[Item]] + ## Map basic name -> pre-processed items. finalMarkup: string ## The items, after RST pass 2 and rendering. ModSections = array[TSymKind, ModSection] TocItem = object ## HTML TOC item @@ -91,12 +97,22 @@ type thisDir*: AbsoluteDir exampleGroups: OrderedTable[string, ExampleGroup] wroteSupportFiles*: bool + nimToRstFid: Table[lineinfos.FileIndex, rstast.FileIndex] + ## map Nim FileIndex -> RST one, it's needed because we keep them separate PDoc* = ref TDocumentor ## Alias to type less. proc add(dest: var ItemPre, rst: PRstNode) = dest.add ItemFragment(isRst: true, rst: rst) proc add(dest: var ItemPre, str: string) = dest.add ItemFragment(isRst: false, str: str) +proc addRstFileIndex(d: PDoc, info: lineinfos.TLineInfo): rstast.FileIndex = + let invalid = rstast.FileIndex(-1) + result = d.nimToRstFid.getOrDefault(info.fileIndex, default = invalid) + if result == invalid: + let fname = toFullPath(d.conf, info) + result = addFilename(d.sharedState, fname) + d.nimToRstFid[info.fileIndex] = result + proc cmpDecimalsIgnoreCase(a, b: string): int = ## For sorting with correct handling of cases like 'uint8' and 'uint16'. ## Also handles leading zeros well (however note that leading zeros are @@ -223,6 +239,7 @@ template declareClosures = of meFootnoteMismatch: k = errRstFootnoteMismatch of mwRedefinitionOfLabel: k = warnRstRedefinitionOfLabel of mwUnknownSubstitution: k = warnRstUnknownSubstitutionX + of mwAmbiguousLink: k = warnRstAmbiguousLink of mwBrokenLink: k = warnRstBrokenLink of mwUnsupportedLanguage: k = warnRstLanguageXNotSupported of mwUnsupportedField: k = warnRstFieldXNotSupported @@ -236,7 +253,7 @@ template declareClosures = result = getCurrentDir() / s if not fileExists(result): result = "" -proc parseRst(text, filename: string, +proc parseRst(text: string, line, column: int, conf: ConfigRef, sharedState: PRstSharedState): PRstNode = declareClosures() @@ -352,7 +369,8 @@ proc getVarIdx(varnames: openArray[string], id: string): int = proc genComment(d: PDoc, n: PNode): PRstNode = if n.comment.len > 0: - result = parseRst(n.comment, toFullPath(d.conf, n.info), + d.sharedState.currFileIdx = addRstFileIndex(d, n.info) + result = parseRst(n.comment, toLinenumber(n.info), toColumn(n.info) + DocColOffset, d.conf, d.sharedState) @@ -885,6 +903,57 @@ proc genSeeSrc(d: PDoc, path: string, line: int): string = "path", path.string, "line", $line, "url", gitUrl, "commit", commit, "devel", develBranch]]) +proc symbolPriority(k: TSymKind): int = + result = case k + of skMacro: -3 + of skTemplate: -2 + of skIterator: -1 + else: 0 # including skProc which have higher priority + # documentation itself has even higher priority 1 + +proc toLangSymbol(k: TSymKind, n: PNode, baseName: string): LangSymbol = + ## Converts symbol info (names/types/parameters) in `n` into format + ## `LangSymbol` convenient for ``rst.nim``/``dochelpers.nim``. + result.name = baseName.nimIdentNormalize + result.symKind = k.toHumanStr + if k in routineKinds: + var + paramTypes: seq[string] + renderParamTypes(paramTypes, n[paramsPos], toNormalize=true) + let paramNames = renderParamNames(n[paramsPos], toNormalize=true) + # In some rare cases (system.typeof) parameter type is not set for default: + doAssert paramTypes.len <= paramNames.len + for i in 0 ..< paramNames.len: + if i < paramTypes.len: + result.parameters.add (paramNames[i], paramTypes[i]) + else: + result.parameters.add (paramNames[i], "") + result.parametersProvided = true + + result.outType = renderOutType(n[paramsPos], toNormalize=true) + + if k in {skProc, skFunc, skType, skIterator}: + # Obtain `result.generics` + # Use `n[miscPos]` since n[genericParamsPos] does not contain constraints + var genNode: PNode = nil + if k == skType: + genNode = n[1] # FIXME: what is index 1? + else: + if n[miscPos].kind != nkEmpty: + genNode = n[miscPos][1] # FIXME: what is index 1? + if genNode != nil: + var literal = "" + var r: TSrcGen + initTokRender(r, genNode, {renderNoBody, renderNoComments, + renderNoPragmas, renderNoProcDefs}) + var kind = tkEof + while true: + getNextTok(r, kind, literal) + if kind == tkEof: + break + if kind != tkSpaces: + result.generics.add(literal.nimIdentNormalize) + proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) = if (docFlags != kForceExport) and not isVisible(d, nameNode): return let @@ -915,6 +984,8 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) = inc(d.id) let plainNameEsc = esc(d.target, plainName.strip) + detailedName = k.toHumanStr & " " & ( + if k in routineKinds: plainName else: name) uniqueName = if k in routineKinds: plainNameEsc else: name sortName = if k in routineKinds: plainName.strip else: name cleanPlainSymbol = renderPlainSymbolName(nameNode) @@ -923,20 +994,32 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) = symbolOrId = d.newUniquePlainSymbol(complexSymbol) symbolOrIdEnc = encodeUrl(symbolOrId, usePlus = false) deprecationMsg = genDeprecationMsg(d, pragmaNode) + rstLangSymbol = toLangSymbol(k, n, cleanPlainSymbol) + + # we generate anchors automatically for subsequent use in doc comments + let lineinfo = rstast.TLineInfo( + line: nameNode.info.line, col: nameNode.info.col, + fileIndex: addRstFileIndex(d, nameNode.info)) + addAnchorNim(d.sharedState, refn = symbolOrId, tooltip = detailedName, + rstLangSymbol, priority = symbolPriority(k), info = lineinfo) nodeToHighlightedHtml(d, n, result, {renderNoBody, renderNoComments, renderDocComments, renderSyms}, symbolOrIdEnc) let seeSrc = genSeeSrc(d, toFullPath(d.conf, n.info), n.info.line.int) - d.section[k].secItems.add Item( + d.section[k].secItems.mgetOrPut(cleanPlainSymbol, newSeq[Item]()).add Item( descRst: comm, sortName: sortName, + info: lineinfo, + anchor: symbolOrId, + detailedName: detailedName, + name: name, substitutions: @[ - "name", name, "uniqueName", uniqueName, + "uniqueName", uniqueName, "header", result, "itemID", $d.id, "header_plain", plainNameEsc, "itemSym", cleanPlainSymbol, - "itemSymOrID", symbolOrId, "itemSymEnc", plainSymbolEnc, + "itemSymEnc", plainSymbolEnc, "itemSymOrIDEnc", symbolOrIdEnc, "seeSrc", seeSrc, "deprecationMsg", deprecationMsg]) @@ -1184,6 +1267,11 @@ proc generateDoc*(d: PDoc, n, orig: PNode, docFlags: DocFlags = kDefault) = if comm.len != 0: d.modDescPre.add(comm) else: discard +proc overloadGroupName(s: string, k: TSymKind): string = + ## Turns a name like `f` into anchor `f-procs-all` + #s & " " & k.toHumanStr & "s all" + s & "-" & k.toHumanStr & "s-all" + proc finishGenerateDoc*(d: var PDoc) = ## Perform 2nd RST pass for resolution of links/footnotes/headings... # copy file map `filenames` to ``rstgen.nim`` for its warnings @@ -1197,6 +1285,21 @@ proc finishGenerateDoc*(d: var PDoc) = break preparePass2(d.sharedState, firstRst) + # add anchors to overload groups before RST resolution + for k in TSymKind: + if k in routineKinds: + for plainName, overloadChoices in d.section[k].secItems: + if overloadChoices.len > 1: + let refn = overloadGroupName(plainName, k) + let tooltip = "$1 ($2 overloads)" % [ + k.toHumanStr & " " & plainName, $overloadChoices.len] + addAnchorNim(d.sharedState, refn, tooltip, + LangSymbol(symKind: k.toHumanStr, name: plainName, + isGroup: true), + priority = symbolPriority(k), + # select index `0` just to have any meaningful warning: + info = overloadChoices[0].info) + # Finalize fragments of ``.nim`` or ``.rst`` file proc renderItemPre(d: PDoc, fragments: ItemPre, result: var string) = for f in fragments: @@ -1207,14 +1310,33 @@ proc finishGenerateDoc*(d: var PDoc) = of false: result &= f.str proc cmp(x, y: Item): int = cmpDecimalsIgnoreCase(x.sortName, y.sortName) for k in TSymKind: - for item in d.section[k].secItems.sorted(cmp): - var itemDesc: string - renderItemPre(d, item.descRst, itemDesc) - d.section[k].finalMarkup.add( - getConfigVar(d.conf, "doc.item") % ( - item.substitutions & @["desc", itemDesc])) - itemDesc = "" - d.section[k].secItems.setLen 0 + # add symbols to section for each `k`, while optionally wrapping + # overloadable items with the same basic name by ``doc.item2`` + let overloadableNames = toSeq(keys(d.section[k].secItems)) + for plainName in overloadableNames.sorted(cmpDecimalsIgnoreCase): + var overloadChoices = d.section[k].secItems[plainName] + overloadChoices.sort(cmp) + var nameContent = "" + for item in overloadChoices: + var itemDesc: string + renderItemPre(d, item.descRst, itemDesc) + nameContent.add( + getConfigVar(d.conf, "doc.item") % ( + item.substitutions & @[ + "desc", itemDesc, + "name", item.name, + "itemSymOrID", item.anchor])) + if k in routineKinds: + let plainNameEsc1 = esc(d.target, plainName.strip) + let plainNameEsc2 = esc(d.target, plainName.strip, escMode=emUrl) + d.section[k].finalMarkup.add( + getConfigVar(d.conf, "doc.item2") % ( + @["header_plain", plainNameEsc1, + "overloadGroupName", overloadGroupName(plainNameEsc2, k), + "content", nameContent])) + else: + d.section[k].finalMarkup.add(nameContent) + d.section[k].secItems.clear renderItemPre(d, d.modDescPre, d.modDescFinal) d.modDescPre.setLen 0 d.hasToc = d.hasToc or d.sharedState.hasToc @@ -1493,7 +1615,7 @@ proc commandRstAux(cache: IdentCache, conf: ConfigRef; filename: AbsoluteFile, outExt: string) = var filen = addFileExt(filename, "txt") var d = newDocumentor(filen, cache, conf, outExt, isPureRst = true) - let rst = parseRst(readFile(filen.string), filen.string, + let rst = parseRst(readFile(filen.string), line=LineRstInit, column=ColRstInit, conf, d.sharedState) d.modDescPre = @[ItemFragment(isRst: true, rst: rst)] diff --git a/compiler/lineinfos.nim b/compiler/lineinfos.nim index 8bd5a0890992..e13387be6041 100644 --- a/compiler/lineinfos.nim +++ b/compiler/lineinfos.nim @@ -50,6 +50,7 @@ type warnSmallLshouldNotBeUsed = "SmallLshouldNotBeUsed", warnUnknownMagic = "UnknownMagic", warnRstRedefinitionOfLabel = "RedefinitionOfLabel", warnRstUnknownSubstitutionX = "UnknownSubstitutionX", + warnRstAmbiguousLink = "AmbiguousLink", warnRstBrokenLink = "BrokenLink", warnRstLanguageXNotSupported = "LanguageXNotSupported", warnRstFieldXNotSupported = "FieldXNotSupported", @@ -123,6 +124,7 @@ const warnUnknownMagic: "unknown magic '$1' might crash the compiler", warnRstRedefinitionOfLabel: "redefinition of label '$1'", warnRstUnknownSubstitutionX: "unknown substitution '$1'", + warnRstAmbiguousLink: "ambiguous doc link $1", warnRstBrokenLink: "broken link '$1'", warnRstLanguageXNotSupported: "language '$1' not supported", warnRstFieldXNotSupported: "field '$1' not supported", diff --git a/compiler/typesrenderer.nim b/compiler/typesrenderer.nim index 0da05d70d64f..c81ff283e598 100644 --- a/compiler/typesrenderer.nim +++ b/compiler/typesrenderer.nim @@ -11,6 +11,12 @@ import renderer, strutils, ast, types const defaultParamSeparator* = "," +template mayNormalize(s: string): string = + if toNormalize: + s.nimIdentNormalize + else: + s + proc renderPlainSymbolName*(n: PNode): string = ## Returns the first non '*' nkIdent node from the tree. ## @@ -30,24 +36,26 @@ proc renderPlainSymbolName*(n: PNode): string = result = "" #internalError(n.info, "renderPlainSymbolName() with " & $n.kind) -proc renderType(n: PNode): string = +proc renderType(n: PNode, toNormalize: bool): string = ## Returns a string with the node type or the empty string. + ## This proc should be kept in sync with `toLangSymbols` from + ## ``lib/packages/docutils/dochelpers.nim``. case n.kind: - of nkIdent: result = n.ident.s - of nkSym: result = typeToString(n.sym.typ) + of nkIdent: result = mayNormalize(n.ident.s) + of nkSym: result = mayNormalize(typeToString(n.sym.typ)) of nkVarTy: if n.len == 1: - result = renderType(n[0]) + result = renderType(n[0], toNormalize) else: result = "var" of nkRefTy: if n.len == 1: - result = "ref." & renderType(n[0]) + result = "ref." & renderType(n[0], toNormalize) else: result = "ref" of nkPtrTy: if n.len == 1: - result = "ptr." & renderType(n[0]) + result = "ptr." & renderType(n[0], toNormalize) else: result = "ptr" of nkProcTy: @@ -57,36 +65,53 @@ proc renderType(n: PNode): string = assert params.kind == nkFormalParams assert params.len > 0 result = "proc(" - for i in 1..= 3 let typePos = n.len - 2 - let typeStr = renderType(n[typePos]) + let typeStr = renderType(n[typePos], toNormalize) result = typeStr for i in 1..= 2 - result = renderType(n[0]) & '[' - for i in 1.. 1: result.add ", " - result.add(renderType(n[i])) + result.add(renderType(n[i], toNormalize)) else: result = "" -proc renderParamTypes(found: var seq[string], n: PNode) = +proc renderParamNames*(n: PNode, toNormalize=false): seq[string] = + ## Returns parameter names of routine `n`. + doAssert n.kind == nkFormalParams + case n.kind + of nkFormalParams: + for i in 1.. 0 - var typeStr = renderType(n[typePos]) + var typeStr = renderType(n[typePos], toNormalize) if typeStr.len < 1 and n[typePos+1].kind != nkEmpty: # Try with the last node, maybe its a default value. let typ = n[typePos+1].typ @@ -111,7 +136,8 @@ proc renderParamTypes(found: var seq[string], n: PNode) = found.add($n) #internalError(n.info, "renderParamTypes(found,n) with " & $n.kind) -proc renderParamTypes*(n: PNode, sep = defaultParamSeparator): string = +proc renderParamTypes*(n: PNode, sep = defaultParamSeparator, + toNormalize=false): string = ## Returns the types contained in `n` joined by `sep`. ## ## This proc expects to be passed as `n` the parameters of any callable. The @@ -120,6 +146,10 @@ proc renderParamTypes*(n: PNode, sep = defaultParamSeparator): string = ## other characters may appear too, like ``[]`` or ``|``. result = "" var found: seq[string] = @[] - renderParamTypes(found, n) + renderParamTypes(found, n, toNormalize) if found.len > 0: result = found.join(sep) + +proc renderOutType*(n: PNode, toNormalize=false): string = + assert n.kind == nkFormalParams + result = renderType(n[0], toNormalize) diff --git a/config/nimdoc.cfg b/config/nimdoc.cfg index 162b2b4a3cce..c1074f3441cb 100644 --- a/config/nimdoc.cfg +++ b/config/nimdoc.cfg @@ -56,6 +56,17 @@ $seeSrc """ +# A wrapper of a few overloaded `doc.item`s with the same basic name +# * $header_plain - see above +# * $overloadGroupName - the anchor for this whole group +# * $content - string containing `doc.item`s themselves +doc.item2 = """ + +
+$content +
+""" + # Chunk of HTML emitted for each entry in the HTML table of contents. # See doc.item for available substitution variables. diff --git a/config/nimdoc.tex.cfg b/config/nimdoc.tex.cfg index 4aff9b379f90..d7ab652ebf90 100644 --- a/config/nimdoc.tex.cfg +++ b/config/nimdoc.tex.cfg @@ -19,7 +19,8 @@ doc.section.toc = "" doc.item = """ \vspace{1em} -\phantomsection\addcontentsline{toc}{subsection}{$uniqueName} +\phantomsection\addcontentsline{toc}{subsubsection}{$uniqueName} +\label{$itemSymOrID}\hypertarget{$itemSymOrID}{} \begin{rstdocitem} $header @@ -30,6 +31,13 @@ $desc \end{addmargin} """ +doc.item2 = """ +\phantomsection\addcontentsline{toc}{subsection}{$header_plain} +\label{$overloadGroupName}\hypertarget{$overloadGroupName}{} + +$content +""" + doc.item.toc = "" doc.toc = r"\tableofcontents \newpage" diff --git a/doc/docgen.rst b/doc/docgen.rst index 2074ee06fe19..7b0680d18d2d 100644 --- a/doc/docgen.rst +++ b/doc/docgen.rst @@ -229,6 +229,168 @@ Output:: Note that the `jsondoc`:option: command outputs its JSON without pretty-printing it, while `jsondoc0`:option: outputs pretty-printed JSON. + +Referencing Nim symbols: simple documentation links +=================================================== + +You can reference Nim identifiers from Nim documentation comments, currently +only inside their ``.nim`` file (or inside a ``.rst`` file included from +the ``.nim``). The point is that such links will be resolved automatically +by `nim doc`:cmd: (or `nim jsondoc`:cmd: or `nim doc2tex`:cmd:). +This pertains to any exported symbol like `proc`, `const`, `iterator`, etc. +Syntax for referencing is basically a normal RST one: addition of +underscore `_` to a *link text*. +Link text is either one word or a group of words enclosed by backticks `\``. +Link text will be displayed *as is* while *link target* will be set to +the anchor [*]_ of Nim symbol that corresponds to link text. + +.. [*] anchors' format is described in `HTML anchor generation`_ section below. + +If you have a constant: + +.. code:: Nim + + const pi* = 3.14 + +then it should be referenced in one of the 2 forms: + +A. non-qualified (no symbol kind specification):: + pi_ +B. qualified (with symbol kind specification):: + `const pi`_ + +For routine kinds there are more options. Consider this definition: + +.. code:: Nim + + proc foo*(a: int, b: float): string + +Generally following syntax is allowed for referencing `foo`: + +* short (without parameters): + + A. non-qualified:: + + foo_ + + B. qualified:: + + `proc foo`_ + +* longer variants (with parameters): + + A. non-qualified: + + 1) specifying parameters names:: + + `foo(a, b)`_ + + 2) specifying parameters types:: + + `foo(int, float)`_ + + 3) specifying both names and types:: + + `foo(a: int, b: float)`_ + + 4) output parameter can also be specified if you wish:: + + `foo(a: int, b: float): string`_ + + B. qualified: all 4 options above are valid. + Particularly you can use the full format:: + + `proc foo(a: int, b: float): string`_ + +.. Tip:: Avoid cluttering your text with extraneous information by using + one of shorter forms:: + + binarySearch_ + `binarySearch(a, key, cmp)`_ + + Brevity is better for reading! If you use a short form and have an + ambiguity problem (see below) then just add some additional info. + +.. Warning:: An ambiguity in resolving documentation links may arise because of: + + 1. clash with other RST anchors + * manually setup anchors + * automatically set up, e.g. section names + 2. collision with other Nim symbols: + + * routines with different parameters can exist e.g. for + `proc` and `template`. In this case they are split between their + corresponding sections in output file + * because in Nim `proc` and `iterator` belong to different namespaces, + so there can be a collision even if parameters are the same. + + Qualified references are useful in this case: just disambiguate + the reference like `\`proc foo\`_`:literal: + or `\`template foo\`_`:literal: or `\`iterator foo\`_`:literal:. + + Any ambiguity is always reported with Nim compiler warnings and an anchor + with higher priority is selected. Manual anchors have highest + priority, then go automatic RST anchors; then Nim-generated anchors + (while procs have higher priority than other Nim symbol kinds). + +Generic parameters can also be used. All in all, this long form will be +recognized fine:: + + `proc binarySearch*[T; K](a: openArray[T], key: K, cmp: proc(T, K)): int`_ + +**Limitations**: + +1. The parameters of a nested routine type can be specified only with types + (without parameter names, see form A.2 above). + E.g. for this signature: + + .. code:: Nim + + proc binarySearch*[T, K](a: openArray[T]; key: K; + cmp: proc (x: T; y: K): int {.closure.}): int + ~~ ~~ ~~~~~ + + you cannot use names underlined by `~~` so it must be referenced with + ``cmp: proc(T, K)``. Hence these forms are valid:: + + `binarySearch(a: openArray[T], key: K, cmp: proc(T, K))`_ + `binarySearch(openArray[T], K, proc(T, K))`_ + `binarySearch(a, key, cmp)`_ +2. Default values in routine parameters are not recognized, one needs to + specify the type and/or name instead. E.g. for referencing `proc f(x = 7)` + use one of the mentioned forms:: + + `f(int)`_ or `f(x)`_ or `f(x: int)`_. +3. Generic parameters must be given the same way as in the + definition of referenced symbol. + + * their names should be the same + * parameters list should be given the same way, e.g. without substitutions + between commas (,) and semicolons (;). + +.. Note:: A bit special case is operators + (as their signature is also defined with `\``): + + .. code:: Nim + + func `$`(x: MyType): string + func `[]`*[T](x: openArray[T]): T + + A short form works without additional backticks:: + + `$`_ + `[]`_ + + However for fully-qualified reference copy-pasting backticks (`) into other + backticks will not work in our RST parser (because we use Markdown-like + inline markup rules). You need either to delete backticks or keep + them and escape with backslash \\:: + + no backticks: `func $`_ + escaped: `func \`$\``_ + no backticks: `func [][T](x: openArray[T]): T`_ + escaped: `func \`[]\`[T](x: openArray[T]): T`_ + Related Options =============== diff --git a/doc/nimdoc.css b/doc/nimdoc.css index 4abea9ce0a6a..331ea1a583e3 100644 --- a/doc/nimdoc.css +++ b/doc/nimdoc.css @@ -255,6 +255,10 @@ a.reference-toplevel { font-weight: bold; } +a.nimdoc { + word-spacing: 0.3em; +} + a.toc-backref { text-decoration: none; color: var(--text); } diff --git a/lib/packages/docutils/dochelpers.nim b/lib/packages/docutils/dochelpers.nim new file mode 100644 index 000000000000..7a08f74bbc6a --- /dev/null +++ b/lib/packages/docutils/dochelpers.nim @@ -0,0 +1,262 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2021 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## Integration helpers between ``docgen.nim`` and ``rst.nim``. +## +## Function `toLangSymbol(linkText)`_ produces a signature `docLink` of +## `type LangSymbol`_ in ``rst.nim``, while `match(generated, docLink)`_ +## matches it with `generated`, produced from `PNode` by ``docgen.rst``. + +import rstast + +type + LangSymbol* = object ## symbol signature in Nim + symKind*: string ## "proc", "const", etc + name*: string ## plain symbol name without any parameters + generics*: string ## generic parameters (without brackets) + isGroup*: bool ## is LangSymbol a group with overloads? + # the following fields are valid iff `isGroup` == false + # (always false when parsed by `toLangSymbol` because link like foo_ + # can point to just a single symbol foo, e.g. proc). + parametersProvided*: bool ## to disambiguate `proc f`_ and `proc f()`_ + parameters*: seq[tuple[name: string, `type`: string]] + ## name-type seq, e.g. for proc + outType*: string ## result type, e.g. for proc + +func nimIdentBackticksNormalize*(s: string): string = + ## Normalizes the string `s` as a Nim identifier. + ## + ## Unlike `nimIdentNormalize` removes spaces and backticks. + ## + ## .. Warning:: No checking (e.g. that identifiers cannot start from + ## digits or '_', or that number of backticks is even) is performed. + runnableExamples: + doAssert nimIdentBackticksNormalize("Foo_bar") == "Foobar" + doAssert nimIdentBackticksNormalize("FoO BAr") == "Foobar" + doAssert nimIdentBackticksNormalize("`Foo BAR`") == "Foobar" + doAssert nimIdentBackticksNormalize("` Foo BAR `") == "Foobar" + # not a valid identifier: + doAssert nimIdentBackticksNormalize("`_x_y`") == "_xy" + result = newString(s.len) + var firstChar = true + var j = 0 + for i in 0..len(s) - 1: + if s[i] in {'A'..'Z'}: + if not firstChar: # to lowercase + result[j] = chr(ord(s[i]) + (ord('a') - ord('A'))) + else: + result[j] = s[i] + firstChar = false + inc j + elif s[i] notin {'_', ' ', '`'}: + result[j] = s[i] + inc j + firstChar = false + elif s[i] == '_' and firstChar: + result[j] = '_' + inc j + firstChar = false + else: discard # just omit '`' or ' ' + if j != s.len: setLen(result, j) + +proc toLangSymbol*(linkText: PRstNode): LangSymbol = + ## Parses `linkText` into a more structured form using a state machine. + ## + ## This proc is designed to allow link syntax with operators even + ## without escaped backticks inside:: + ## + ## `proc *`_ + ## `proc []`_ + ## + ## This proc should be kept in sync with the `renderTypes` proc from + ## ``compiler/typesrenderer.nim``. + assert linkText.kind in {rnRef, rnInner} + + const NimDefs = ["proc", "func", "macro", "method", "iterator", + "template", "converter", "const", "type", "var"] + type + State = enum + inBeginning + afterSymKind + beforeSymbolName # auxiliary state to catch situations like `proc []`_ after space + atSymbolName + afterSymbolName + genericsPar + parameterName + parameterType + outType + var state = inBeginning + var curIdent = "" + template flushIdent() = + if curIdent != "": + case state + of inBeginning: doAssert false, "incorrect state inBeginning" + of afterSymKind: result.symKind = curIdent + of beforeSymbolName: doAssert false, "incorrect state beforeSymbolName" + of atSymbolName: result.name = curIdent.nimIdentBackticksNormalize + of afterSymbolName: doAssert false, "incorrect state afterSymbolName" + of genericsPar: result.generics = curIdent + of parameterName: result.parameters.add (curIdent, "") + of parameterType: + for a in countdown(result.parameters.len - 1, 0): + if result.parameters[a].`type` == "": + result.parameters[a].`type` = curIdent + of outType: result.outType = curIdent + curIdent = "" + var parens = 0 + let L = linkText.sons.len + template s(i: int): string = linkText.sons[i].text + var i = 0 + template nextState = + case s(i) + of " ": + if state == afterSymKind: + flushIdent + state = beforeSymbolName + of "`": + curIdent.add "`" + inc i + while i < L: # add contents between ` ` as a whole + curIdent.add s(i) + if s(i) == "`": + break + inc i + curIdent = curIdent.nimIdentBackticksNormalize + if state in {inBeginning, afterSymKind, beforeSymbolName}: + state = atSymbolName + flushIdent + state = afterSymbolName + of "[": + if state notin {inBeginning, afterSymKind, beforeSymbolName}: + inc parens + if state in {inBeginning, afterSymKind, beforeSymbolName}: + state = atSymbolName + curIdent.add s(i) + elif state in {atSymbolName, afterSymbolName} and parens == 1: + flushIdent + state = genericsPar + curIdent.add s(i) + else: curIdent.add s(i) + of "]": + if state notin {inBeginning, afterSymKind, beforeSymbolName, atSymbolName}: + dec parens + if state == genericsPar and parens == 0: + curIdent.add s(i) + flushIdent + else: curIdent.add s(i) + of "(": + inc parens + if state in {inBeginning, afterSymKind, beforeSymbolName}: + result.parametersProvided = true + state = atSymbolName + flushIdent + state = parameterName + elif state in {atSymbolName, afterSymbolName, genericsPar} and parens == 1: + result.parametersProvided = true + flushIdent + state = parameterName + else: curIdent.add s(i) + of ")": + dec parens + if state in {parameterName, parameterType} and parens == 0: + flushIdent + state = outType + else: curIdent.add s(i) + of "{": # remove pragmas + while i < L: + if s(i) == "}": + break + inc i + of ",", ";": + if state in {parameterName, parameterType} and parens == 1: + flushIdent + state = parameterName + else: curIdent.add s(i) + of "*": # skip export symbol + if state == atSymbolName: + flushIdent + state = afterSymbolName + elif state == afterSymbolName: + discard + else: curIdent.add "*" + of ":": + if state == outType: discard + elif state == parameterName: + flushIdent + state = parameterType + else: curIdent.add ":" + else: + case state + of inBeginning: + if s(i) in NimDefs: + state = afterSymKind + else: + state = atSymbolName + curIdent.add s(i) + of afterSymKind, beforeSymbolName: + state = atSymbolName + curIdent.add s(i) + of parameterType: + case s(i) + of "ref": curIdent.add "ref." + of "ptr": curIdent.add "ptr." + of "var": discard + else: curIdent.add s(i).nimIdentBackticksNormalize + of atSymbolName: + curIdent.add s(i) + else: + curIdent.add s(i).nimIdentBackticksNormalize + while i < L: + nextState + inc i + if state == afterSymKind: # treat `type`_ as link to symbol `type` + state = atSymbolName + flushIdent + result.isGroup = false + +proc match*(generated: LangSymbol, docLink: LangSymbol): bool = + ## Returns true if `generated` can be a target for `docLink`. + ## If `generated` is an overload group then only `symKind` and `name` + ## are compared for success. + result = true + if docLink.symKind != "": + if generated.symKind == "proc": + result = docLink.symKind in ["proc", "func"] + else: + result = generated.symKind == docLink.symKind + if not result: return + result = generated.name == docLink.name + if not result: return + if generated.isGroup: + # if `()` were added then it's not a reference to the whole group: + return not docLink.parametersProvided + if docLink.generics != "": + result = generated.generics == docLink.generics + if not result: return + if docLink.outType != "": + result = generated.outType == docLink.outType + if not result: return + if docLink.parametersProvided: + result = generated.parameters.len == docLink.parameters.len + if not result: return + var onlyType = false + for i in 0 ..< generated.parameters.len: + let g = generated.parameters[i] + let d = docLink.parameters[i] + if i == 0: + if g.`type` == d.name: + onlyType = true # only types, not names, are provided in `docLink` + if onlyType: + result = g.`type` == d.name: + else: + if d.`type` != "": + result = g.`type` == d.`type` + if not result: return + result = g.name == d.name + if not result: return diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index 29234f28ba51..2e908f4e5859 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -114,7 +114,7 @@ ## .. _`extra features`: ## ## Optional additional features, turned on by ``options: RstParseOption`` in -## `rstParse proc <#rstParse,string,string,int,int,bool,RstParseOptions,FindFileHandler,MsgHandler>`_: +## `proc rstParse`_: ## ## * emoji / smiley symbols ## * Markdown tables @@ -196,7 +196,7 @@ ## .. _Sphinx roles: https://www.sphinx-doc.org/en/master/usage/restructuredtext/roles.html import - os, strutils, rstast, std/enumutils, algorithm, lists, sequtils, + os, strutils, rstast, dochelpers, std/enumutils, algorithm, lists, sequtils, std/private/miscdollars, tables from highlite import SourceLanguage, getSourceLanguage @@ -231,6 +231,7 @@ type meFootnoteMismatch = "mismatch in number of footnotes and their refs: $1", mwRedefinitionOfLabel = "redefinition of label '$1'", mwUnknownSubstitution = "unknown substitution '$1'", + mwAmbiguousLink = "ambiguous doc link $1", mwBrokenLink = "broken link '$1'", mwUnsupportedLanguage = "language '$1' not supported", mwUnsupportedField = "field '$1' not supported", @@ -473,12 +474,42 @@ type hasPeers: bool # has headings on the same level of hierarchy? LevelMap = seq[LevelInfo] # Saves for each possible title adornment # style its level in the current document. + SubstitutionKind = enum + rstSubstitution = "substitution", + hyperlinkAlias = "hyperlink alias", + implicitHyperlinkAlias = "implicitly-generated hyperlink alias" Substitution = object + kind*: SubstitutionKind key*: string value*: PRstNode - AnchorSubst = tuple - mainAnchor: string - aliases: seq[string] + info*: TLineInfo # place where the substitution was defined + AnchorRule = enum + arInternalRst, ## For automatically generated RST anchors (from + ## headings, footnotes, inline internal targets): + ## case-insensitive, 1-space-significant (by RST spec) + arNim ## For anchors generated by ``docgen.rst``: Nim-style case + ## sensitivity, etc. (see `proc normalizeNimName`_ for details) + arHyperlink, ## For links with manually set anchors in + ## form `text `_ + RstAnchorKind = enum + manualDirectiveAnchor = "manual directive anchor", + manualInlineAnchor = "manual inline anchor", + footnoteAnchor = "footnote anchor", + headlineAnchor = "implicitly-generated headline anchor" + AnchorSubst = object + mainAnchor: ref string # A reference name that will be inserted directly + # into HTML/Latex. It's declared as `ref` because + # it can be shared between aliases. + info: TLineInfo # where the anchor was defined + priority: int + case kind: range[arInternalRst .. arNim] + of arInternalRst: + anchorType: RstAnchorKind + of arNim: + tooltip: string # displayed tooltip for Nim-generated anchors + langSym: LangSymbol + AnchorSubstTable = Table[string, seq[AnchorSubst]] + # use `seq` to account for duplicate anchors FootnoteType = enum fnManualNumber, # manually numbered footnote like [3] fnAutoNumber, # auto-numbered footnote [#] @@ -505,7 +536,8 @@ type currRoleKind: RstNodeKind # ... and its node kind subs: seq[Substitution] # substitutions refs*: seq[Substitution] # references - anchors*: seq[AnchorSubst] # internal target substitutions + anchors*: AnchorSubstTable + # internal target substitutions lineFootnoteNum: seq[TLineInfo] # footnote line, auto numbers .. [#] lineFootnoteNumRef: seq[TLineInfo] # footnote line, their reference [#]_ currFootnoteNumRef: int # ... their counter for `resolveSubs` @@ -518,7 +550,7 @@ type findFile: FindFileHandler # How to find files. filenames*: RstFileTable # map file name <-> FileIndex (for storing # file names for warnings after 1st stage) - currFileIdx: FileIndex # current index in `filesnames` + currFileIdx*: FileIndex # current index in `filenames` hasToc*: bool PRstSharedState* = ref RstSharedState @@ -532,6 +564,7 @@ type ## in case of error/warning reporting to ## (relative) line/column of the token. curAnchor*: string # variable to track latest anchor in s.anchors + curAnchorName*: string # corresponding name in human-readable format EParseError* = object of ValueError @@ -590,13 +623,16 @@ proc whichRoleAux(sym: string): RstNodeKind = proc len(filenames: RstFileTable): int = filenames.idxToFilename.len -proc setCurrFilename(s: PRstSharedState, file1: string) = +proc addFilename*(s: PRstSharedState, file1: string): FileIndex = + ## Returns index of filename, adding it if it has not been used before let nextIdx = s.filenames.len.FileIndex - let v = getOrDefault(s.filenames.filenameToIdx, file1, default = nextIdx) - if v == nextIdx: - s.filenames.filenameToIdx[file1] = v + result = getOrDefault(s.filenames.filenameToIdx, file1, default = nextIdx) + if result == nextIdx: + s.filenames.filenameToIdx[file1] = result s.filenames.idxToFilename.add file1 - s.currFileIdx = v + +proc setCurrFilename*(s: PRstSharedState, file1: string) = + s.currFileIdx = addFilename(s, file1) proc getFilename(filenames: RstFileTable, fid: FileIndex): string = doAssert(0 <= fid.int and fid.int < filenames.len, @@ -730,6 +766,8 @@ proc initParser(p: var RstParser, sharedState: PRstSharedState) = p.s = sharedState proc addNodesAux(n: PRstNode, result: var string) = + if n == nil: + return if n.kind == rnLeaf: result.add(n.text) else: @@ -738,6 +776,11 @@ proc addNodesAux(n: PRstNode, result: var string) = proc addNodes(n: PRstNode): string = n.addNodesAux(result) +proc linkName(n: PRstNode): string = + ## Returns a normalized reference name, see: + ## https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html#reference-names + n.addNodes.toLowerAscii + proc rstnodeToRefnameAux(n: PRstNode, r: var string, b: var bool) = template special(s) = if b: @@ -804,15 +847,26 @@ proc findSub(s: PRstSharedState, n: PRstNode): int = return i result = -1 +proc lineInfo(p: RstParser, iTok: int): TLineInfo = + result.col = int16(p.col + p.tok[iTok].col) + result.line = uint16(p.line + p.tok[iTok].line) + result.fileIndex = p.s.currFileIdx + +proc lineInfo(p: RstParser): TLineInfo = lineInfo(p, p.idx) +# TODO: we need this simplification because we don't preserve exact starting +# token of currently parsed element: +proc prevLineInfo(p: RstParser): TLineInfo = lineInfo(p, p.idx-1) + proc setSub(p: var RstParser, key: string, value: PRstNode) = var length = p.s.subs.len for i in 0 ..< length: if key == p.s.subs[i].key: p.s.subs[i].value = value return - p.s.subs.add(Substitution(key: key, value: value)) + p.s.subs.add(Substitution(key: key, value: value, info: prevLineInfo(p))) -proc setRef(p: var RstParser, key: string, value: PRstNode) = +proc setRef(p: var RstParser, key: string, value: PRstNode, + refType: SubstitutionKind) = var length = p.s.refs.len for i in 0 ..< length: if key == p.s.refs[i].key: @@ -820,37 +874,111 @@ proc setRef(p: var RstParser, key: string, value: PRstNode) = rstMessage(p, mwRedefinitionOfLabel, key) p.s.refs[i].value = value return - p.s.refs.add(Substitution(key: key, value: value)) + p.s.refs.add(Substitution(kind: refType, key: key, value: value, + info: prevLineInfo(p))) -proc findRef(s: PRstSharedState, key: string): PRstNode = +proc findRef(s: PRstSharedState, key: string): seq[Substitution] = for i in countup(0, high(s.refs)): if key == s.refs[i].key: - return s.refs[i].value - -proc addAnchor(p: var RstParser, refn: string, reset: bool) = - ## add anchor `refn` to anchor aliases and update last anchor ``curAnchor`` - if p.curAnchor == "": - p.s.anchors.add (refn, @[refn]) + result.add s.refs[i] + +# Ambiguity in links: we don't follow procedure of removing implicit targets +# defined in https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html#implicit-hyperlink-targets +# Instead we just give explicit links a higher priority than to implicit ones +# and report ambiguities as warnings. Hopefully it is easy to remove +# ambiguities manually. Nim auto-generated links from ``docgen.nim`` +# have lowest priority: 1 (for procs) and below for other symbol types. + +proc refPriority(k: SubstitutionKind): int = + case k + of rstSubstitution: result = 8 + of hyperlinkAlias: result = 7 + of implicitHyperlinkAlias: result = 2 + +proc internalRefPriority(k: RstAnchorKind): int = + case k + of manualDirectiveAnchor: result = 6 + of manualInlineAnchor: result = 5 + of footnoteAnchor: result = 4 + of headlineAnchor: result = 3 + +proc addAnchorRst(p: var RstParser, name: string, refn: string, reset: bool, + anchorType: RstAnchorKind) = + ## Adds anchor `refn` with an alias `name` and + ## updates the corresponding `curAnchor` / `curAnchorName`. + let prio = internalRefPriority(anchorType) + if p.curAnchorName == "": + var anchRef = new string + anchRef[] = refn + p.s.anchors.mgetOrPut(name, newSeq[AnchorSubst]()).add( + AnchorSubst(kind: arInternalRst, mainAnchor: anchRef, priority: prio, + info: prevLineInfo(p), anchorType: anchorType)) else: - p.s.anchors[^1].mainAnchor = refn - p.s.anchors[^1].aliases.add refn + # override previous mainAnchor by `ref` in all aliases + var anchRef = p.s.anchors[p.curAnchorName][0].mainAnchor + anchRef[] = refn + p.s.anchors.mgetOrPut(name, newSeq[AnchorSubst]()).add( + AnchorSubst(kind: arInternalRst, mainAnchor: anchRef, priority: prio, + info: prevLineInfo(p), anchorType: anchorType)) if reset: p.curAnchor = "" + p.curAnchorName = "" else: p.curAnchor = refn + p.curAnchorName = name + +proc addAnchorNim*(s: var PRstSharedState, refn: string, tooltip: string, + langSym: LangSymbol, priority: int, + info: TLineInfo) = + ## Adds an anchor `refn` (`mainAnchor`), which follows + ## the rule `arNim` (i.e. a symbol in ``*.nim`` file) + var anchRef = new string + anchRef[] = refn + s.anchors.mgetOrPut(langSym.name, newSeq[AnchorSubst]()).add( + AnchorSubst(kind: arNim, mainAnchor: anchRef, langSym: langSym, + tooltip: tooltip, priority: priority, + info: info)) + +proc findMainAnchorNim(s: PRstSharedState, signature: PRstNode, + info: TLineInfo): + seq[AnchorSubst] = + let langSym = toLangSymbol(signature) + let substitutions = s.anchors.getOrDefault(langSym.name, + newSeq[AnchorSubst]()) + if substitutions.len == 0: + return + # map symKind (like "proc") -> found symbols/groups: + var found: Table[string, seq[AnchorSubst]] + for s in substitutions: + if s.kind == arNim: + if match(s.langSym, langSym): + found.mgetOrPut(s.langSym.symKind, newSeq[AnchorSubst]()).add s + for symKind, sList in found: + if sList.len == 1: + result.add sList[0] + else: # > 1, there are overloads, potential ambiguity in this `symKind` + if langSym.parametersProvided: + # there are non-group signatures, select only them + for s in sList: + if not s.langSym.isGroup: + result.add s + else: # when there are many overloads a link like foo_ points to all + # of them, so selecting the group + var foundGroup = true + for s in sList: + if s.langSym.isGroup: + result.add s + foundGroup = true + break + doAssert foundGroup, "docgen has not generated the group" -proc findMainAnchor(s: PRstSharedState, refn: string): string = - for subst in s.anchors: - if subst.mainAnchor == refn: # no need to rename - result = subst.mainAnchor - break - var toLeave = false - for anchor in subst.aliases: - if anchor == refn: # this anchor will be named as mainAnchor - result = subst.mainAnchor - toLeave = true - if toLeave: - break +proc findMainAnchorRst(s: PRstSharedState, linkText: string, info: TLineInfo): + seq[AnchorSubst] = + let name = linkText.toLowerAscii + let substitutions = s.anchors.getOrDefault(name, newSeq[AnchorSubst]()) + for s in substitutions: + if s.kind == arInternalRst: + result.add s proc addFootnoteNumManual(p: var RstParser, num: int) = ## add manually-numbered footnote @@ -860,13 +988,6 @@ proc addFootnoteNumManual(p: var RstParser, num: int) = return p.s.footnotes.add((fnManualNumber, num, -1, -1, $num)) -proc lineInfo(p: RstParser, iTok: int): TLineInfo = - result.col = int16(p.col + p.tok[iTok].col) - result.line = uint16(p.line + p.tok[iTok].line) - result.fileIndex = p.s.currFileIdx - -proc lineInfo(p: RstParser): TLineInfo = lineInfo(p, p.idx) - proc addFootnoteNumAuto(p: var RstParser, label: string) = ## add auto-numbered footnote. ## Empty label [#] means it'll be resolved by the occurrence. @@ -989,6 +1110,7 @@ proc newRstNodeA(p: var RstParser, kind: RstNodeKind): PRstNode = if p.curAnchor != "": result.anchor = p.curAnchor p.curAnchor = "" + p.curAnchorName = "" template newLeaf(s: string): PRstNode = newRstLeaf(s) @@ -1255,7 +1377,7 @@ proc parsePostfix(p: var RstParser, n: PRstNode): PRstNode = else: newKind = rnHyperlink newSons = @[a, b] - setRef(p, rstnodeToRefname(a), b) + setRef(p, rstnodeToRefname(a), b, implicitHyperlinkAlias) result = newRstNode(newKind, newSons) else: # some link that will be resolved in `resolveSubs` newKind = rnRef @@ -1562,7 +1684,8 @@ proc parseInline(p: var RstParser, father: PRstNode) = inc p.idx parseUntil(p, n, "`", false) let refn = rstnodeToRefname(n) - p.s.anchors.add (refn, @[refn]) + addAnchorRst(p, name = linkName(n), refn = refn, reset = true, + anchorType=manualInlineAnchor) father.add(n) elif roSupportMarkdown in p.s.options and currentTok(p).symbol == "```": inc p.idx @@ -2084,7 +2207,8 @@ proc parseHeadline(p: var RstParser): PRstNode = result.level = getLevel(p, c, hasOverline=false) checkHeadingHierarchy(p, result.level) p.s.hCurLevel = result.level - addAnchor(p, rstnodeToRefname(result), reset=true) + addAnchorRst(p, linkName(result), rstnodeToRefname(result), reset=true, + anchorType=headlineAnchor) proc parseOverline(p: var RstParser): PRstNode = var c = currentTok(p).symbol[0] @@ -2106,7 +2230,8 @@ proc parseOverline(p: var RstParser): PRstNode = if currentTok(p).kind == tkAdornment: inc p.idx if currentTok(p).kind == tkIndent: inc p.idx - addAnchor(p, rstnodeToRefname(result), reset=true) + addAnchorRst(p, linkName(result), rstnodeToRefname(result), reset=true, + anchorType=headlineAnchor) type IntSeq = seq[int] @@ -2837,7 +2962,7 @@ proc parseFootnote(p: var RstParser): PRstNode = anchor.add $p.s.lineFootnoteSym.len of fnCitation: anchor.add rstnodeToRefname(label) - addAnchor(p, anchor, reset=true) + addAnchorRst(p, anchor, anchor, reset=true, anchorType=footnoteAnchor) result.anchor = anchor if currentTok(p).kind == tkWhite: inc p.idx discard parseBlockContent(p, result, parseSectionWrapper) @@ -2858,13 +2983,23 @@ proc parseDotDot(p: var RstParser): PRstNode = elif match(p, p.idx, " _"): # hyperlink target: inc p.idx, 2 - var a = getReferenceName(p, ":") + var ending = ":" + if currentTok(p).symbol == "`": + inc p.idx + ending = "`" + var a = getReferenceName(p, ending) + if ending == "`": + if currentTok(p).symbol == ":": + inc p.idx + else: + rstMessage(p, meExpected, ":") if currentTok(p).kind == tkWhite: inc p.idx var b = untilEol(p) if len(b) == 0: # set internal anchor - addAnchor(p, rstnodeToRefname(a), reset=false) + addAnchorRst(p, linkName(a), rstnodeToRefname(a), reset=false, + anchorType=manualDirectiveAnchor) else: # external hyperlink - setRef(p, rstnodeToRefname(a), b) + setRef(p, rstnodeToRefname(a), b, refType=hyperlinkAlias) elif match(p, p.idx, " |"): # substitution definitions: inc p.idx, 2 @@ -2892,7 +3027,7 @@ proc rstParsePass1*(fragment: string, sharedState: PRstSharedState): PRstNode = ## Parses an RST `fragment`. ## The result should be further processed by - ## `preparePass2` and `resolveSubs` (which is pass 2). + ## preparePass2_ and resolveSubs_ (which is pass 2). var p: RstParser initParser(p, sharedState) p.line = line @@ -2905,6 +3040,65 @@ proc preparePass2*(s: PRstSharedState, mainNode: PRstNode) = countTitles(s, mainNode) orderFootnotes(s) +proc resolveLink(s: PRstSharedState, n: PRstNode) : PRstNode = + # Associate this link alias with its target and change node kind to + # rnHyperlink or rnInternalRef appropriately. + type LinkDef = object + ar: AnchorRule + priority: int + tooltip: string + target: PRstNode + info: TLineInfo + proc cmp(x, y: LinkDef): int = + result = cmp(x.priority, y.priority) + if result == 0: + result = cmp(x.target, y.target) + var foundLinks: seq[LinkDef] + let text = newRstNode(rnInner, n.sons) + let refn = rstnodeToRefname(n) + var hyperlinks = findRef(s, refn) + for y in hyperlinks: + foundLinks.add LinkDef(ar: arHyperlink, priority: refPriority(y.kind), + target: y.value, info: y.info, + tooltip: "(" & $y.kind & ")") + let substRst = findMainAnchorRst(s, text.addNodes, n.info) + for subst in substRst: + foundLinks.add LinkDef(ar: arInternalRst, priority: subst.priority, + target: newLeaf(subst.mainAnchor[]), + info: subst.info, + tooltip: "(" & $subst.anchorType & ")") + if roNimFile in s.options: + let substNim = findMainAnchorNim(s, signature=text, n.info) + for subst in substNim: + foundLinks.add LinkDef(ar: arNim, priority: subst.priority, + target: newLeaf(subst.mainAnchor[]), + info: subst.info, tooltip: subst.tooltip) + foundLinks.sort(cmp = cmp, order = Descending) + let linkText = addNodes(n) + if foundLinks.len >= 1: + let kind = if foundLinks[0].ar == arHyperlink: rnHyperlink + elif foundLinks[0].ar == arNim: rnNimdocRef + else: rnInternalRef + result = newRstNode(kind) + result.sons = @[text, foundLinks[0].target] + if kind == rnNimdocRef: result.tooltip = foundLinks[0].tooltip + if foundLinks.len > 1: # report ambiguous link + var targets = newSeq[string]() + for l in foundLinks: + var t = " " + if s.filenames.len > 1: + t.add getFilename(s.filenames, l.info.fileIndex) + let n = l.info.line + let c = l.info.col + ColRstOffset + t.add "($1, $2): $3" % [$n, $c, l.tooltip] + targets.add t + rstMessage(s.filenames, s.msgHandler, n.info, mwAmbiguousLink, + "`$1`\n clash:\n$2" % [ + linkText, targets.join("\n")]) + else: # nothing found + result = n + rstMessage(s.filenames, s.msgHandler, n.info, mwBrokenLink, linkText) + proc resolveSubs*(s: PRstSharedState, n: PRstNode): PRstNode = ## Makes pass 2 of RST parsing. ## Resolves substitutions and anchor aliases, groups footnotes. @@ -2933,21 +3127,7 @@ proc resolveSubs*(s: PRstSharedState, n: PRstNode): PRstNode = elif s.hTitleCnt == 0: n.level += 1 of rnRef: - let refn = rstnodeToRefname(n) - var y = findRef(s, refn) - if y != nil: - result = newRstNode(rnHyperlink) - let text = newRstNode(rnInner, n.sons) - result.sons = @[text, y] - else: - let anchor = findMainAnchor(s, refn) - if anchor != "": - result = newRstNode(rnInternalRef) - let text = newRstNode(rnInner, n.sons) - result.sons = @[text, # visible text of reference - newLeaf(anchor)] # link itself - else: - rstMessage(s.filenames, s.msgHandler, n.info, mwBrokenLink, refn) + result = resolveLink(s, n) of rnFootnote: var (fnType, num) = getFootnoteType(n.sons[0]) case fnType @@ -2993,9 +3173,10 @@ proc resolveSubs*(s: PRstSharedState, n: PRstNode): PRstNode = of fnCitation: result.add n.sons[0] refn.add rstnodeToRefname(n) - let anch = findMainAnchor(s, refn) - if anch != "": - result.add newLeaf(anch) # add link + # TODO: correctly report ambiguities + let anchorInfo = findMainAnchorRst(s, refn, n.info) + if anchorInfo.len != 0: + result.add newLeaf(anchorInfo[0].mainAnchor[]) # add link else: rstMessage(s.filenames, s.msgHandler, n.info, mwBrokenLink, refn) result.add newLeaf(refn) # add link diff --git a/lib/packages/docutils/rstast.nim b/lib/packages/docutils/rstast.nim index e1ed7c0991fd..bc7b9a650179 100644 --- a/lib/packages/docutils/rstast.nim +++ b/lib/packages/docutils/rstast.nim @@ -43,6 +43,7 @@ type rnFootnoteGroup, # footnote group - exists for a purely stylistic # reason: to display a few footnotes as 1 block rnStandaloneHyperlink, rnHyperlink, rnRef, rnInternalRef, rnFootnoteRef, + rnNimdocRef, # reference to automatically generated Nim symbol rnDirective, # a general directive rnDirArg, # a directive argument (for some directives). # here are directives that are not rnDirective: @@ -104,6 +105,8 @@ type rnInterpretedText, rnField, rnInlineCode, rnCodeBlock, rnFootnoteRef: info*: TLineInfo ## To have line/column info for warnings at ## nodes that are post-processed after parsing + of rnNimdocRef: + tooltip*: string else: discard anchor*: string ## anchor, internal link target diff --git a/lib/packages/docutils/rstgen.nim b/lib/packages/docutils/rstgen.nim index 8eac373072c9..a556aa2e3bb4 100644 --- a/lib/packages/docutils/rstgen.nim +++ b/lib/packages/docutils/rstgen.nim @@ -60,8 +60,8 @@ type MetaEnum* = enum metaNone, metaTitle, metaSubtitle, metaAuthor, metaVersion - EscapeMode = enum # in Latex text inside options [] and URLs is - # escaped slightly differently than in normal text + EscapeMode* = enum # in Latex text inside options [] and URLs is + # escaped slightly differently than in normal text emText, emOption, emUrl # emText is currently used for code also RstGenerator* = object of RootObj @@ -201,7 +201,9 @@ proc addTexChar(dest: var string, c: char, escMode: EscapeMode) = ## All escapes that need to work in text and code blocks (`emText` mode) ## should start from \ (to be compatible with fancyvrb/fvextra). case c - of '_', '$', '&', '#', '%': add(dest, "\\" & c) + of '_', '&', '#', '%': add(dest, "\\" & c) + # commands \label and \pageref don't accept \$ by some reason but OK with $: + of '$': (if escMode == emUrl: add(dest, c) else: add(dest, "\\" & c)) # \~ and \^ have a special meaning unless they are followed by {} of '~', '^': add(dest, "\\" & c & "{}") # Latex loves to substitute ` to opening quote, even in texttt mode! @@ -1180,7 +1182,8 @@ proc renderAdmonition(d: PDoc, n: PRstNode, result: var string) = "$1\n\\end{rstadmonition}\n", result) -proc renderHyperlink(d: PDoc, text, link: PRstNode, result: var string, external: bool) = +proc renderHyperlink(d: PDoc, text, link: PRstNode, result: var string, + external: bool, nimdoc = false, tooltip="") = var linkStr = "" block: let mode = d.escMode @@ -1189,14 +1192,19 @@ proc renderHyperlink(d: PDoc, text, link: PRstNode, result: var string, external d.escMode = mode var textStr = "" renderRstToOut(d, text, textStr) + let nimDocStr = if nimdoc: " nimdoc" else: "" + var tooltipStr = "" + if tooltip != "": + tooltipStr = """ title="$1"""" % [ esc(d.target, tooltip) ] if external: dispA(d.target, result, - "$1", - "\\href{$2}{$1}", [textStr, linkStr]) + "$1", + "\\href{$2}{$1}", [textStr, linkStr, nimDocStr, tooltipStr]) else: dispA(d.target, result, - "$1", - "\\hyperlink{$2}{$1} (p.~\\pageref{$2})", [textStr, linkStr]) + "$1", + "\\hyperlink{$2}{$1} (p.~\\pageref{$2})", + [textStr, linkStr, nimDocStr, tooltipStr]) proc renderRstToOut(d: PDoc, n: PRstNode, result: var string) = if n == nil: return @@ -1329,6 +1337,9 @@ proc renderRstToOut(d: PDoc, n: PRstNode, result: var string) = renderHyperlink(d, text=n.sons[0], link=n.sons[0], result, external=true) of rnInternalRef: renderHyperlink(d, text=n.sons[0], link=n.sons[1], result, external=false) + of rnNimdocRef: + renderHyperlink(d, text=n.sons[0], link=n.sons[1], result, external=false, + nimdoc=true, tooltip=n.tooltip) of rnHyperlink: renderHyperlink(d, text=n.sons[0], link=n.sons[1], result, external=true) of rnFootnoteRef: diff --git a/nimdoc/test_out_index_dot_html/expected/index.html b/nimdoc/test_out_index_dot_html/expected/index.html index 3499b5326d27..69dd2eb2e7f2 100644 --- a/nimdoc/test_out_index_dot_html/expected/index.html +++ b/nimdoc/test_out_index_dot_html/expected/index.html @@ -121,6 +121,8 @@

nimdoc/test_out_index_dot_html/foo

Procs

+ +
proc foo() {....raises: [], tags: [].}
@@ -130,6 +132,8 @@

Procs

+
+
diff --git a/nimdoc/testproject/expected/nimdoc.out.css b/nimdoc/testproject/expected/nimdoc.out.css index 4abea9ce0a6a..331ea1a583e3 100644 --- a/nimdoc/testproject/expected/nimdoc.out.css +++ b/nimdoc/testproject/expected/nimdoc.out.css @@ -255,6 +255,10 @@ a.reference-toplevel { font-weight: bold; } +a.nimdoc { + word-spacing: 0.3em; +} + a.toc-backref { text-decoration: none; color: var(--text); } diff --git a/nimdoc/testproject/expected/subdir/subdir_b/utils.html b/nimdoc/testproject/expected/subdir/subdir_b/utils.html index 47901ce0dea7..dfdd40bf0ca9 100644 --- a/nimdoc/testproject/expected/subdir/subdir_b/utils.html +++ b/nimdoc/testproject/expected/subdir/subdir_b/utils.html @@ -106,7 +106,10 @@

subdir/subdir_b/utils

  • Types
      -
    • G
      • +
    • SomeType
      • @@ -115,9 +118,54 @@

      subdir/subdir_b/utils

    • Procs
        -
          fn2 + + + + + + + + + +
        • + Iterators + +
        • Templates
            @@ -197,7 +261,13 @@

            subdir/subdir_b/utils

            -

            +

            This is a description of the utils module.

            +

            Links work:

            + +

            This is now a header

            Next header

            And so on

            @@ -209,10 +279,29 @@
          • Other case value
          • Second case.
            • -

            +

            Ref group fn2 or specific function like fn2() or fn2( int ) or fn2(int, float).

            +

            Ref generics like this: binarySearch or binarySearch(openArray[T], K, proc (T, K)) or proc binarySearch(openArray[T], K, proc (T, K)) or in different style: proc binarysearch(openarray[T], K, proc(T, K)). Can be combined with export symbols and type parameters: binarysearch*[T, K](openArray[T], K, proc (T, K)). With spaces binary search.

            +

            Ref. type like G and type G and G[T] and type G*[T].

            +Ref. [] is the same as proc `[]`(G[T]) because there are no overloads. The full form: proc `[]`*[T](x: G[T]): TRef. []= aka `[]=`(G[T], int, T).Ref. $ aka proc $ or proc `$`.Ref. $(a: ref SomeType).Ref. foo_bar aka iterator foo_bar_.Ref. fn[T; U,V: SomeFloat]().Ref. 'big or func `'big` or `'big`(string).

            Types

            +
            +
            G[T] = object
+  val: T
+
            +
            + + + +
            +
            SomeType = enum
   enumValueA, enumValueB, enumValueC
            @@ -227,6 +316,109 @@

            Types

            Procs

            + +
            +
            +
            proc `$`[T](a: G[T]): string
            +
            + + + +
            +
            +
            +
            proc `$`[T](a: ref SomeType): string
            +
            + + + +
            +
            + +
            + +
            +
            +
            func `'big`(a: string): SomeType {....raises: [], tags: [].}
            +
            + + + +
            +
            + +
            + +
            +
            +
            proc `[]`[T](x: G[T]): T
            +
            + + + +
            +
            + +
            + +
            +
            +
            proc `[]=`[T](a: var G[T]; index: int; value: T)
            +
            + + + +
            +
            + +
            + +
            +
            +
            proc binarySearch[T, K](a: openArray[T]; key: K;
+                        cmp: proc (x: T; y: K): int {.closure.}): int
            +
            + + + +
            +
            + +
            + +
            +
            +
            proc f(x: G[int]) {....raises: [], tags: [].}
            +
            + +There is also variant f(G[string]) + +
            +
            +
            +
            proc f(x: G[string]) {....raises: [], tags: [].}
            +
            + +See also f(G[int]). + +
            +
            + +
            + +
            +
            +
            proc fn[T; U, V: SomeFloat]()
            +
            + + + +
            +
            + +
            + +
            proc fn2() {....raises: [], tags: [].}
            @@ -235,6 +427,26 @@

            Procs

            +
            +
            proc fn2(x: int) {....raises: [], tags: [].}
            +
            + +fn2 comment + +
            +
            +
            +
            proc fn2(x: int; y: float) {....raises: [], tags: [].}
            +
            + + + +
            +
            + +
            + +
            proc fn3(): auto {....raises: [], tags: [].}
            @@ -243,6 +455,10 @@

            Procs

            + +
            + +
            proc fn4(): auto {....raises: [], tags: [].}
            @@ -251,6 +467,10 @@

            Procs

            + +
            + +
            proc fn5() {....raises: [], tags: [].}
            @@ -259,6 +479,10 @@

            Procs

            + +
            + +
            proc fn6() {....raises: [], tags: [].}
            @@ -267,6 +491,10 @@

            Procs

            + +
            + +
            proc fn7() {....raises: [], tags: [].}
            @@ -275,6 +503,10 @@

            Procs

            + +
            + +
            proc fn8(): auto {....raises: [], tags: [].}
            @@ -283,6 +515,10 @@

            Procs

            + +
            + +
            func fn9(a: int): int {....raises: [], tags: [].}
            @@ -291,6 +527,10 @@

            Procs

            + +
            + +
            func fn10(a: int): int {....raises: [], tags: [].}
            @@ -299,6 +539,22 @@

            Procs

            + +
            + +
            +
            +
            proc funWithGenerics[T, U: SomeFloat](a: T; b: U)
            +
            + + + +
            +
            + +
            + +
            proc someType(): SomeType {....raises: [], tags: [].}
            @@ -308,10 +564,31 @@

            Procs

            +
            + +
            +
            +

            Iterators

            +
            + +
            +
            +
            iterator fooBar(a: seq[SomeType]): int {....raises: [], tags: [].}
            +
            + + + +
            +
            + +
            +

            Templates

            + +
            template aEnum(): untyped
            @@ -320,6 +597,10 @@

            Templates

            + +
            + +
            template bEnum(): untyped
            @@ -328,6 +609,10 @@

            Templates

            + +
            + +
            template fromUtilsGen(): untyped
            @@ -339,6 +624,8 @@

            Templates

            +
            +
            diff --git a/nimdoc/testproject/expected/subdir/subdir_b/utils.idx b/nimdoc/testproject/expected/subdir/subdir_b/utils.idx index a87393f0952a..441bd8675701 100644 --- a/nimdoc/testproject/expected/subdir/subdir_b/utils.idx +++ b/nimdoc/testproject/expected/subdir/subdir_b/utils.idx @@ -1,9 +1,14 @@ +funWithGenerics subdir/subdir_b/utils.html#funWithGenerics,T,U utils: funWithGenerics[T, U: SomeFloat](a: T; b: U) enumValueA subdir/subdir_b/utils.html#enumValueA SomeType.enumValueA enumValueB subdir/subdir_b/utils.html#enumValueB SomeType.enumValueB enumValueC subdir/subdir_b/utils.html#enumValueC SomeType.enumValueC SomeType subdir/subdir_b/utils.html#SomeType utils: SomeType +G subdir/subdir_b/utils.html#G utils: G someType subdir/subdir_b/utils.html#someType_2 utils: someType(): SomeType fn2 subdir/subdir_b/utils.html#fn2 utils: fn2() +fn2 subdir/subdir_b/utils.html#fn2,int utils: fn2(x: int) +fn2 subdir/subdir_b/utils.html#fn2,int,float utils: fn2(x: int; y: float) +binarySearch subdir/subdir_b/utils.html#binarySearch,openArray[T],K,proc(T,K) utils: binarySearch[T, K](a: openArray[T]; key: K;\n cmp: proc (x: T; y: K): int {.closure.}): int fn3 subdir/subdir_b/utils.html#fn3 utils: fn3(): auto fn4 subdir/subdir_b/utils.html#fn4 utils: fn4(): auto fn5 subdir/subdir_b/utils.html#fn5 utils: fn5() @@ -15,6 +20,15 @@ fn10 subdir/subdir_b/utils.html#fn10,int utils: fn10(a: int): int aEnum subdir/subdir_b/utils.html#aEnum.t utils: aEnum(): untyped bEnum subdir/subdir_b/utils.html#bEnum.t utils: bEnum(): untyped fromUtilsGen subdir/subdir_b/utils.html#fromUtilsGen.t utils: fromUtilsGen(): untyped +f subdir/subdir_b/utils.html#f,G[int] utils: f(x: G[int]) +f subdir/subdir_b/utils.html#f,G[string] utils: f(x: G[string]) +`[]` subdir/subdir_b/utils.html#[],G[T] utils: `[]`[T](x: G[T]): T +`[]=` subdir/subdir_b/utils.html#[]=,G[T],int,T utils: `[]=`[T](a: var G[T]; index: int; value: T) +`$` subdir/subdir_b/utils.html#$,G[T] utils: `$`[T](a: G[T]): string +`$` subdir/subdir_b/utils.html#$,ref.SomeType utils: `$`[T](a: ref SomeType): string +fooBar subdir/subdir_b/utils.html#fooBar.i,seq[SomeType] utils: fooBar(a: seq[SomeType]): int +fn subdir/subdir_b/utils.html#fn utils: fn[T; U, V: SomeFloat]() +`'big` subdir/subdir_b/utils.html#'big,string utils: `'big`(a: string): SomeType This is now a header subdir/subdir_b/utils.html#this-is-now-a-header This is now a header Next header subdir/subdir_b/utils.html#this-is-now-a-header-next-header Next header And so on subdir/subdir_b/utils.html#next-header-and-so-on And so on diff --git a/nimdoc/testproject/expected/testproject.html b/nimdoc/testproject/expected/testproject.html index dbacb57a7493..e45492dace65 100644 --- a/nimdoc/testproject/expected/testproject.html +++ b/nimdoc/testproject/expected/testproject.html @@ -559,6 +559,8 @@

            Consts

            Procs

            + +
            proc addfBug14485() {....raises: [], tags: [].}
            @@ -579,6 +581,10 @@

            Procs

            + +
            + +
            proc anything() {....raises: [], tags: [].}
            @@ -587,6 +593,10 @@

            Procs

            + +
            + +
            proc asyncFun1(): Future[int] {....raises: [Exception, ValueError],
                                 tags: [RootEffect].}
            @@ -596,6 +606,10 @@

            Procs

            + +
            + +
            proc asyncFun2(): owned(Future[void]) {....raises: [Exception], tags: [RootEffect].}
            @@ -604,6 +618,10 @@

            Procs

            + +
            + +
            proc asyncFun3(): owned(Future[void]) {....raises: [Exception], tags: [RootEffect].}
            @@ -614,6 +632,10 @@

            Procs

            + +
            + +
            proc bar[T](a, b: T): T
            @@ -622,6 +644,10 @@

            Procs

            + +
            + +
            proc baz() {....raises: [], tags: [].}
            @@ -641,6 +667,10 @@

            Procs

            + +
            + +
            proc buzz[T](a, b: T): T {....deprecated: "since v0.20".}
            @@ -652,6 +682,10 @@

            Procs

            + +
            + +
            proc c_nonexistent(frmt: cstring): cint {.importc: "nonexistent",
     header: "<stdio.h>", varargs, discardable, ...raises: [], tags: [].}
            @@ -661,6 +695,10 @@

            Procs

            + +
            + +
            proc c_printf(frmt: cstring): cint {.importc: "printf", header: "<stdio.h>",
                                      varargs, discardable, ...raises: [], tags: [].}
            @@ -670,6 +708,10 @@

            Procs

            + +
            + +
            proc fromUtils3() {....raises: [], tags: [].}
            @@ -681,6 +723,10 @@

            Procs

            + +
            + +
            proc isValid[T](x: T): bool
            @@ -689,34 +735,46 @@

            Procs

            -
            -
            proc low2[T: Ordinal | enum | range](x: T): T {.magic: "Low", noSideEffect,
+
+
            + +
            +
            +
            proc low[T: Ordinal | enum | range](x: T): T {.magic: "Low", noSideEffect,
     ...raises: [], tags: [].}

            Returns the lowest possible value of an ordinal value x. As a special semantic rule, x may also be a type identifier.

            See also:

            -
            • low(T)
              • + -
              low2(2) # => -9223372036854775808
              -

              Example:

              -
              discard "in low2"
              +
              low(2) # => -9223372036854775808
            -
            -
            proc low[T: Ordinal | enum | range](x: T): T {.magic: "Low", noSideEffect,
+
+
            + +
            +
            +
            proc low2[T: Ordinal | enum | range](x: T): T {.magic: "Low", noSideEffect,
     ...raises: [], tags: [].}

            Returns the lowest possible value of an ordinal value x. As a special semantic rule, x may also be a type identifier.

            See also:

            -
            • low2(T)
              • + -
              low(2) # => -9223372036854775808
              +
              low2(2) # => -9223372036854775808
              +

              Example:

              +
              discard "in low2"
            + +
            + +
            proc p1() {....raises: [], tags: [].}
            @@ -741,6 +799,10 @@

            Procs

            + +
            + +
            func someFunc() {....raises: [], tags: [].}
            @@ -749,6 +811,10 @@

            Procs

            + +
            + +
            proc tripleStrLitTest() {....raises: [], tags: [].}
            @@ -793,6 +859,10 @@

            Procs

            + +
            + +
            proc z1(): Foo {....raises: [], tags: [].}
            @@ -801,6 +871,10 @@

            Procs

            + +
            + +
            proc z2() {....raises: [], tags: [].}
            @@ -811,6 +885,10 @@

            Procs

            + +
            + +
            proc z3() {....raises: [], tags: [].}
            @@ -819,6 +897,10 @@

            Procs

            + +
            + +
            proc z4() {....raises: [], tags: [].}
            @@ -827,6 +909,10 @@

            Procs

            + +
            + +
            proc z5(): int {....raises: [], tags: [].}
            @@ -835,6 +921,10 @@

            Procs

            + +
            + +
            proc z6(): int {....raises: [], tags: [].}
            @@ -843,6 +933,10 @@

            Procs

            + +
            + +
            proc z7(): int {....raises: [], tags: [].}
            @@ -851,6 +945,10 @@

            Procs

            + +
            + +
            proc z8(): int {....raises: [], tags: [].}
            @@ -859,6 +957,10 @@

            Procs

            + +
            + +
            proc z9() {....raises: [], tags: [].}
            @@ -869,6 +971,10 @@

            Procs

            + +
            + +
            proc z10() {....raises: [], tags: [].}
            @@ -879,6 +985,10 @@

            Procs

            + +
            + +
            proc z11() {....raises: [], tags: [].}
            @@ -889,6 +999,10 @@

            Procs

            + +
            + +
            proc z12(): int {....raises: [], tags: [].}
            @@ -899,6 +1013,10 @@

            Procs

            + +
            + +
            proc z13() {....raises: [], tags: [].}
            @@ -909,6 +1027,10 @@

            Procs

            + +
            + +
            proc z17() {....raises: [], tags: [].}
            @@ -920,10 +1042,14 @@

            Procs

            +
            +

            Methods

            + +
            method method1(self: Moo) {.base, ...raises: [], tags: [].}
            @@ -932,6 +1058,10 @@

            Methods

            + +
            + +
            method method2(self: Moo): int {.base, ...raises: [], tags: [].}
            @@ -940,6 +1070,10 @@

            Methods

            + +
            + +
            method method3(self: Moo): int {.base, ...raises: [], tags: [].}
            @@ -949,10 +1083,14 @@

            Methods

            +
            +

            Iterators

            + +
            iterator fromUtils1(): int {....raises: [], tags: [].}
            @@ -965,6 +1103,10 @@

            Iterators

            + +
            + +
            iterator iter1(n: int): int {....raises: [], tags: [].}
            @@ -973,6 +1115,10 @@

            Iterators

            + +
            + +
            iterator iter2(n: int): int {....raises: [], tags: [].}
            @@ -984,10 +1130,14 @@

            Iterators

            +
            +

            Macros

            + +
            macro bar(): untyped
            @@ -996,6 +1146,10 @@

            Macros

            + +
            + +
            macro z16()
            @@ -1008,6 +1162,10 @@

            Macros

            + +
            + +
            macro z18(): int
            @@ -1017,10 +1175,14 @@

            Macros

            +
            +

            Templates

            + +
            template foo(a, b: SomeType)
            @@ -1029,6 +1191,10 @@

            Templates

            + +
            + +
            template fromUtils2()
            @@ -1040,6 +1206,10 @@

            Templates

            + +
            + +
            template myfn()
            @@ -1063,6 +1233,10 @@

            Templates

            + +
            + +
            template testNimDocTrailingExample()
            @@ -1073,6 +1247,10 @@

            Templates

            + +
            + +
            template z6t(): int
            @@ -1081,6 +1259,10 @@

            Templates

            + +
            + +
            template z14()
            @@ -1091,6 +1273,10 @@

            Templates

            + +
            + +
            template z15()
            @@ -1110,6 +1296,8 @@

            Templates

            +
            +
            diff --git a/nimdoc/testproject/expected/theindex.html b/nimdoc/testproject/expected/theindex.html index f72f6c37e183..da76334d4579 100644 --- a/nimdoc/testproject/expected/theindex.html +++ b/nimdoc/testproject/expected/theindex.html @@ -71,7 +71,25 @@

            Index

            Modules: subdir/subdir_b/utils, testproject.

            API symbols

            -
            A:
            +
            binarySearch:
            buzz:
            +
            f:
            +
            fn:
            fn10:
          • testproject: foo(a, b: SomeType)
          +
          fooBar:
          FooBuzz:
          +
          funWithGenerics:
          +
          G:
          isValid:
          • testproject: isValid[T](x: T): bool
            • diff --git a/nimdoc/testproject/subdir/subdir_b/utils.nim b/nimdoc/testproject/subdir/subdir_b/utils.nim index 37c91dd3c70a..563ac0365f4d 100644 --- a/nimdoc/testproject/subdir/subdir_b/utils.nim +++ b/nimdoc/testproject/subdir/subdir_b/utils.nim @@ -1,5 +1,7 @@ ##[ +.. include:: ./utils_overview.rst + # This is now a header ## Next header @@ -19,13 +21,30 @@ More text. 1. Other case value 2. Second case. +Ref group fn2_ or specific function like `fn2()`_ +or `fn2( int )`_ or `fn2(int, +float)`_. + +Ref generics like this: binarySearch_ or `binarySearch(openArray[T], K, +proc (T, K))`_ or `proc binarySearch(openArray[T], K, proc (T, K))`_ or +in different style: `proc binarysearch(openarray[T], K, proc(T, K))`_. +Can be combined with export symbols and type parameters: +`binarysearch*[T, K](openArray[T], K, proc (T, K))`_. +With spaces `binary search`_. + +Ref. type like G_ and `type G`_ and `G[T]`_ and `type G*[T]`_. + ]## +include ./utils_helpers + type SomeType* = enum enumValueA, enumValueB, enumValueC + G*[T] = object + val: T proc someType*(): SomeType = ## constructor. @@ -33,6 +52,14 @@ proc someType*(): SomeType = proc fn2*() = discard ## comment +proc fn2*(x: int) = + ## fn2 comment + discard +proc fn2*(x: int, y: float) = + discard +proc binarySearch*[T, K](a: openArray[T]; key: K; + cmp: proc (x: T; y: K): int {.closure.}): int = + discard proc fn3*(): auto = 1 ## comment proc fn4*(): auto = 2 * 3 + 4 ## comment proc fn5*() ## comment @@ -89,3 +116,39 @@ template fromUtilsGen*(): untyped = ## came form utils but should be shown where `fromUtilsGen` is called runnableExamples: discard """should be shown as examples for fromUtils3 in module calling fromUtilsGen""" + +proc f*(x: G[int]) = + ## There is also variant `f(G[string])`_ + discard +proc f*(x: G[string]) = + ## See also `f(G[int])`_. + discard + +## Ref. `[]`_ is the same as `proc \`[]\`(G[T])`_ because there are no +## overloads. The full form: `proc \`[]\`*[T](x: G[T]): T`_ + +proc `[]`*[T](x: G[T]): T = x.val + +## Ref. `[]=`_ aka `\`[]=\`(G[T], int, T)`_. + +proc `[]=`*[T](a: var G[T], index: int, value: T) = discard + +## Ref. `$`_ aka `proc $`_ or `proc \`$\``_. + +proc `$`*[T](a: G[T]): string = "" + +## Ref. `$(a: ref SomeType)`_. + +proc `$`*[T](a: ref SomeType): string = "" + +## Ref. foo_bar_ aka `iterator foo_bar_`_. + +iterator fooBar*(a: seq[SomeType]): int = discard + +## Ref. `fn[T; U,V: SomeFloat]()`_. + +proc fn*[T; U, V: SomeFloat]() = discard + +## Ref. `'big`_ or `func \`'big\``_ or `\`'big\`(string)`_. + +func `'big`*(a: string): SomeType = discard diff --git a/nimdoc/testproject/subdir/subdir_b/utils_helpers.nim b/nimdoc/testproject/subdir/subdir_b/utils_helpers.nim new file mode 100644 index 000000000000..2c45ffb8347b --- /dev/null +++ b/nimdoc/testproject/subdir/subdir_b/utils_helpers.nim @@ -0,0 +1 @@ +proc funWithGenerics*[T, U: SomeFloat](a: T, b: U) = discard diff --git a/nimdoc/testproject/subdir/subdir_b/utils_overview.rst b/nimdoc/testproject/subdir/subdir_b/utils_overview.rst new file mode 100644 index 000000000000..58ce930bfc07 --- /dev/null +++ b/nimdoc/testproject/subdir/subdir_b/utils_overview.rst @@ -0,0 +1,8 @@ +This is a description of the utils module. + +Links work: + +* other module: `iterators `_ (not in this dir, just an example) +* internal: `fn2(x)`_ +* internal included from another module: `funWithGenerics*[T, U: + SomeFloat](a: T, b: U)`_. diff --git a/tests/stdlib/trst.nim b/tests/stdlib/trst.nim index e9f67324cb70..ee00525b2dbf 100644 --- a/tests/stdlib/trst.nim +++ b/tests/stdlib/trst.nim @@ -12,14 +12,17 @@ discard """ [Suite] RST escaping [Suite] RST inline markup + +[Suite] Integration with Nim ''' """ # tests for rst module import ../../lib/packages/docutils/rstgen -import ../../lib/packages/docutils/rst +import ../../lib/packages/docutils/rst {.all.} import ../../lib/packages/docutils/rstast +import ../../lib/packages/docutils/dochelpers import unittest, strutils import std/private/miscdollars import os @@ -60,7 +63,62 @@ proc toAst(input: string, if e.msg != "": result = e.msg +proc rstParseTest(text: string): PRstNode = + proc testMsgHandler(filename: string, line, col: int, msgkind: MsgKind, + arg: string) = + doAssert msgkind == mwBrokenLink + let r = rstParse(text, "-input-", LineRstInit, ColRstInit, + {roPreferMarkdown, roSupportMarkdown, roNimFile}, + msgHandler=testMsgHandler) + result = r.node + suite "RST parsing": + test "References are whitespace-neutral and case-insensitive": + # refname is 'lexical-analysis', the same for all the 3 variants: + check(dedent""" + Lexical Analysis + ================ + + Ref. `Lexical Analysis`_ or `Lexical analysis`_ or `lexical analysis`_. + """.toAst == + dedent""" + rnInner + rnHeadline level=1 + rnLeaf 'Lexical' + rnLeaf ' ' + rnLeaf 'Analysis' + rnParagraph + rnLeaf 'Ref' + rnLeaf '.' + rnLeaf ' ' + rnInternalRef + rnInner + rnLeaf 'Lexical' + rnLeaf ' ' + rnLeaf 'Analysis' + rnLeaf 'lexical-analysis' + rnLeaf ' ' + rnLeaf 'or' + rnLeaf ' ' + rnInternalRef + rnInner + rnLeaf 'Lexical' + rnLeaf ' ' + rnLeaf 'analysis' + rnLeaf 'lexical-analysis' + rnLeaf ' ' + rnLeaf 'or' + rnLeaf ' ' + rnInternalRef + rnInner + rnLeaf 'lexical' + rnLeaf ' ' + rnLeaf 'analysis' + rnLeaf 'lexical-analysis' + rnLeaf '.' + rnLeaf ' ' + """) + test "option list has priority over definition list": check(dedent""" --defusages @@ -376,9 +434,9 @@ suite "Warnings": let output = input.toAst(warnings=warnings) check(warnings[] == @[ "input(3, 14) Warning: broken link 'citation-som'", - "input(5, 7) Warning: broken link 'a-broken-link'", + "input(5, 7) Warning: broken link 'a broken Link'", "input(7, 15) Warning: unknown substitution 'undefined subst'", - "input(9, 6) Warning: broken link 'shortdotlink'" + "input(9, 6) Warning: broken link 'short.link'" ]) test "With include directive and blank lines at the beginning": @@ -391,7 +449,7 @@ suite "Warnings": let input = ".. include:: other.rst" var warnings = new seq[string] let output = input.toAst(warnings=warnings) - check warnings[] == @["other.rst(5, 6) Warning: broken link 'brokenlink'"] + check warnings[] == @["other.rst(5, 6) Warning: broken link 'brokenLink'"] check(output == dedent""" rnInner rnParagraph @@ -404,6 +462,59 @@ suite "Warnings": """) removeFile("other.rst") + test "warnings for ambiguous links (references + anchors)": + # Reference like `x`_ generates a link alias x that may clash with others + let input = dedent""" + Manual reference: `foo <#foo,string,string>`_ + + .. _foo: + + Paragraph. + + Ref foo_ + """ + var warnings = new seq[string] + let output = input.toAst(warnings=warnings) + check(warnings[] == @[ + dedent """ + input(7, 5) Warning: ambiguous doc link `foo` + clash: + (3, 8): (manual directive anchor) + (1, 45): (implicitly-generated hyperlink alias)""" + ]) + # reference should be resolved to the manually set anchor: + check(output == + dedent""" + rnInner + rnParagraph + rnLeaf 'Manual' + rnLeaf ' ' + rnLeaf 'reference' + rnLeaf ':' + rnLeaf ' ' + rnHyperlink + rnInner + rnLeaf 'foo' + rnInner + rnLeaf '#' + rnLeaf 'foo' + rnLeaf ',' + rnLeaf 'string' + rnLeaf ',' + rnLeaf 'string' + rnParagraph anchor='foo' + rnLeaf 'Paragraph' + rnLeaf '.' + rnParagraph + rnLeaf 'Ref' + rnLeaf ' ' + rnInternalRef + rnInner + rnLeaf 'foo' + rnLeaf 'foo' + rnLeaf ' ' + """) + suite "RST include directive": test "Include whole": "other.rst".writeFile("**test1**") @@ -798,3 +909,117 @@ suite "RST inline markup": rnLeaf 'my {link example' rnLeaf 'http://example.com/bracket_(symbol_[)' """) + +suite "Integration with Nim": + test "simple symbol parsing (shortest form)": + let input1 = "g_".rstParseTest + check input1.toLangSymbol == LangSymbol(symKind: "", name: "g") + + test "simple symbol parsing (group of words)": + let input1 = "`Y`_".rstParseTest + check input1.toLangSymbol == LangSymbol(symKind: "", name: "Y") + + # this means not a statement 'type', it's a backticked identifier `type`: + let input2 = "`type`_".rstParseTest + check input2.toLangSymbol == LangSymbol(symKind: "", name: "type") + + let input3 = "`[]`_".rstParseTest + check input3.toLangSymbol == LangSymbol(symKind: "", name: "[]") + + let input4 = "`X Y Z`_".rstParseTest + check input4.toLangSymbol == LangSymbol(symKind: "", name: "Xyz") + + test "simple proc parsing": + let input1 = "proc f".rstParseTest + check input1.toLangSymbol == LangSymbol(symKind: "proc", name: "f") + + test "another backticked name": + let input1 = """`template \`type\``_""".rstParseTest + check input1.toLangSymbol == LangSymbol(symKind: "template", name: "type") + + test "simple proc parsing with parameters": + let input1 = "`proc f*()`_".rstParseTest + let input2 = "`proc f()`_".rstParseTest + let expected = LangSymbol(symKind: "proc", name: "f", + parametersProvided: true) + check input1.toLangSymbol == expected + check input2.toLangSymbol == expected + + test "symbol parsing with 1 parameter": + let input = "`f(G[int])`_".rstParseTest + let expected = LangSymbol(symKind: "", name: "f", + parameters: @[("G[int]", "")], + parametersProvided: true) + check input.toLangSymbol == expected + + test "more proc parsing": + let input1 = "`proc f[T](x:G[T]):M[T]`_".rstParseTest + let input2 = "`proc f[ T ] ( x: G [T] ): M[T]`_".rstParseTest + let input3 = "`proc f*[T](x: G[T]): M[T]`_".rstParseTest + let expected = LangSymbol(symKind: "proc", + name: "f", + generics: "[T]", + parameters: @[("x", "G[T]")], + parametersProvided: true, + outType: "M[T]") + check(input1.toLangSymbol == expected) + check(input2.toLangSymbol == expected) + check(input3.toLangSymbol == expected) + + test "advanced proc parsing with Nim identifier normalization": + let input = """`proc binarySearch*[T, K](a: openArray[T]; key: K; + cmp: proc (x: T; y: K): int)`_""".rstParseTest + let expected = LangSymbol(symKind: "proc", + name: "binarysearch", + generics: "[T,K]", + parameters: @[ + ("a", "openarray[T]"), + ("key", "K"), + ("cmp", "proc(x:T;y:K):int")], + parametersProvided: true, + outType: "") + check(input.toLangSymbol == expected) + + test "the same without proc": + let input = """`binarySearch*[T, K](a: openArray[T]; key: K; + cmp: proc (x: T; y: K): int {.closure.})`_""".rstParseTest + let expected = LangSymbol(symKind: "", + name: "binarysearch", + generics: "[T,K]", + parameters: @[ + ("a", "openarray[T]"), + ("key", "K"), + ("cmp", "proc(x:T;y:K):int")], + parametersProvided: true, + outType: "") + check(input.toLangSymbol == expected) + + test "operator $ with and without backticks": + let input1 = """`func \`$\`*[T](a: \`open Array\`[T]): string`_""". + rstParseTest + let input2 = """`func $*[T](a: \`open Array\`[T]): string`_""". + rstParseTest + let expected = LangSymbol(symKind: "func", + name: "$", + generics: "[T]", + parameters: @[("a", "openarray[T]")], + parametersProvided: true, + outType: "string") + check(input1.toLangSymbol == expected) + check(input2.toLangSymbol == expected) + + test "operator [] with and without backticks": + let input1 = """`func \`[]\`[T](a: \`open Array\`[T], idx: int): T`_""". + rstParseTest + let input2 = """`func [][T](a: \`open Array\`[T], idx: int): T`_""". + rstParseTest + let expected = LangSymbol(symKind: "func", + name: "[]", + generics: "[T]", + parameters: @[("a", "openarray[T]"), + ("idx", "int")], + parametersProvided: true, + outType: "T") + check(input1.toLangSymbol == expected) + check(input2.toLangSymbol == expected) + diff --git a/tests/stdlib/trstgen.nim b/tests/stdlib/trstgen.nim index 867677404342..45cf1a68e050 100644 --- a/tests/stdlib/trstgen.nim +++ b/tests/stdlib/trstgen.nim @@ -1054,8 +1054,9 @@ Test1 Paragraph2 ref `internal anchor`_. """ let output9 = input9.toHtml - #doAssert "id=\"internal-anchor\"" in output9 - #doAssert "internal anchor" notin output9 + # _`internal anchor` got erased: + check "href=\"#internal-anchor\"" notin output9 + check "href=\"#citation-another\"" in output9 doAssert output9.count("
            " & "
            ") == 1 doAssert output9.count("
            ") == 3 @@ -1330,12 +1331,12 @@ Test1 """ let output1 = input1.toHtml # "target101" should be erased and changed to "section-xyz": - doAssert "href=\"#target101\"" notin output1 - doAssert "id=\"target101\"" notin output1 - doAssert "href=\"#target102\"" notin output1 - doAssert "id=\"target102\"" notin output1 - doAssert "id=\"section-xyz\"" in output1 - doAssert "href=\"#section-xyz\"" in output1 + check "href=\"#target101\"" notin output1 + check "id=\"target101\"" notin output1 + check "href=\"#target102\"" notin output1 + check "id=\"target102\"" notin output1 + check "id=\"section-xyz\"" in output1 + check "href=\"#section-xyz\"" in output1 let input2 = dedent """ .. _target300: @@ -1405,7 +1406,7 @@ Test1 let output1 = input1.toHtml doAssert "id=\"secdot1\"" in output1 doAssert "id=\"Z2minusothercolonsecplusc-2\"" in output1 - doAssert "id=\"linkdot1-2021\"" in output1 + check "id=\"linkdot1-2021\"" in output1 let ref1 = "sec.1" let ref2 = "2-other:sec+c_2" let ref3 = "link.1_2021" From d69ea9dbe278ed8f0f723e225d7542bb12c004ff Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Sun, 12 Sep 2021 20:59:16 +0300 Subject: [PATCH 2/4] Allow postfix form of symbol kinds (e.g. proc) --- doc/docgen.rst | 22 +++++++--- lib/packages/docutils/dochelpers.nim | 43 +++++++++++-------- .../expected/subdir/subdir_b/utils.html | 2 + nimdoc/testproject/subdir/subdir_b/utils.nim | 2 + tests/stdlib/trst.nim | 22 ++++++++++ 5 files changed, 65 insertions(+), 26 deletions(-) diff --git a/doc/docgen.rst b/doc/docgen.rst index 7b0680d18d2d..51e7102eb0b3 100644 --- a/doc/docgen.rst +++ b/doc/docgen.rst @@ -235,12 +235,13 @@ Referencing Nim symbols: simple documentation links You can reference Nim identifiers from Nim documentation comments, currently only inside their ``.nim`` file (or inside a ``.rst`` file included from -the ``.nim``). The point is that such links will be resolved automatically +a ``.nim``). The point is that such links will be resolved automatically by `nim doc`:cmd: (or `nim jsondoc`:cmd: or `nim doc2tex`:cmd:). This pertains to any exported symbol like `proc`, `const`, `iterator`, etc. Syntax for referencing is basically a normal RST one: addition of underscore `_` to a *link text*. -Link text is either one word or a group of words enclosed by backticks `\``. +Link text is either one word or a group of words enclosed by backticks `\`` +(for a one word case backticks are usually omitted). Link text will be displayed *as is* while *link target* will be set to the anchor [*]_ of Nim symbol that corresponds to link text. @@ -311,6 +312,11 @@ Generally following syntax is allowed for referencing `foo`: Brevity is better for reading! If you use a short form and have an ambiguity problem (see below) then just add some additional info. +Symbol kind like `proc` can also be specified in the postfix form:: + + `foo proc`_ + `walkDir(d: string) iterator`_ + .. Warning:: An ambiguity in resolving documentation links may arise because of: 1. clash with other RST anchors @@ -320,13 +326,15 @@ Generally following syntax is allowed for referencing `foo`: * routines with different parameters can exist e.g. for `proc` and `template`. In this case they are split between their - corresponding sections in output file + corresponding sections in output file. Qualified references are + useful in this case -- just disambiguate by referring to these + sections explicitly:: + + See `foo proc`_ and `foo template`_. + * because in Nim `proc` and `iterator` belong to different namespaces, so there can be a collision even if parameters are the same. - - Qualified references are useful in this case: just disambiguate - the reference like `\`proc foo\`_`:literal: - or `\`template foo\`_`:literal: or `\`iterator foo\`_`:literal:. + Use `\`proc foo\`_`:literal: or `\`iterator foo\`_`:literal: then. Any ambiguity is always reported with Nim compiler warnings and an anchor with higher priority is selected. Manual anchors have highest diff --git a/lib/packages/docutils/dochelpers.nim b/lib/packages/docutils/dochelpers.nim index 7a08f74bbc6a..c488c4d99a3c 100644 --- a/lib/packages/docutils/dochelpers.nim +++ b/lib/packages/docutils/dochelpers.nim @@ -192,26 +192,31 @@ proc toLangSymbol*(linkText: PRstNode): LangSymbol = state = parameterType else: curIdent.add ":" else: - case state - of inBeginning: - if s(i) in NimDefs: - state = afterSymKind - else: - state = atSymbolName - curIdent.add s(i) - of afterSymKind, beforeSymbolName: - state = atSymbolName - curIdent.add s(i) - of parameterType: - case s(i) - of "ref": curIdent.add "ref." - of "ptr": curIdent.add "ptr." - of "var": discard - else: curIdent.add s(i).nimIdentBackticksNormalize - of atSymbolName: - curIdent.add s(i) + let isPostfixSymKind = i > 0 and i == L - 1 and + result.symKind == "" and s(i) in NimDefs + if isPostfixSymKind: # for links like `foo proc`_ + result.symKind = s(i) else: - curIdent.add s(i).nimIdentBackticksNormalize + case state + of inBeginning: + if s(i) in NimDefs: + state = afterSymKind + else: + state = atSymbolName + curIdent.add s(i) + of afterSymKind, beforeSymbolName: + state = atSymbolName + curIdent.add s(i) + of parameterType: + case s(i) + of "ref": curIdent.add "ref." + of "ptr": curIdent.add "ptr." + of "var": discard + else: curIdent.add s(i).nimIdentBackticksNormalize + of atSymbolName: + curIdent.add s(i) + else: + curIdent.add s(i).nimIdentBackticksNormalize while i < L: nextState inc i diff --git a/nimdoc/testproject/expected/subdir/subdir_b/utils.html b/nimdoc/testproject/expected/subdir/subdir_b/utils.html index dfdd40bf0ca9..c640399417c0 100644 --- a/nimdoc/testproject/expected/subdir/subdir_b/utils.html +++ b/nimdoc/testproject/expected/subdir/subdir_b/utils.html @@ -287,6 +287,8 @@
            proc binarysearch(openarray[T], K, proc(T, K)). Can be combined with export symbols and type parameters: binarysearch*[T, K](openArray[T], K, proc (T, K)). With spaces binary search.

            +

            Note that proc can be used in postfix form: binarySearch proc.

            Ref. type like G and type G and G[T] and type G*[T].

            Ref. [] is the same as proc `[]`(G[T]) because there are no overloads. The full form: proc `[]`*[T](x: G[T]): TRef. []= aka `[]=`(G[T], int, T).Ref. $ aka proc $ or proc `$`.Ref. $(a: ref SomeType).Ref. foo_bar aka iterator foo_bar_.Ref. fn[T; U,V: SomeFloat]().Ref. 'big or func `'big` or `'big`(string).

            diff --git a/nimdoc/testproject/subdir/subdir_b/utils.nim b/nimdoc/testproject/subdir/subdir_b/utils.nim index 563ac0365f4d..5d881e620379 100644 --- a/nimdoc/testproject/subdir/subdir_b/utils.nim +++ b/nimdoc/testproject/subdir/subdir_b/utils.nim @@ -32,6 +32,8 @@ Can be combined with export symbols and type parameters: `binarysearch*[T, K](openArray[T], K, proc (T, K))`_. With spaces `binary search`_. +Note that `proc` can be used in postfix form: `binarySearch proc`_. + Ref. type like G_ and `type G`_ and `G[T]`_ and `type G*[T]`_. ]## diff --git a/tests/stdlib/trst.nim b/tests/stdlib/trst.nim index ee00525b2dbf..14772b1181f2 100644 --- a/tests/stdlib/trst.nim +++ b/tests/stdlib/trst.nim @@ -1023,3 +1023,25 @@ suite "Integration with Nim": check(input1.toLangSymbol == expected) check(input2.toLangSymbol == expected) + test "postfix symbol specifier #1": + let input = """`walkDir iterator`_""". + rstParseTest + let expected = LangSymbol(symKind: "iterator", + name: "walkdir") + check(input.toLangSymbol == expected) + + test "postfix symbol specifier #2": + let input1 = """`\`[]\`[T](a: \`open Array\`[T], idx: int): T func`_""". + rstParseTest + let input2 = """`[][T](a: \`open Array\`[T], idx: int): T func`_""". + rstParseTest + let expected = LangSymbol(symKind: "func", + name: "[]", + generics: "[T]", + parameters: @[("a", "openarray[T]"), + ("idx", "int")], + parametersProvided: true, + outType: "T") + check(input1.toLangSymbol == expected) + check(input2.toLangSymbol == expected) + From b047fba3a3ac38322bc9f701886b653495985d18 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Wed, 15 Sep 2021 01:10:03 +0300 Subject: [PATCH 3/4] move dochelpers tests to a dedicated file --- tests/stdlib/tdochelpers.nim | 155 +++++++++++++++++++++++++++++++++++ tests/stdlib/trst.nim | 152 +--------------------------------- 2 files changed, 156 insertions(+), 151 deletions(-) create mode 100644 tests/stdlib/tdochelpers.nim diff --git a/tests/stdlib/tdochelpers.nim b/tests/stdlib/tdochelpers.nim new file mode 100644 index 000000000000..5bcdf32d52e4 --- /dev/null +++ b/tests/stdlib/tdochelpers.nim @@ -0,0 +1,155 @@ +discard """ + output: ''' + +[Suite] Integration with Nim +''' +""" + +# tests for dochelpers.nim module + +import ../../lib/packages/docutils/[rstast, rst, dochelpers] +import unittest + +proc rstParseTest(text: string): PRstNode = + proc testMsgHandler(filename: string, line, col: int, msgkind: MsgKind, + arg: string) = + doAssert msgkind == mwBrokenLink + let r = rstParse(text, "-input-", LineRstInit, ColRstInit, + {roPreferMarkdown, roSupportMarkdown, roNimFile}, + msgHandler=testMsgHandler) + result = r.node + +suite "Integration with Nim": + test "simple symbol parsing (shortest form)": + let input1 = "g_".rstParseTest + check input1.toLangSymbol == LangSymbol(symKind: "", name: "g") + + test "simple symbol parsing (group of words)": + let input1 = "`Y`_".rstParseTest + check input1.toLangSymbol == LangSymbol(symKind: "", name: "Y") + + # this means not a statement 'type', it's a backticked identifier `type`: + let input2 = "`type`_".rstParseTest + check input2.toLangSymbol == LangSymbol(symKind: "", name: "type") + + let input3 = "`[]`_".rstParseTest + check input3.toLangSymbol == LangSymbol(symKind: "", name: "[]") + + let input4 = "`X Y Z`_".rstParseTest + check input4.toLangSymbol == LangSymbol(symKind: "", name: "Xyz") + + test "simple proc parsing": + let input1 = "proc f".rstParseTest + check input1.toLangSymbol == LangSymbol(symKind: "proc", name: "f") + + test "another backticked name": + let input1 = """`template \`type\``_""".rstParseTest + check input1.toLangSymbol == LangSymbol(symKind: "template", name: "type") + + test "simple proc parsing with parameters": + let input1 = "`proc f*()`_".rstParseTest + let input2 = "`proc f()`_".rstParseTest + let expected = LangSymbol(symKind: "proc", name: "f", + parametersProvided: true) + check input1.toLangSymbol == expected + check input2.toLangSymbol == expected + + test "symbol parsing with 1 parameter": + let input = "`f(G[int])`_".rstParseTest + let expected = LangSymbol(symKind: "", name: "f", + parameters: @[("G[int]", "")], + parametersProvided: true) + check input.toLangSymbol == expected + + test "more proc parsing": + let input1 = "`proc f[T](x:G[T]):M[T]`_".rstParseTest + let input2 = "`proc f[ T ] ( x: G [T] ): M[T]`_".rstParseTest + let input3 = "`proc f*[T](x: G[T]): M[T]`_".rstParseTest + let expected = LangSymbol(symKind: "proc", + name: "f", + generics: "[T]", + parameters: @[("x", "G[T]")], + parametersProvided: true, + outType: "M[T]") + check(input1.toLangSymbol == expected) + check(input2.toLangSymbol == expected) + check(input3.toLangSymbol == expected) + + test "advanced proc parsing with Nim identifier normalization": + let input = """`proc binarySearch*[T, K](a: openArray[T]; key: K; + cmp: proc (x: T; y: K): int)`_""".rstParseTest + let expected = LangSymbol(symKind: "proc", + name: "binarysearch", + generics: "[T,K]", + parameters: @[ + ("a", "openarray[T]"), + ("key", "K"), + ("cmp", "proc(x:T;y:K):int")], + parametersProvided: true, + outType: "") + check(input.toLangSymbol == expected) + + test "the same without proc": + let input = """`binarySearch*[T, K](a: openArray[T]; key: K; + cmp: proc (x: T; y: K): int {.closure.})`_""".rstParseTest + let expected = LangSymbol(symKind: "", + name: "binarysearch", + generics: "[T,K]", + parameters: @[ + ("a", "openarray[T]"), + ("key", "K"), + ("cmp", "proc(x:T;y:K):int")], + parametersProvided: true, + outType: "") + check(input.toLangSymbol == expected) + + test "operator $ with and without backticks": + let input1 = """`func \`$\`*[T](a: \`open Array\`[T]): string`_""". + rstParseTest + let input2 = """`func $*[T](a: \`open Array\`[T]): string`_""". + rstParseTest + let expected = LangSymbol(symKind: "func", + name: "$", + generics: "[T]", + parameters: @[("a", "openarray[T]")], + parametersProvided: true, + outType: "string") + check(input1.toLangSymbol == expected) + check(input2.toLangSymbol == expected) + + test "operator [] with and without backticks": + let input1 = """`func \`[]\`[T](a: \`open Array\`[T], idx: int): T`_""". + rstParseTest + let input2 = """`func [][T](a: \`open Array\`[T], idx: int): T`_""". + rstParseTest + let expected = LangSymbol(symKind: "func", + name: "[]", + generics: "[T]", + parameters: @[("a", "openarray[T]"), + ("idx", "int")], + parametersProvided: true, + outType: "T") + check(input1.toLangSymbol == expected) + check(input2.toLangSymbol == expected) + + test "postfix symbol specifier #1": + let input = """`walkDir iterator`_""". + rstParseTest + let expected = LangSymbol(symKind: "iterator", + name: "walkdir") + check(input.toLangSymbol == expected) + + test "postfix symbol specifier #2": + let input1 = """`\`[]\`[T](a: \`open Array\`[T], idx: int): T func`_""". + rstParseTest + let input2 = """`[][T](a: \`open Array\`[T], idx: int): T func`_""". + rstParseTest + let expected = LangSymbol(symKind: "func", + name: "[]", + generics: "[T]", + parameters: @[("a", "openarray[T]"), + ("idx", "int")], + parametersProvided: true, + outType: "T") + check(input1.toLangSymbol == expected) + check(input2.toLangSymbol == expected) diff --git a/tests/stdlib/trst.nim b/tests/stdlib/trst.nim index 14772b1181f2..e4609a671398 100644 --- a/tests/stdlib/trst.nim +++ b/tests/stdlib/trst.nim @@ -12,17 +12,12 @@ discard """ [Suite] RST escaping [Suite] RST inline markup - -[Suite] Integration with Nim ''' """ # tests for rst module -import ../../lib/packages/docutils/rstgen -import ../../lib/packages/docutils/rst {.all.} -import ../../lib/packages/docutils/rstast -import ../../lib/packages/docutils/dochelpers +import ../../lib/packages/docutils/[rstgen, rst, rstast] import unittest, strutils import std/private/miscdollars import os @@ -63,15 +58,6 @@ proc toAst(input: string, if e.msg != "": result = e.msg -proc rstParseTest(text: string): PRstNode = - proc testMsgHandler(filename: string, line, col: int, msgkind: MsgKind, - arg: string) = - doAssert msgkind == mwBrokenLink - let r = rstParse(text, "-input-", LineRstInit, ColRstInit, - {roPreferMarkdown, roSupportMarkdown, roNimFile}, - msgHandler=testMsgHandler) - result = r.node - suite "RST parsing": test "References are whitespace-neutral and case-insensitive": # refname is 'lexical-analysis', the same for all the 3 variants: @@ -909,139 +895,3 @@ suite "RST inline markup": rnLeaf 'my {link example' rnLeaf 'http://example.com/bracket_(symbol_[)' """) - -suite "Integration with Nim": - test "simple symbol parsing (shortest form)": - let input1 = "g_".rstParseTest - check input1.toLangSymbol == LangSymbol(symKind: "", name: "g") - - test "simple symbol parsing (group of words)": - let input1 = "`Y`_".rstParseTest - check input1.toLangSymbol == LangSymbol(symKind: "", name: "Y") - - # this means not a statement 'type', it's a backticked identifier `type`: - let input2 = "`type`_".rstParseTest - check input2.toLangSymbol == LangSymbol(symKind: "", name: "type") - - let input3 = "`[]`_".rstParseTest - check input3.toLangSymbol == LangSymbol(symKind: "", name: "[]") - - let input4 = "`X Y Z`_".rstParseTest - check input4.toLangSymbol == LangSymbol(symKind: "", name: "Xyz") - - test "simple proc parsing": - let input1 = "proc f".rstParseTest - check input1.toLangSymbol == LangSymbol(symKind: "proc", name: "f") - - test "another backticked name": - let input1 = """`template \`type\``_""".rstParseTest - check input1.toLangSymbol == LangSymbol(symKind: "template", name: "type") - - test "simple proc parsing with parameters": - let input1 = "`proc f*()`_".rstParseTest - let input2 = "`proc f()`_".rstParseTest - let expected = LangSymbol(symKind: "proc", name: "f", - parametersProvided: true) - check input1.toLangSymbol == expected - check input2.toLangSymbol == expected - - test "symbol parsing with 1 parameter": - let input = "`f(G[int])`_".rstParseTest - let expected = LangSymbol(symKind: "", name: "f", - parameters: @[("G[int]", "")], - parametersProvided: true) - check input.toLangSymbol == expected - - test "more proc parsing": - let input1 = "`proc f[T](x:G[T]):M[T]`_".rstParseTest - let input2 = "`proc f[ T ] ( x: G [T] ): M[T]`_".rstParseTest - let input3 = "`proc f*[T](x: G[T]): M[T]`_".rstParseTest - let expected = LangSymbol(symKind: "proc", - name: "f", - generics: "[T]", - parameters: @[("x", "G[T]")], - parametersProvided: true, - outType: "M[T]") - check(input1.toLangSymbol == expected) - check(input2.toLangSymbol == expected) - check(input3.toLangSymbol == expected) - - test "advanced proc parsing with Nim identifier normalization": - let input = """`proc binarySearch*[T, K](a: openArray[T]; key: K; - cmp: proc (x: T; y: K): int)`_""".rstParseTest - let expected = LangSymbol(symKind: "proc", - name: "binarysearch", - generics: "[T,K]", - parameters: @[ - ("a", "openarray[T]"), - ("key", "K"), - ("cmp", "proc(x:T;y:K):int")], - parametersProvided: true, - outType: "") - check(input.toLangSymbol == expected) - - test "the same without proc": - let input = """`binarySearch*[T, K](a: openArray[T]; key: K; - cmp: proc (x: T; y: K): int {.closure.})`_""".rstParseTest - let expected = LangSymbol(symKind: "", - name: "binarysearch", - generics: "[T,K]", - parameters: @[ - ("a", "openarray[T]"), - ("key", "K"), - ("cmp", "proc(x:T;y:K):int")], - parametersProvided: true, - outType: "") - check(input.toLangSymbol == expected) - - test "operator $ with and without backticks": - let input1 = """`func \`$\`*[T](a: \`open Array\`[T]): string`_""". - rstParseTest - let input2 = """`func $*[T](a: \`open Array\`[T]): string`_""". - rstParseTest - let expected = LangSymbol(symKind: "func", - name: "$", - generics: "[T]", - parameters: @[("a", "openarray[T]")], - parametersProvided: true, - outType: "string") - check(input1.toLangSymbol == expected) - check(input2.toLangSymbol == expected) - - test "operator [] with and without backticks": - let input1 = """`func \`[]\`[T](a: \`open Array\`[T], idx: int): T`_""". - rstParseTest - let input2 = """`func [][T](a: \`open Array\`[T], idx: int): T`_""". - rstParseTest - let expected = LangSymbol(symKind: "func", - name: "[]", - generics: "[T]", - parameters: @[("a", "openarray[T]"), - ("idx", "int")], - parametersProvided: true, - outType: "T") - check(input1.toLangSymbol == expected) - check(input2.toLangSymbol == expected) - - test "postfix symbol specifier #1": - let input = """`walkDir iterator`_""". - rstParseTest - let expected = LangSymbol(symKind: "iterator", - name: "walkdir") - check(input.toLangSymbol == expected) - - test "postfix symbol specifier #2": - let input1 = """`\`[]\`[T](a: \`open Array\`[T], idx: int): T func`_""". - rstParseTest - let input2 = """`[][T](a: \`open Array\`[T], idx: int): T func`_""". - rstParseTest - let expected = LangSymbol(symKind: "func", - name: "[]", - generics: "[T]", - parameters: @[("a", "openarray[T]"), - ("idx", "int")], - parametersProvided: true, - outType: "T") - check(input1.toLangSymbol == expected) - check(input2.toLangSymbol == expected) - From 014df3e7c16786ca4efcd0e233c7406507046882 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Thu, 21 Oct 2021 00:47:11 +0300 Subject: [PATCH 4/4] add a note in strutils.nimIdentNormalize --- lib/pure/strutils.nim | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index 873949ca6745..2234a8625eb1 100644 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -273,6 +273,11 @@ func nimIdentNormalize*(s: string): string = ## ## That means to convert to lower case and remove any '_' on all characters ## except first one. + ## + ## .. Warning:: Backticks (`) are not handled: they remain *as is* and + ## spaces are preserved. See `nimIdentBackticksNormalize + ## `_ for + ## an alternative approach. runnableExamples: doAssert nimIdentNormalize("Foo_bar") == "Foobar" result = newString(s.len)