Skip to content

Commit

Permalink
docgen: implement doc link resolution in current module
Browse files Browse the repository at this point in the history
no special handling of routineKinds here

fix segfault

fix references in the Manual

a supplemental test

generics: rm type parameters in normalized names

allow underscore in names and export (*)

many fixes of warnings:

- always resolve all the 3 cases:
  manual references, RST anchors, Nim anchors and print
  warnings in case of ambiguity
- preserve anchor/substitution definition places
- correctly handle `include` in ``.nim`` files for doc comment
  warnings/errors
- make warning messages more detailed

change wording & add test regarding brackets in input parameter types

some progress:

- fixes for handling symbols with backticks
- now reference points to concrete function (not group) if there is no
  other overloads
- link text is the full name (with function parameters) if there is
  only 1 overload. If there are > 1 then it's like
  "proc binarySearch (2 overloads)"

add more testcases, fix func

add specification of the feature

[skip ci] update the spec

reworked the implementation + more tests

fix ambiguity handling and reporting,

including adding RST explicit hyperlinks to ambiguity printing

Default priorities now conform to the `docgen.rst` spec also.

enable `doc2tex`

fix generics parameters for types

some cleanups and clarifications

tests on inclusion of .rst & .nim into .nim

update forgotten files for tests
  • Loading branch information
a-mr committed Sep 10, 2021
1 parent 0ef8305 commit fcd064b
Show file tree
Hide file tree
Showing 22 changed files with 1,782 additions and 141 deletions.
154 changes: 138 additions & 16 deletions compiler/docgen.nim
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
import
ast, strutils, strtabs, algorithm, sequtils, options, msgs, os, idents,
wordrecg, syntaxes, renderer, lexer,
packages/docutils/rst, packages/docutils/rstgen,
packages/docutils/[rst, rstgen, dochelpers],
json, xmltree, trees, types,
typesrenderer, astalgo, lineinfos, intsets,
pathutils, tables, nimpaths, renderverbatim, osproc
Expand Down Expand Up @@ -44,8 +44,14 @@ type
## runnableExamples).
substitutions: seq[string] ## Variable names in `doc.item`...
sortName: string ## The string used for sorting in output
info: rstast.TLineInfo ## place where symbol was defined (for messages)
anchor: string ## e.g. HTML anchor
name: string ## short name of the symbol, not unique
## (includes backticks ` if present)
detailedName: string ## longer name like `proc search(x: int): int`
ModSection = object ## Section like Procs, Types, etc.
secItems: seq[Item] ## Pre-processed items.
secItems: Table[string, seq[Item]]
## Map basic name -> pre-processed items.
finalMarkup: string ## The items, after RST pass 2 and rendering.
ModSections = array[TSymKind, ModSection]
TocItem = object ## HTML TOC item
Expand Down Expand Up @@ -91,12 +97,22 @@ type
thisDir*: AbsoluteDir
exampleGroups: OrderedTable[string, ExampleGroup]
wroteSupportFiles*: bool
nimToRstFid: Table[lineinfos.FileIndex, rstast.FileIndex]
## map Nim FileIndex -> RST one, it's needed because we keep them separate

PDoc* = ref TDocumentor ## Alias to type less.

proc add(dest: var ItemPre, rst: PRstNode) = dest.add ItemFragment(isRst: true, rst: rst)
proc add(dest: var ItemPre, str: string) = dest.add ItemFragment(isRst: false, str: str)

proc addRstFileIndex(d: PDoc, info: lineinfos.TLineInfo): rstast.FileIndex =
let invalid = rstast.FileIndex(-1)
result = d.nimToRstFid.getOrDefault(info.fileIndex, default = invalid)
if result == invalid:
let fname = toFullPath(d.conf, info)
result = addFilename(d.sharedState, fname)
d.nimToRstFid[info.fileIndex] = result

proc cmpDecimalsIgnoreCase(a, b: string): int =
## For sorting with correct handling of cases like 'uint8' and 'uint16'.
## Also handles leading zeros well (however note that leading zeros are
Expand Down Expand Up @@ -223,6 +239,7 @@ template declareClosures =
of meFootnoteMismatch: k = errRstFootnoteMismatch
of mwRedefinitionOfLabel: k = warnRstRedefinitionOfLabel
of mwUnknownSubstitution: k = warnRstUnknownSubstitutionX
of mwAmbiguousLink: k = warnRstAmbiguousLink
of mwBrokenLink: k = warnRstBrokenLink
of mwUnsupportedLanguage: k = warnRstLanguageXNotSupported
of mwUnsupportedField: k = warnRstFieldXNotSupported
Expand All @@ -236,7 +253,7 @@ template declareClosures =
result = getCurrentDir() / s
if not fileExists(result): result = ""

proc parseRst(text, filename: string,
proc parseRst(text: string,
line, column: int,
conf: ConfigRef, sharedState: PRstSharedState): PRstNode =
declareClosures()
Expand Down Expand Up @@ -352,7 +369,8 @@ proc getVarIdx(varnames: openArray[string], id: string): int =

proc genComment(d: PDoc, n: PNode): PRstNode =
if n.comment.len > 0:
result = parseRst(n.comment, toFullPath(d.conf, n.info),
d.sharedState.currFileIdx = addRstFileIndex(d, n.info)
result = parseRst(n.comment,
toLinenumber(n.info),
toColumn(n.info) + DocColOffset,
d.conf, d.sharedState)
Expand Down Expand Up @@ -885,6 +903,57 @@ proc genSeeSrc(d: PDoc, path: string, line: int): string =
"path", path.string, "line", $line, "url", gitUrl,
"commit", commit, "devel", develBranch]])

proc symbolPriority(k: TSymKind): int =
result = case k
of skMacro: -3
of skTemplate: -2
of skIterator: -1
else: 0 # including skProc which have higher priority
# documentation itself has even higher priority 1

proc toLangSymbol(k: TSymKind, n: PNode, baseName: string): LangSymbol =
## Converts symbol info (names/types/parameters) in `n` into format
## `LangSymbol` convenient for ``rst.nim``/``dochelpers.nim``.
result.name = baseName.nimIdentNormalize
result.symKind = k.toHumanStr
if k in routineKinds:
var
paramTypes: seq[string]
renderParamTypes(paramTypes, n[paramsPos], toNormalize=true)
let paramNames = renderParamNames(n[paramsPos], toNormalize=true)
# In some rare cases (system.typeof) parameter type is not set for default:
doAssert paramTypes.len <= paramNames.len
for i in 0 ..< paramNames.len:
if i < paramTypes.len:
result.parameters.add (paramNames[i], paramTypes[i])
else:
result.parameters.add (paramNames[i], "")
result.parametersProvided = true

result.outType = renderOutType(n[paramsPos], toNormalize=true)

if k in {skProc, skFunc, skType, skIterator}:
# Obtain `result.generics`
# Use `n[miscPos]` since n[genericParamsPos] does not contain constraints
var genNode: PNode = nil
if k == skType:
genNode = n[1] # FIXME: what is index 1?
else:
if n[miscPos].kind != nkEmpty:
genNode = n[miscPos][1] # FIXME: what is index 1?
if genNode != nil:
var literal = ""
var r: TSrcGen
initTokRender(r, genNode, {renderNoBody, renderNoComments,
renderNoPragmas, renderNoProcDefs})
var kind = tkEof
while true:
getNextTok(r, kind, literal)
if kind == tkEof:
break
if kind != tkSpaces:
result.generics.add(literal.nimIdentNormalize)

proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) =
if (docFlags != kForceExport) and not isVisible(d, nameNode): return
let
Expand Down Expand Up @@ -915,6 +984,8 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) =
inc(d.id)
let
plainNameEsc = esc(d.target, plainName.strip)
detailedName = k.toHumanStr & " " & (
if k in routineKinds: plainName else: name)
uniqueName = if k in routineKinds: plainNameEsc else: name
sortName = if k in routineKinds: plainName.strip else: name
cleanPlainSymbol = renderPlainSymbolName(nameNode)
Expand All @@ -923,20 +994,32 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) =
symbolOrId = d.newUniquePlainSymbol(complexSymbol)
symbolOrIdEnc = encodeUrl(symbolOrId, usePlus = false)
deprecationMsg = genDeprecationMsg(d, pragmaNode)
rstLangSymbol = toLangSymbol(k, n, cleanPlainSymbol)

# we generate anchors automatically for subsequent use in doc comments
let lineinfo = rstast.TLineInfo(
line: nameNode.info.line, col: nameNode.info.col,
fileIndex: addRstFileIndex(d, nameNode.info))
addAnchorNim(d.sharedState, refn = symbolOrId, tooltip = detailedName,
rstLangSymbol, priority = symbolPriority(k), info = lineinfo)

nodeToHighlightedHtml(d, n, result, {renderNoBody, renderNoComments,
renderDocComments, renderSyms}, symbolOrIdEnc)

let seeSrc = genSeeSrc(d, toFullPath(d.conf, n.info), n.info.line.int)

d.section[k].secItems.add Item(
d.section[k].secItems.mgetOrPut(cleanPlainSymbol, newSeq[Item]()).add Item(
descRst: comm,
sortName: sortName,
info: lineinfo,
anchor: symbolOrId,
detailedName: detailedName,
name: name,
substitutions: @[
"name", name, "uniqueName", uniqueName,
"uniqueName", uniqueName,
"header", result, "itemID", $d.id,
"header_plain", plainNameEsc, "itemSym", cleanPlainSymbol,
"itemSymOrID", symbolOrId, "itemSymEnc", plainSymbolEnc,
"itemSymEnc", plainSymbolEnc,
"itemSymOrIDEnc", symbolOrIdEnc, "seeSrc", seeSrc,
"deprecationMsg", deprecationMsg])

Expand Down Expand Up @@ -1184,6 +1267,11 @@ proc generateDoc*(d: PDoc, n, orig: PNode, docFlags: DocFlags = kDefault) =
if comm.len != 0: d.modDescPre.add(comm)
else: discard

proc overloadGroupName(s: string, k: TSymKind): string =
## Turns a name like `f` into anchor `f-procs-all`
#s & " " & k.toHumanStr & "s all"
s & "-" & k.toHumanStr & "s-all"

proc finishGenerateDoc*(d: var PDoc) =
## Perform 2nd RST pass for resolution of links/footnotes/headings...
# copy file map `filenames` to ``rstgen.nim`` for its warnings
Expand All @@ -1197,6 +1285,21 @@ proc finishGenerateDoc*(d: var PDoc) =
break
preparePass2(d.sharedState, firstRst)

# add anchors to overload groups before RST resolution
for k in TSymKind:
if k in routineKinds:
for plainName, overloadChoices in d.section[k].secItems:
if overloadChoices.len > 1:
let refn = overloadGroupName(plainName, k)
let tooltip = "$1 ($2 overloads)" % [
k.toHumanStr & " " & plainName, $overloadChoices.len]
addAnchorNim(d.sharedState, refn, tooltip,
LangSymbol(symKind: k.toHumanStr, name: plainName,
isGroup: true),
priority = symbolPriority(k),
# select index `0` just to have any meaningful warning:
info = overloadChoices[0].info)

# Finalize fragments of ``.nim`` or ``.rst`` file
proc renderItemPre(d: PDoc, fragments: ItemPre, result: var string) =
for f in fragments:
Expand All @@ -1207,14 +1310,33 @@ proc finishGenerateDoc*(d: var PDoc) =
of false: result &= f.str
proc cmp(x, y: Item): int = cmpDecimalsIgnoreCase(x.sortName, y.sortName)
for k in TSymKind:
for item in d.section[k].secItems.sorted(cmp):
var itemDesc: string
renderItemPre(d, item.descRst, itemDesc)
d.section[k].finalMarkup.add(
getConfigVar(d.conf, "doc.item") % (
item.substitutions & @["desc", itemDesc]))
itemDesc = ""
d.section[k].secItems.setLen 0
# add symbols to section for each `k`, while optionally wrapping
# overloadable items with the same basic name by ``doc.item2``
let overloadableNames = toSeq(keys(d.section[k].secItems))
for plainName in overloadableNames.sorted(cmpDecimalsIgnoreCase):
var overloadChoices = d.section[k].secItems[plainName]
overloadChoices.sort(cmp)
var nameContent = ""
for item in overloadChoices:
var itemDesc: string
renderItemPre(d, item.descRst, itemDesc)
nameContent.add(
getConfigVar(d.conf, "doc.item") % (
item.substitutions & @[
"desc", itemDesc,
"name", item.name,
"itemSymOrID", item.anchor]))
if k in routineKinds:
let plainNameEsc1 = esc(d.target, plainName.strip)
let plainNameEsc2 = esc(d.target, plainName.strip, escMode=emUrl)
d.section[k].finalMarkup.add(
getConfigVar(d.conf, "doc.item2") % (
@["header_plain", plainNameEsc1,
"overloadGroupName", overloadGroupName(plainNameEsc2, k),
"content", nameContent]))
else:
d.section[k].finalMarkup.add(nameContent)
d.section[k].secItems.clear
renderItemPre(d, d.modDescPre, d.modDescFinal)
d.modDescPre.setLen 0
d.hasToc = d.hasToc or d.sharedState.hasToc
Expand Down Expand Up @@ -1493,7 +1615,7 @@ proc commandRstAux(cache: IdentCache, conf: ConfigRef;
filename: AbsoluteFile, outExt: string) =
var filen = addFileExt(filename, "txt")
var d = newDocumentor(filen, cache, conf, outExt, isPureRst = true)
let rst = parseRst(readFile(filen.string), filen.string,
let rst = parseRst(readFile(filen.string),
line=LineRstInit, column=ColRstInit,
conf, d.sharedState)
d.modDescPre = @[ItemFragment(isRst: true, rst: rst)]
Expand Down
2 changes: 2 additions & 0 deletions compiler/lineinfos.nim
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ type
warnSmallLshouldNotBeUsed = "SmallLshouldNotBeUsed", warnUnknownMagic = "UnknownMagic",
warnRstRedefinitionOfLabel = "RedefinitionOfLabel",
warnRstUnknownSubstitutionX = "UnknownSubstitutionX",
warnRstAmbiguousLink = "AmbiguousLink",
warnRstBrokenLink = "BrokenLink",
warnRstLanguageXNotSupported = "LanguageXNotSupported",
warnRstFieldXNotSupported = "FieldXNotSupported",
Expand Down Expand Up @@ -123,6 +124,7 @@ const
warnUnknownMagic: "unknown magic '$1' might crash the compiler",
warnRstRedefinitionOfLabel: "redefinition of label '$1'",
warnRstUnknownSubstitutionX: "unknown substitution '$1'",
warnRstAmbiguousLink: "ambiguous doc link $1",
warnRstBrokenLink: "broken link '$1'",
warnRstLanguageXNotSupported: "language '$1' not supported",
warnRstFieldXNotSupported: "field '$1' not supported",
Expand Down
Loading

0 comments on commit fcd064b

Please sign in to comment.