Skip to content

Commit 0b0e6ba

Browse files
committed
further progress on rst roles & dir-s (fix nim-lang#17646)
1 parent 34c1c63 commit 0b0e6ba

File tree

2 files changed

+106
-33
lines changed

2 files changed

+106
-33
lines changed

lib/packages/docutils/rst.nim

+74-33
Original file line numberDiff line numberDiff line change
@@ -912,6 +912,38 @@ template newLeaf(s: string): PRstNode = newRstLeaf(s)
912912
proc newLeaf(p: var RstParser): PRstNode =
913913
result = newLeaf(currentTok(p).symbol)
914914

915+
proc validRefnamePunct(x: string): bool =
916+
## https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html#reference-names
917+
x.len == 1 and x[0] in {'-', '_', '.', ':', '+'}
918+
919+
func getRefnameIdx(p: RstParser, startIdx: int): int =
920+
## Gets last token index of a refname ("word" in RST terminology):
921+
##
922+
## reference names are single words consisting of alphanumerics plus
923+
## isolated (no two adjacent) internal hyphens, underscores, periods,
924+
## colons and plus signs; no whitespace or other characters are allowed.
925+
##
926+
## Refnames are used for:
927+
## - reference names
928+
## - role names
929+
## - directive names
930+
## - footnote labels
931+
##
932+
# TODO: use this func in all other relevant places
933+
var j = startIdx
934+
if p.tok[j].kind == tkWord:
935+
inc j
936+
while p.tok[j].kind == tkPunct and validRefnamePunct(p.tok[j].symbol) and
937+
p.tok[j+1].kind == tkWord:
938+
inc j, 2
939+
result = j - 1
940+
941+
func getRefname(p: RstParser, startIdx: int): (string, int) =
942+
let lastIdx = getRefnameIdx(p, startIdx)
943+
result[1] = lastIdx
944+
for j in startIdx..lastIdx:
945+
result[0].add p.tok[j].symbol
946+
915947
proc getReferenceName(p: var RstParser, endStr: string): PRstNode =
916948
var res = newRstNode(rnInner)
917949
while true:
@@ -1011,7 +1043,10 @@ proc match(p: RstParser, start: int, expr: string): bool =
10111043
var last = expr.len - 1
10121044
while i <= last:
10131045
case expr[i]
1014-
of 'w': result = p.tok[j].kind == tkWord
1046+
of 'w':
1047+
let lastIdx = getRefnameIdx(p, j)
1048+
result = lastIdx >= j
1049+
if result: j = lastIdx
10151050
of ' ': result = p.tok[j].kind == tkWhite
10161051
of 'i': result = p.tok[j].kind == tkIndent
10171052
of 'I': result = p.tok[j].kind in {tkIndent, tkEof}
@@ -1058,7 +1093,7 @@ proc fixupEmbeddedRef(n, a, b: PRstNode) =
10581093
proc whichRole(p: RstParser, sym: string): RstNodeKind =
10591094
result = whichRoleAux(sym)
10601095
if result == rnUnknownRole:
1061-
rstMessage(p, mwUnsupportedLanguage, p.s.currRole)
1096+
rstMessage(p, mwUnsupportedLanguage, sym)
10621097

10631098
proc toInlineCode(n: PRstNode, language: string): PRstNode =
10641099
## Creates rnInlineCode and attaches `n` contents as code (in 3rd son).
@@ -1078,6 +1113,11 @@ proc toInlineCode(n: PRstNode, language: string): PRstNode =
10781113
lb.add newLeaf(s)
10791114
result.add lb
10801115

1116+
proc toUnknownRole(n: PRstNode, roleName: string): PRstNode =
1117+
let newN = newRstNode(rnInner, n.sons)
1118+
let newSons = @[newN, newLeaf(roleName)]
1119+
result = newRstNode(rnUnknownRole, newSons)
1120+
10811121
proc parsePostfix(p: var RstParser, n: PRstNode): PRstNode =
10821122
var newKind = n.kind
10831123
var newSons = n.sons
@@ -1102,17 +1142,15 @@ proc parsePostfix(p: var RstParser, n: PRstNode): PRstNode =
11021142
result = newRstNode(newKind, newSons)
11031143
elif match(p, p.idx, ":w:"):
11041144
# a role:
1105-
let roleName = nextTok(p).symbol
1145+
let (roleName, lastIdx) = getRefname(p, p.idx+1)
11061146
newKind = whichRole(p, roleName)
11071147
if newKind == rnUnknownRole:
1108-
let newN = newRstNode(rnInner, n.sons)
1109-
newSons = @[newN, newLeaf(roleName)]
1110-
result = newRstNode(newKind, newSons)
1148+
result = n.toUnknownRole(roleName)
11111149
elif newKind == rnInlineCode:
11121150
result = n.toInlineCode(language=roleName)
11131151
else:
11141152
result = newRstNode(newKind, newSons)
1115-
inc p.idx, 3
1153+
p.idx = lastIdx + 2
11161154
else:
11171155
if p.s.currRoleKind == rnInlineCode:
11181156
result = n.toInlineCode(language=p.s.currRole)
@@ -1139,10 +1177,6 @@ proc parseSmiley(p: var RstParser): PRstNode =
11391177
result.text = val
11401178
return
11411179

1142-
proc validRefnamePunct(x: string): bool =
1143-
## https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html#reference-names
1144-
x.len == 1 and x[0] in {'-', '_', '.', ':', '+'}
1145-
11461180
proc isUrl(p: RstParser, i: int): bool =
11471181
result = p.tok[i+1].symbol == ":" and p.tok[i+2].symbol == "//" and
11481182
p.tok[i+3].kind == tkWord and
@@ -1373,14 +1407,18 @@ proc parseInline(p: var RstParser, father: PRstNode) =
13731407
var n = newRstNode(rnInlineLiteral)
13741408
parseUntil(p, n, "``", false)
13751409
father.add(n)
1376-
elif match(p, p.idx, ":w:") and p.tok[p.idx+3].symbol == "`":
1377-
let roleName = nextTok(p).symbol
1410+
elif match(p, p.idx, ":w:") and
1411+
(var lastIdx = getRefnameIdx(p, p.idx + 1);
1412+
p.tok[lastIdx+2].symbol == "`"):
1413+
let (roleName, _) = getRefname(p, p.idx+1)
13781414
let k = whichRole(p, roleName)
13791415
var n = newRstNode(k)
1380-
inc p.idx, 3
1416+
p.idx = lastIdx + 2
13811417
if k == rnInlineCode:
13821418
n = n.toInlineCode(language=roleName)
13831419
parseUntil(p, n, "`", false) # bug #17260
1420+
if k == rnUnknownRole:
1421+
n = n.toUnknownRole(roleName)
13841422
father.add(n)
13851423
elif isInlineMarkupStart(p, "`"):
13861424
var n = newRstNode(rnInterpretedText)
@@ -1438,25 +1476,28 @@ proc parseInline(p: var RstParser, father: PRstNode) =
14381476
else: discard
14391477

14401478
proc getDirective(p: var RstParser): string =
1441-
if currentTok(p).kind == tkWhite and nextTok(p).kind == tkWord:
1442-
var j = p.idx
1443-
inc p.idx
1444-
result = currentTok(p).symbol
1445-
inc p.idx
1446-
while currentTok(p).kind in {tkWord, tkPunct, tkAdornment, tkOther}:
1447-
if currentTok(p).symbol == "::": break
1448-
result.add(currentTok(p).symbol)
1449-
inc p.idx
1450-
if currentTok(p).kind == tkWhite: inc p.idx
1451-
if currentTok(p).symbol == "::":
1452-
inc p.idx
1453-
if currentTok(p).kind == tkWhite: inc p.idx
1454-
else:
1455-
p.idx = j # set back
1456-
result = "" # error
1457-
else:
1458-
result = ""
1459-
result = result.toLowerAscii()
1479+
result = ""
1480+
if currentTok(p).kind == tkWhite:
1481+
let (name, lastIdx) = getRefname(p, p.idx + 1)
1482+
let afterIdx = lastIdx + 1
1483+
if name.len > 0:
1484+
if p.tok[afterIdx].symbol == "::":
1485+
result = name
1486+
p.idx = afterIdx + 1
1487+
if currentTok(p).kind == tkWhite:
1488+
inc p.idx
1489+
elif currentTok(p).kind != tkIndent:
1490+
rstMessage(p, mwRstStyle,
1491+
"whitespace or newline expected after directive " & name)
1492+
result = result.toLowerAscii()
1493+
elif p.tok[afterIdx].symbol == ":":
1494+
rstMessage(p, mwRstStyle,
1495+
"double colon :: may be missing at end of '" & name & "'",
1496+
p.tok[afterIdx].line, p.tok[afterIdx].col)
1497+
elif p.tok[afterIdx].kind == tkPunct and p.tok[afterIdx].symbol[0] == ':':
1498+
rstMessage(p, mwRstStyle,
1499+
"too many colons for a directive (should be ::)",
1500+
p.tok[afterIdx].line, p.tok[afterIdx].col)
14601501

14611502
proc parseComment(p: var RstParser): PRstNode =
14621503
case currentTok(p).kind

tests/stdlib/trstgen.nim

+32
Original file line numberDiff line numberDiff line change
@@ -136,6 +136,21 @@ suite "YAML syntax highlighting":
136136
<span class="StringLit">not numbers</span><span class="Punctuation">:</span> <span class="Punctuation">[</span> <span class="StringLit">42e</span><span class="Punctuation">,</span> <span class="StringLit">0023</span><span class="Punctuation">,</span> <span class="StringLit">+32.37</span><span class="Punctuation">,</span> <span class="StringLit">8 ball</span><span class="Punctuation">]</span>
137137
<span class="Punctuation">}</span></pre>"""
138138

139+
test "Directives: warnings":
140+
let input = dedent"""
141+
.. non-existant-warning: Paragraph.
142+
143+
.. another.wrong:warning::: Paragraph.
144+
"""
145+
var warnings = new seq[string]
146+
let output = input.toHtml(warnings=warnings)
147+
check output == ""
148+
doAssert warnings[].len == 2
149+
check "(1, 24) Warning: RST style:" in warnings[0]
150+
check "double colon :: may be missing at end of 'non-existant-warning'" in warnings[0]
151+
check "(3, 25) Warning: RST style:" in warnings[1]
152+
check "RST style: too many colons for a directive (should be ::)" in warnings[1]
153+
139154
test "Anchors, Aliases, Tags":
140155
let input = """.. code-block:: yaml
141156
--- !!map
@@ -1403,6 +1418,23 @@ Test1
14031418
check """`3`:sup:\ He is an isotope of helium.""".toHtml == expected
14041419
check """`3`:superscript:\ He is an isotope of helium.""".toHtml == expected
14051420

1421+
test "Roles: warnings":
1422+
let input = dedent"""
1423+
See function :py:func:`spam`.
1424+
1425+
See also `egg`:py:class:.
1426+
"""
1427+
var warnings = new seq[string]
1428+
let output = input.toHtml(warnings=warnings)
1429+
doAssert warnings[].len == 2
1430+
check "(1, 14) Warning: " in warnings[0]
1431+
check "language 'py:func' not supported" in warnings[0]
1432+
check "(3, 15) Warning: " in warnings[1]
1433+
check "language 'py:class' not supported" in warnings[1]
1434+
check("""<p>See function <span class="py:func">spam</span>.</p>""" & "\n" &
1435+
"""<p>See also <span class="py:class">egg</span>. </p>""" & "\n" ==
1436+
output)
1437+
14061438
test "(not) Roles: check escaping 1":
14071439
let expected = """See :subscript:<tt class="docutils literal">""" &
14081440
"""<span class="pre">""" & id"some" & " " & id"text" &

0 commit comments

Comments
 (0)