Skip to content

Commit 523c0fa

Browse files
mbrukmanmbrukman
committed
Automatically include sample code into user guide
This change add scripts to generate `user_guide.md` automatically from special markers that are now visible in the source file `user_guide.md.in`. This allows us to easily keep the source code up-to-date in the docs without having to manually copy-paste the code, and to ensure that the code we test (in the example files) is exactly the code that ends up in the documentation. Closes #54
1 parent 1a65517 commit 523c0fa

21 files changed

+553
-28
lines changed

‎.gitignore

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
# C compiled objects
2+
*.o
3+
4+
# Python generated files
5+
*.pyc

‎docs/Makefile

+31
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
# Copyright 2019 Google Inc.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
VERB = @
16+
ifeq ($(VERBOSE),1)
17+
VERB =
18+
endif
19+
20+
gen: user_guide.md
21+
22+
%.md: %.md.in doc_gen.py
23+
$(VERB) ./doc_gen.py $< > $@
24+
25+
diff:
26+
$(VERB) ./doc_gen_diff.sh
27+
28+
test:
29+
$(VERB) python -m unittest discover -p '*_test.py'
30+
31+
all-tests: diff test

‎docs/doc_gen.py

+107
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
#!/usr/bin/python
2+
#
3+
# Copyright 2019 Google Inc.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
"""Generates user_guide.md from user_guide.md.in ."""
18+
19+
import json
20+
import os
21+
import re
22+
import sys
23+
24+
25+
def ReadFileRaw(filename):
26+
with open(filename, 'r') as source:
27+
for line in source:
28+
yield line
29+
30+
31+
def ReadFileContentsWithMarker(filename, marker):
32+
begin_comment_c = re.compile(r'^/\* BEGIN: (\w+) \*/$')
33+
end_comment_c = re.compile(r'^/\* END: (\w+) \*/$')
34+
begin_comment_cpp = re.compile(r'^// BEGIN: (\w+)$')
35+
end_comment_cpp = re.compile(r'^// END: (\w+)$')
36+
37+
def Matches(matcherA, matcherB, content):
38+
return (matcherA.match(content), matcherB.match(content))
39+
40+
def Valid(matches):
41+
return matches[0] or matches[1]
42+
43+
def Group1(matches):
44+
if matches[0]:
45+
return matches[0].group(1)
46+
else:
47+
return matches[1].group(1)
48+
49+
output = False
50+
for line in ReadFileRaw(filename):
51+
begin_matches = Matches(begin_comment_c, begin_comment_cpp, line)
52+
if Valid(begin_matches):
53+
begin_marker = Group1(begin_matches)
54+
if begin_marker == marker:
55+
yield '~~~c\n' # Markdown C formatting header
56+
output = True
57+
continue # avoid outputting our own begin line
58+
59+
end_matches = Matches(end_comment_c, end_comment_cpp, line)
60+
if Valid(end_matches):
61+
end_marker = Group1(end_matches)
62+
if end_marker == marker:
63+
yield '~~~\n' # Markdown formatting end block
64+
return # we are done with this region
65+
66+
if output:
67+
yield line # enables outputting nested region markers
68+
69+
70+
def ProcessFile(source):
71+
pattern = re.compile(r'^{({.*})}$')
72+
for line in source:
73+
match = pattern.match(line)
74+
if not match:
75+
yield line
76+
continue
77+
78+
# Handle special include
79+
params = json.loads(match.group(1))
80+
full_path = params['source']
81+
base_name = os.path.basename(full_path)
82+
yield '[`%s`](%s)\n' % (base_name, full_path)
83+
yield '\n'
84+
marker = params['marker']
85+
for item in ReadFileContentsWithMarker(full_path, marker):
86+
yield item
87+
88+
89+
def main(argv):
90+
if len(argv) < 2:
91+
sys.stderr.write('Syntax: %s [filename]\n' % argv[0])
92+
sys.exit(1)
93+
94+
filename = argv[1]
95+
with open(filename, 'r') as source:
96+
sys.stdout.write('''\
97+
<!-- This file was auto-generated by `%s %s`.
98+
Do not modify manually; changes will be overwritten. -->
99+
100+
''' % (argv[0], argv[1]))
101+
results = ProcessFile(source.readlines())
102+
for line in results:
103+
sys.stdout.write('%s' % line)
104+
105+
106+
if __name__ == '__main__':
107+
main(sys.argv)

‎docs/doc_gen_diff.sh

+23
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
#!/bin/bash -eu
2+
#
3+
# Copyright 2019 Google Inc.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
# TODO(mbrukman): figure out how to encode this in Makefile syntax for
18+
# simplicity. In particular, the diff command with subcommand diff does not seem
19+
# to work and it's unclear what the correct syntax is for this, so keeping this
20+
# in a shell script for now.
21+
for md in *.md.in ; do
22+
diff -u <(./doc_gen.py "${md}") "${md%.in}"
23+
done

‎docs/doc_gen_test.py

+119
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
#!/usr/bin/python
2+
#
3+
# Copyright 2019 Google Inc.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
"""Tests for the `doc_gen` module."""
18+
19+
import unittest
20+
import doc_gen
21+
22+
23+
class ProcessFileTest(unittest.TestCase):
24+
25+
def setUp(self):
26+
self.docgen_readfile = None
27+
28+
29+
def tearDown(self):
30+
if self.docgen_readfile is not None:
31+
doc_gen.ReadFileRaw = self.docgen_readfile
32+
33+
34+
def _setCustomReadFile(self, filepath, contents):
35+
self.docgen_readfile = doc_gen.ReadFileRaw
36+
def readFileCustom(filename):
37+
if filename == filepath:
38+
return contents
39+
else:
40+
raise Exception('Expected file: "%s", received: "%s"' %
41+
(filepath, filename))
42+
43+
doc_gen.ReadFileRaw = readFileCustom
44+
45+
46+
def testPreC99Comments(self):
47+
source_file = 'file.c'
48+
source_file_fullpath = '../src/examples/%s' % source_file
49+
marker = 'docs'
50+
source_file_contents = [
51+
'/* license header comment */\n',
52+
'\n',
53+
'/* BEGIN: %s */\n' % marker,
54+
'#include <stdio.h>\n',
55+
'/* END: %s */\n' % marker,
56+
'int main(int argc, char** argv) {\n',
57+
' return 0;\n',
58+
'}\n',
59+
]
60+
61+
md_file_contents = [
62+
'text before code\n',
63+
'{{"source": "%s", "marker": "%s"}}\n' % (source_file_fullpath, marker),
64+
'text after code\n',
65+
]
66+
67+
expected_output = [
68+
'text before code\n',
69+
'[`%s`](%s)\n' % (source_file, source_file_fullpath),
70+
'\n',
71+
'~~~c\n',
72+
'#include <stdio.h>\n',
73+
'~~~\n',
74+
'text after code\n',
75+
]
76+
77+
self._setCustomReadFile(source_file_fullpath, source_file_contents)
78+
actual_output = list(doc_gen.ProcessFile(md_file_contents))
79+
self.assertEquals(expected_output, actual_output)
80+
81+
82+
def testC99Comments(self):
83+
source_file = 'file.c'
84+
source_file_fullpath = '../src/examples/%s' % source_file
85+
marker = 'docs'
86+
source_file_contents = [
87+
'/* license header comment */\n',
88+
'\n',
89+
'// BEGIN: %s\n' % marker,
90+
'#include <stdio.h>\n',
91+
'// END: %s\n' % marker,
92+
'int main(int argc, char** argv) {\n',
93+
' return 0;\n',
94+
'}\n',
95+
]
96+
97+
md_file_contents = [
98+
'text before code\n',
99+
'{{"source": "%s", "marker": "%s"}}\n' % (source_file_fullpath, marker),
100+
'text after code\n',
101+
]
102+
103+
expected_output = [
104+
'text before code\n',
105+
'[`%s`](%s)\n' % (source_file, source_file_fullpath),
106+
'\n',
107+
'~~~c\n',
108+
'#include <stdio.h>\n',
109+
'~~~\n',
110+
'text after code\n',
111+
]
112+
113+
self._setCustomReadFile(source_file_fullpath, source_file_contents)
114+
actual_output = list(doc_gen.ProcessFile(md_file_contents))
115+
self.assertEquals(expected_output, actual_output)
116+
117+
118+
if __name__ == '__main__':
119+
unittest.main()

0 commit comments

Comments
 (0)