Skip to content

Commit e070e20

Browse files
ehennestadbendichter
ehennestad
and
bendichter
authored
Add developer tools for livescripts (#626)
* Add functions for exporting tutorials * Create .gitattributes * Rerun and save all livescripts No content changes * Add copy of all livescripts as m-code files * Add section header for developer's section * Update exportTutorials.m Added more options to function * Update intro.mlx * Update export tutorials * prefix functions in tools folder with "matnwb_" to avoid potential name conflicts For now, use prefix instead of namespaces to avoid unnecessary folders * Create matnwb_listModifiedFiles.m * Update functions for exporting tutorials * Create matnwb_checkTutorials.m Function to run tests for and exports of modified tutorials. * Add facilities for installing and running a git pre-commit hook * move misc.generateDocs to tools and rename to matnwb_generateDocs + Anchor to matnwb root directory, not current directory * Update matnwb_generateDocs.m Update url for m2html source * Add function to install m2html dependency * Update and rename setup to matnwb_setup * Ignore some livescripts during export * Update livescript m-files variants * Restore livescripts to previous state * Fix livescript typos * Add optional mode and function to clean text for matnwb_listTutorialFiles * Update matnwb_checkTutorials - Rename variables to make logic clearer * Update pre-commit hooks. Undo running matlab code as this is very slow Provide shell scripts for mac and linux * Update matnwb_installGitHooks.m * Update pre-commit hooks to also check if html for main nwb api functions are up to date * Fix docstrings plus add useful warning * Update tutorials/private/README.md --------- Co-authored-by: Ben Dichter <[email protected]>
1 parent fa04ab0 commit e070e20

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

44 files changed

+4776
-27
lines changed

‎tools/documentation/matnwb_exportModifiedTutorials.m

+33
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
function matnwb_exportModifiedTutorials()
2+
% matnwb_exportModifiedTutorials - Export modified livescript tutorials to html
3+
%
4+
% See also matnwb_exportTutorials
5+
6+
if exist("isMATLABReleaseOlderThan", "file") == 2
7+
hasGitRepo = ~isMATLABReleaseOlderThan("R2023b");
8+
else
9+
hasGitRepo = false;
10+
end
11+
12+
if hasGitRepo
13+
repo = gitrepo(misc.getMatnwbDir);
14+
modifiedFiles = repo.ModifiedFiles;
15+
else
16+
modifiedFiles = matnwb_listModifiedFiles();
17+
end
18+
19+
tutorialFolder = fullfile(misc.getMatnwbDir, 'tutorials');
20+
isInTutorialFolder = startsWith(modifiedFiles, tutorialFolder);
21+
isLivescript = endsWith(modifiedFiles, ".mlx");
22+
23+
tutorialFiles = modifiedFiles(isInTutorialFolder & isLivescript);
24+
25+
filesToIgnore = ["basicUsage", "read_demo", "remote_read"];
26+
isIgnored = endsWith(tutorialFiles, filesToIgnore + ".mlx");
27+
if any(isIgnored)
28+
warning('Skipping export for the following files (see matnwb_exportTutorials):\n%s', ...
29+
strjoin(" - " + filesToIgnore(isIgnored) + ".mlx", newline))
30+
end
31+
32+
matnwb_exportTutorials("FilePaths", tutorialFiles, "IgnoreFiles", filesToIgnore)
33+
end

‎tools/documentation/matnwb_exportTutorials.m

+93
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
function matnwb_exportTutorials(options)
2+
% matnwb_exportTutorials - Export mlx tutorial files to the specified output format
3+
%
4+
% Note: This function will ignore the following live scripts:
5+
% - basicUsage.mlx : depends on output from convertTrials.m
6+
% - read_demo.mlx : depends on external data, potentially slow
7+
% - remote_read.mlx : Uses nwbRead on s3 url, potentially very slow]
8+
%
9+
% To export all livescripts (assuming you have made sure the above-mentioned
10+
% files will run) call the function with IgnoreFiles set to empty, i.e:
11+
% matnwb_exportTutorials(..., "IgnoreFiles", string.empty)
12+
13+
arguments
14+
options.ExportFormat (1,:) string {mustStartWithDot} = [".m", ".html"]
15+
options.Expression (1,1) string = "*" % Filter by expression
16+
options.FileNames (1,:) string = string.empty % Filter by file names
17+
options.FilePaths (1,:) string = string.empty % Export specified files
18+
options.IgnoreFiles (1,:) string = ["basicUsage", "read_demo", "remote_read"];
19+
options.RunLivescript (1,1) logical = true
20+
end
21+
22+
[exportFormat, targetFolderNames] = deal(options.ExportFormat);
23+
24+
targetFolderNames = extractAfter(targetFolderNames, ".");
25+
targetFolderNames(strcmp(targetFolderNames, "m")) = fullfile("private", "mcode");
26+
27+
nwbTutorialDir = fullfile(misc.getMatnwbDir, "tutorials");
28+
targetFolderPaths = fullfile(nwbTutorialDir, targetFolderNames);
29+
30+
for folderPath = targetFolderPaths
31+
if ~isfolder(folderPath); mkdir(folderPath); end
32+
end
33+
34+
if isempty(options.FilePaths)
35+
if endsWith(options.Expression, "*")
36+
expression = options.Expression + ".mlx";
37+
else
38+
expression = options.Expression + "*.mlx";
39+
end
40+
41+
L = dir(fullfile(nwbTutorialDir, expression));
42+
filePaths = string( fullfile({L.folder}, {L.name}) );
43+
else
44+
filePaths = options.FilePaths;
45+
end
46+
47+
[~, fileNames] = fileparts(filePaths);
48+
if ~isempty(options.FileNames)
49+
[fileNames, iA] = intersect(fileNames, options.FileNames, 'stable');
50+
filePaths = filePaths(iA);
51+
end
52+
53+
if ~isempty(options.IgnoreFiles)
54+
[~, fileNames] = fileparts(filePaths);
55+
[fileNames, iA] = setdiff(fileNames, options.IgnoreFiles, 'stable');
56+
filePaths = filePaths(iA);
57+
end
58+
59+
% Go to a temporary directory, so that tutorials are exported in a
60+
% temporary folder which is cleaned up afterwards
61+
currentDir = pwd();
62+
cleanupWorkdir = onCleanup(@(fp) cd(currentDir));
63+
64+
tempDir = fullfile(tempdir, 'nwbTutorials');
65+
if ~isfolder(tempDir); mkdir(tempDir); end
66+
disp('Changing into temporary directory:')
67+
cd(tempDir)
68+
69+
cleanupDeleteTempFiles = onCleanup(@(fp) rmdir(tempDir, 's'));
70+
disp(tempDir)
71+
72+
for i = 1:numel(filePaths)
73+
sourcePath = char( fullfile(filePaths(i)) );
74+
if options.RunLivescript
75+
fprintf('Running livescript "%s"\n', fileNames(i))
76+
77+
matlab.internal.liveeditor.executeAndSave(sourcePath);
78+
end
79+
80+
for j = 1:numel(exportFormat)
81+
targetPath = fullfile(targetFolderPaths(j), fileNames(i) + exportFormat(j));
82+
fprintf('Exporting livescript "%s" to "%s"\n', fileNames(i), exportFormat(j))
83+
export(sourcePath, strrep(targetPath, '.mlx', exportFormat(j)));
84+
end
85+
end
86+
end
87+
88+
function mustStartWithDot(value)
89+
for i = 1:numel(value)
90+
assert(startsWith(value(i), '.'), ...
91+
'Value must be a file extension starting with a period, e.g ".html"')
92+
end
93+
end

‎+misc/generateDocs.m renamed to ‎tools/documentation/matnwb_generateDocs.m

+20-10
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,35 @@
1-
function generateDocs()
2-
% GENERATEDOCS generates docs for MatNWB user API
3-
% GENERATEDOCS() generate documentation for MATLAB files in the current working directory.
1+
function matnwb_generateDocs()
2+
% MATNWB_GENERATEDOCS generates html docs for MatNWB API functions
43
%
5-
% Requires <a href="matlab:
6-
% web('https://www.artefact.tk/software/matlab/m2html/')">m2html</a> in your path.
7-
rootFiles = dir('.');
4+
% matnwb_generateDocs() generates html documentation for MATLAB files in the
5+
% current matnwb root directory.
6+
%
7+
% The following files are included:
8+
% - generateCore.m
9+
% - generateExtension.m
10+
% - nwbRead.m
11+
% - nwbExport.m
12+
%
13+
% Requires <a href="matlab:web('https://github.com/gllmflndn/m2html')">m2html</a> in your path.
14+
15+
rootDir = misc.getMatnwbDir();
16+
rootFiles = dir(rootDir);
817
rootFiles = {rootFiles.name};
918
rootWhitelist = {'generateCore.m', 'generateExtension.m', 'nwbRead.m', 'nwbExport.m'};
1019
isWhitelisted = ismember(rootFiles, rootWhitelist);
1120
rootFiles(~isWhitelisted) = [];
1221

13-
m2html('mfiles', rootFiles, 'htmldir', 'doc');
22+
docDir = fullfile(rootDir, 'doc');
23+
m2html('mfiles', rootFiles, 'htmldir', docDir);
1424

1525
% correct html files in root directory as the stylesheets will be broken
1626
fprintf('Correcting files in root directory...\n');
17-
rootFiles = dir('doc');
27+
rootFiles = dir(docDir);
1828
rootFiles = {rootFiles.name};
1929
htmlMatches = regexp(rootFiles, '\.html$', 'once');
2030
isHtmlFile = ~cellfun('isempty', htmlMatches);
2131
rootFiles(~isHtmlFile) = [];
22-
rootFiles = fullfile('doc', rootFiles);
32+
rootFiles = fullfile(docDir, rootFiles);
2333

2434
for iDoc=1:length(rootFiles)
2535
fileName = rootFiles{iDoc};
@@ -29,7 +39,7 @@ function generateDocs()
2939

3040
% correct index.html so the header indicates MatNWB
3141
fprintf('Correcting index.html Header...\n');
32-
indexPath = fullfile('doc', 'index.html');
42+
indexPath = fullfile(docDir, 'index.html');
3343
fileReplace(indexPath, 'Index for \.', 'Index for MatNWB');
3444

3545
% remove directories listing in index.html

‎tools/githooks/pre-commit-linux

+70
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
#!/bin/bash
2+
# NB: Not tested
3+
4+
# Relative paths
5+
TUTORIAL_FOLDER="tutorials"
6+
PRIVATE_FOLDER="$TUTORIAL_FOLDER/private"
7+
8+
# Define file mappings (script files and their corresponding documentation)
9+
SCRIPT_FILES=("generateCore.m" "generateExtension.m" "nwbRead.m" "nwbExport.m")
10+
DOC_FOLDER="doc"
11+
12+
# Get modified files (staged + unstaged)
13+
MODIFIED_FILES=$(git diff --cached --name-only)
14+
15+
# Check for .mlx files in the tutorials folder
16+
TUTORIAL_FILES=$(echo "$MODIFIED_FILES" | grep "^$TUTORIAL_FOLDER" | grep "\.mlx$")
17+
18+
# If there are tutorial files, validate them
19+
if [[ -n "$TUTORIAL_FILES" ]]; then
20+
echo "Checking tutorial files..."
21+
22+
for TUTORIAL_FILE in $TUTORIAL_FILES; do
23+
# Get the base name without extension
24+
BASENAME=$(basename "$TUTORIAL_FILE" .mlx)
25+
26+
# Find corresponding .html and .m files
27+
HTML_FILE=$(find "$TUTORIAL_FOLDER" -name "$BASENAME.html" -print -quit)
28+
MC_FILE=$(find "$PRIVATE_FOLDER" -name "$BASENAME.m" -print -quit)
29+
30+
# Get modification dates (default to 0 if file doesn't exist)
31+
TUTORIAL_FILE_DATE=$(stat -c "%Y" "$TUTORIAL_FILE")
32+
HTML_FILE_DATE=$(stat -c "%Y" "$HTML_FILE" 2>/dev/null || echo 0)
33+
MC_FILE_DATE=$(stat -c "%Y" "$MC_FILE" 2>/dev/null || echo 0)
34+
35+
# Check if .html or .m file is outdated
36+
if [[ "$TUTORIAL_FILE_DATE" -gt "$HTML_FILE_DATE" || "$TUTORIAL_FILE_DATE" -gt "$MC_FILE_DATE" ]]; then
37+
echo "Error: Please re-export live script \"$BASENAME.mlx\"." >&2
38+
exit 1
39+
fi
40+
done
41+
fi
42+
43+
# Flag to track if any files are outdated
44+
OUTDATED_FOUND=0
45+
46+
# Loop through each script file
47+
for SCRIPT_FILE in "${SCRIPT_FILES[@]}"; do
48+
# Check if the script file has been modified
49+
if echo "$MODIFIED_FILES" | grep -q "^$SCRIPT_FILE$"; then
50+
# Get the corresponding HTML file in the doc folder
51+
HTML_FILE="$DOC_FOLDER/${SCRIPT_FILE%.m}.html"
52+
53+
# Get modification dates (default to 0 if file doesn't exist)
54+
SCRIPT_FILE_DATE=$(stat -c "%Y" "$SCRIPT_FILE")
55+
HTML_FILE_DATE=$(stat -c "%Y" "$HTML_FILE" 2>/dev/null || echo 0)
56+
57+
# Check if the script is newer than the HTML file
58+
if [[ "$SCRIPT_FILE_DATE" -gt "$HTML_FILE_DATE" ]]; then
59+
echo "Error: Please re-export documentation for \"$SCRIPT_FILE\"." >&2
60+
OUTDATED_FOUND=1
61+
fi
62+
fi
63+
done
64+
65+
# Exit with error if any files are outdated
66+
if [[ $OUTDATED_FOUND -eq 1 ]]; then
67+
exit 1
68+
fi
69+
70+
exit 0

‎tools/githooks/pre-commit-mac

+68
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
#!/bin/bash
2+
3+
# Relative paths
4+
TUTORIAL_FOLDER="tutorials"
5+
PRIVATE_FOLDER="$TUTORIAL_FOLDER/private"
6+
7+
# Define file mappings (script files and their corresponding documentation)
8+
SCRIPT_FILES=("generateCore.m" "generateExtension.m" "nwbRead.m" "nwbExport.m")
9+
DOC_FOLDER="doc"
10+
11+
# Get modified files (staged)
12+
MODIFIED_FILES=$(git diff --cached --name-only)
13+
14+
# Check for .mlx files in the tutorials folder
15+
TUTORIAL_FILES=$(echo "$MODIFIED_FILES" | grep "^$TUTORIAL_FOLDER" | grep "\.mlx$")
16+
17+
# If there are tutorial files, check that they have been exported
18+
if [[ -n "$TUTORIAL_FILES" ]]; then
19+
20+
for TUTORIAL_FILE in $TUTORIAL_FILES; do
21+
# Get the base name without extension
22+
BASENAME=$(basename "$TUTORIAL_FILE" .mlx)
23+
24+
# Find corresponding .html and .m files
25+
HTML_FILE=$(find "$TUTORIAL_FOLDER" -name "$BASENAME.html" -print -quit)
26+
MC_FILE=$(find "$PRIVATE_FOLDER" -name "$BASENAME.m" -print -quit)
27+
28+
# Get modification dates (default to 0 if file doesn't exist)
29+
TUTORIAL_FILE_DATE=$(stat -f "%m" "$TUTORIAL_FILE")
30+
HTML_FILE_DATE=$(stat -f "%m" "$HTML_FILE" 2>/dev/null || echo 0)
31+
MC_FILE_DATE=$(stat -f "%m" "$MC_FILE" 2>/dev/null || echo 0)
32+
33+
# Check if .html or .m file is outdated
34+
if [[ "$TUTORIAL_FILE_DATE" -gt "$HTML_FILE_DATE" || "$TUTORIAL_FILE_DATE" -gt "$MC_FILE_DATE" ]]; then
35+
echo "Error: Please re-export html/m-files for live script \"$BASENAME.mlx\"." >&2
36+
exit 1
37+
fi
38+
done
39+
fi
40+
41+
# Flag to track if any files are outdated
42+
OUTDATED_FOUND=0
43+
44+
# Loop through each script file
45+
for SCRIPT_FILE in "${SCRIPT_FILES[@]}"; do
46+
# Check if the script file has been modified
47+
if echo "$MODIFIED_FILES" | grep -q "^$SCRIPT_FILE$"; then
48+
# Get the corresponding HTML file in the doc folder
49+
HTML_FILE="$DOC_FOLDER/${SCRIPT_FILE%.m}.html"
50+
51+
# Get modification dates (default to 0 if file doesn't exist)
52+
SCRIPT_FILE_DATE=$(stat -f "%m" "$SCRIPT_FILE")
53+
HTML_FILE_DATE=$(stat -f "%m" "$HTML_FILE" 2>/dev/null || echo 0)
54+
55+
# Check if the script is newer than the HTML file
56+
if [[ "$SCRIPT_FILE_DATE" -gt "$HTML_FILE_DATE" ]]; then
57+
echo "Error: Please re-export documentation for \"$SCRIPT_FILE\"." >&2
58+
OUTDATED_FOUND=1
59+
fi
60+
fi
61+
done
62+
63+
# Exit with error if any files are outdated
64+
if [[ $OUTDATED_FOUND -eq 1 ]]; then
65+
exit 1
66+
fi
67+
68+
exit 0

‎tools/maintenance/matnwb_checkTutorials.m

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
function matnwb_checkTutorials()
2+
% matnwb_checkTutorials - Checks for modified MATLAB Live Script tutorial files
3+
% in the repository and executes tests and html exports if found.
4+
%
5+
% This function determines whether any tutorial files in the `tutorials`
6+
% directory have been modified in the matnwb repository. If such files exist,
7+
% the function performs the following actions:
8+
% 1. Runs unit tests matching the tutorial names.
9+
% 2. Exports the modified tutorial files using the `matnwb_exportTutorials`
10+
% function.
11+
%
12+
% Usage:
13+
% matnwb_checkTutorials()
14+
%
15+
% See also matnwb_listModifiedFiles, matnwb_exportTutorials
16+
17+
tutorialFolder = fullfile(misc.getMatnwbDir, 'tutorials');
18+
19+
modifiedFiles = matnwb_listModifiedFiles("all");
20+
21+
isInTutorialFolder = startsWith(modifiedFiles, tutorialFolder);
22+
isLivescript = endsWith(modifiedFiles, ".mlx");
23+
24+
tutorialFiles = modifiedFiles(isInTutorialFolder & isLivescript);
25+
26+
if ~isempty(tutorialFiles)
27+
[~, fileNames] = fileparts(tutorialFiles);
28+
fileNames = string(fileNames) + ".mlx";
29+
nwbtest('Name', 'tests.unit.Tutorial*', 'ParameterName', fileNames')
30+
matnwb_exportTutorials("FilePaths", tutorialFiles)
31+
end
32+
end

0 commit comments

Comments
 (0)