Add developer tools for livescripts (#626)

* Add functions for exporting tutorials

* Create .gitattributes

* Rerun and save all livescripts

No content changes

* Add copy of all livescripts as m-code files

* Add section header for developer's section

* Update exportTutorials.m

Added more options to function

* Update intro.mlx

* Update export tutorials

* prefix functions in tools folder with "matnwb_" to avoid potential name conflicts

For now, use prefix instead of namespaces to avoid unnecessary folders

* Create matnwb_listModifiedFiles.m

* Update functions for exporting tutorials

* Create matnwb_checkTutorials.m

Function to run tests for and exports of modified tutorials.

* Add facilities for installing and running a git pre-commit hook

* move misc.generateDocs to tools and rename to matnwb_generateDocs

+ Anchor to matnwb root directory, not current directory

* Update matnwb_generateDocs.m

Update url for m2html source

* Add function to install m2html dependency

* Update and rename setup to matnwb_setup

* Ignore some livescripts during export

* Update livescript m-files variants

* Restore livescripts to previous state

* Fix livescript typos

* Add optional mode and function to clean text for matnwb_listTutorialFiles

* Update matnwb_checkTutorials - Rename variables to make logic clearer

* Update pre-commit hooks.

Undo running matlab code as this is very slow
Provide shell scripts for mac and linux

* Update matnwb_installGitHooks.m

* Update pre-commit hooks to also check if html for main nwb api functions are up to date

* Fix docstrings plus add useful warning

* Update tutorials/private/README.md

---------

Co-authored-by: Ben Dichter <[email protected]>

tools/documentation/matnwb_exportModifiedTutorials.m

@@ -0,0 +1,33 @@
+function matnwb_exportModifiedTutorials()
+% matnwb_exportModifiedTutorials - Export modified livescript tutorials to html
+%
+% See also matnwb_exportTutorials
+
+    if exist("isMATLABReleaseOlderThan", "file") == 2
+        hasGitRepo = ~isMATLABReleaseOlderThan("R2023b");
+    else
+        hasGitRepo = false;
+    end
+
+    if hasGitRepo
+        repo = gitrepo(misc.getMatnwbDir);
+        modifiedFiles = repo.ModifiedFiles;
+    else
+        modifiedFiles = matnwb_listModifiedFiles();
+    end
+
+    tutorialFolder = fullfile(misc.getMatnwbDir, 'tutorials');
+    isInTutorialFolder = startsWith(modifiedFiles, tutorialFolder);
+    isLivescript = endsWith(modifiedFiles, ".mlx");
+
+    tutorialFiles = modifiedFiles(isInTutorialFolder & isLivescript);
+
+    filesToIgnore = ["basicUsage", "read_demo", "remote_read"];
+    isIgnored = endsWith(tutorialFiles, filesToIgnore + ".mlx");
+    if any(isIgnored)
+        warning('Skipping export for the following files (see matnwb_exportTutorials):\n%s', ...
+            strjoin("  - " + filesToIgnore(isIgnored) + ".mlx", newline))
+    end
+
+    matnwb_exportTutorials("FilePaths", tutorialFiles, "IgnoreFiles", filesToIgnore)
+end

tools/documentation/matnwb_exportTutorials.m

@@ -0,0 +1,93 @@
+function matnwb_exportTutorials(options)
+% matnwb_exportTutorials - Export mlx tutorial files to the specified output format
+%
+% Note: This function will ignore the following live scripts:
+%  - basicUsage.mlx : depends on output from convertTrials.m
+%  - read_demo.mlx : depends on external data, potentially slow
+%  - remote_read.mlx : Uses nwbRead on s3 url, potentially very slow]
+%
+%   To export all livescripts (assuming you have made sure the above-mentioned 
+%   files will run) call the function with IgnoreFiles set to empty, i.e:
+%       matnwb_exportTutorials(..., "IgnoreFiles", string.empty)
+
+    arguments
+        options.ExportFormat (1,:) string {mustStartWithDot} = [".m", ".html"]
+        options.Expression (1,1) string = "*" % Filter by expression
+        options.FileNames (1,:) string = string.empty % Filter by file names
+        options.FilePaths (1,:) string = string.empty % Export specified files
+        options.IgnoreFiles (1,:) string = ["basicUsage", "read_demo", "remote_read"];
+        options.RunLivescript (1,1) logical = true
+    end
+
+    [exportFormat, targetFolderNames] = deal(options.ExportFormat);
+
+    targetFolderNames = extractAfter(targetFolderNames, ".");
+    targetFolderNames(strcmp(targetFolderNames, "m")) = fullfile("private", "mcode");
+
+    nwbTutorialDir = fullfile(misc.getMatnwbDir, "tutorials");
+    targetFolderPaths = fullfile(nwbTutorialDir, targetFolderNames);
+
+    for folderPath = targetFolderPaths
+        if ~isfolder(folderPath); mkdir(folderPath); end
+    end
+
+    if isempty(options.FilePaths)
+        if endsWith(options.Expression, "*")
+            expression = options.Expression + ".mlx";
+        else
+            expression = options.Expression + "*.mlx";
+        end
+
+        L = dir(fullfile(nwbTutorialDir, expression));
+        filePaths = string( fullfile({L.folder}, {L.name}) );
+    else
+        filePaths = options.FilePaths;
+    end
+
+    [~, fileNames] = fileparts(filePaths);
+    if ~isempty(options.FileNames)
+        [fileNames, iA] = intersect(fileNames, options.FileNames, 'stable');
+        filePaths = filePaths(iA);
+    end
+
+    if ~isempty(options.IgnoreFiles)
+        [~, fileNames] = fileparts(filePaths);
+        [fileNames, iA] = setdiff(fileNames, options.IgnoreFiles, 'stable');
+        filePaths = filePaths(iA);
+    end
+
+    % Go to a temporary directory, so that tutorials are exported in a
+    % temporary folder which is cleaned up afterwards
+    currentDir = pwd();
+    cleanupWorkdir = onCleanup(@(fp) cd(currentDir));
+
+    tempDir = fullfile(tempdir, 'nwbTutorials');
+    if ~isfolder(tempDir); mkdir(tempDir); end
+    disp('Changing into temporary directory:')
+    cd(tempDir)
+
+    cleanupDeleteTempFiles = onCleanup(@(fp) rmdir(tempDir, 's'));
+    disp(tempDir)
+
+    for i = 1:numel(filePaths)
+        sourcePath = char( fullfile(filePaths(i)) );
+        if options.RunLivescript
+            fprintf('Running livescript "%s"\n', fileNames(i))
+
+            matlab.internal.liveeditor.executeAndSave(sourcePath);
+        end
+
+        for j = 1:numel(exportFormat)
+            targetPath = fullfile(targetFolderPaths(j), fileNames(i) + exportFormat(j));
+            fprintf('Exporting livescript "%s" to "%s"\n', fileNames(i), exportFormat(j))
+            export(sourcePath, strrep(targetPath, '.mlx', exportFormat(j)));
+        end
+    end
+end
+
+function mustStartWithDot(value)
+    for i = 1:numel(value)
+        assert(startsWith(value(i), '.'), ...
+            'Value must be a file extension starting with a period, e.g ".html"')
+    end
+end

+misc/generateDocs.m → tools/documentation/matnwb_generateDocs.m

@@ -1,25 +1,35 @@
-function generateDocs()
-% GENERATEDOCS generates docs for MatNWB user API
-%   GENERATEDOCS() generate documentation for MATLAB files in the current working directory.
+function matnwb_generateDocs()
+% MATNWB_GENERATEDOCS generates html docs for MatNWB API functions
 %
-%   Requires <a href="matlab:
-%   web('https://www.artefact.tk/software/matlab/m2html/')">m2html</a> in your path.
-rootFiles = dir('.');
+%   matnwb_generateDocs() generates html documentation for MATLAB files in the 
+%   current matnwb root directory.
+%
+%   The following files are included:
+%       - generateCore.m
+%       - generateExtension.m
+%       - nwbRead.m
+%       - nwbExport.m
+%
+%   Requires <a href="matlab:web('https://github.com/gllmflndn/m2html')">m2html</a> in your path.
+
+rootDir = misc.getMatnwbDir();
+rootFiles = dir(rootDir);
 rootFiles = {rootFiles.name};
 rootWhitelist = {'generateCore.m', 'generateExtension.m', 'nwbRead.m', 'nwbExport.m'};
 isWhitelisted = ismember(rootFiles, rootWhitelist);
 rootFiles(~isWhitelisted) = [];
 
-m2html('mfiles', rootFiles, 'htmldir', 'doc');
+docDir = fullfile(rootDir, 'doc');
+m2html('mfiles', rootFiles, 'htmldir', docDir);
 
 % correct html files in root directory as the stylesheets will be broken
 fprintf('Correcting files in root directory...\n');
-rootFiles = dir('doc');
+rootFiles = dir(docDir);
 rootFiles = {rootFiles.name};
 htmlMatches = regexp(rootFiles, '\.html$', 'once');
 isHtmlFile = ~cellfun('isempty', htmlMatches);
 rootFiles(~isHtmlFile) = [];
-rootFiles = fullfile('doc', rootFiles);
+rootFiles = fullfile(docDir, rootFiles);
 
 for iDoc=1:length(rootFiles)
     fileName = rootFiles{iDoc};
@@ -29,7 +39,7 @@ function generateDocs()
 
 % correct index.html so the header indicates MatNWB
 fprintf('Correcting index.html Header...\n');
-indexPath = fullfile('doc', 'index.html');
+indexPath = fullfile(docDir, 'index.html');
 fileReplace(indexPath, 'Index for \.', 'Index for MatNWB');
 
 % remove directories listing in index.html

tools/githooks/pre-commit-linux

@@ -0,0 +1,70 @@
+#!/bin/bash
+# NB: Not tested
+
+# Relative paths
+TUTORIAL_FOLDER="tutorials"
+PRIVATE_FOLDER="$TUTORIAL_FOLDER/private"
+
+# Define file mappings (script files and their corresponding documentation)
+SCRIPT_FILES=("generateCore.m" "generateExtension.m" "nwbRead.m" "nwbExport.m")
+DOC_FOLDER="doc"
+
+# Get modified files (staged + unstaged)
+MODIFIED_FILES=$(git diff --cached --name-only)
+
+# Check for .mlx files in the tutorials folder
+TUTORIAL_FILES=$(echo "$MODIFIED_FILES" | grep "^$TUTORIAL_FOLDER" | grep "\.mlx$")
+
+# If there are tutorial files, validate them
+if [[ -n "$TUTORIAL_FILES" ]]; then
+    echo "Checking tutorial files..."
+
+    for TUTORIAL_FILE in $TUTORIAL_FILES; do
+        # Get the base name without extension
+        BASENAME=$(basename "$TUTORIAL_FILE" .mlx)
+
+        # Find corresponding .html and .m files
+        HTML_FILE=$(find "$TUTORIAL_FOLDER" -name "$BASENAME.html" -print -quit)
+        MC_FILE=$(find "$PRIVATE_FOLDER" -name "$BASENAME.m" -print -quit)
+
+        # Get modification dates (default to 0 if file doesn't exist)
+        TUTORIAL_FILE_DATE=$(stat -c "%Y" "$TUTORIAL_FILE")
+        HTML_FILE_DATE=$(stat -c "%Y" "$HTML_FILE" 2>/dev/null || echo 0)
+        MC_FILE_DATE=$(stat -c "%Y" "$MC_FILE" 2>/dev/null || echo 0)
+
+        # Check if .html or .m file is outdated
+        if [[ "$TUTORIAL_FILE_DATE" -gt "$HTML_FILE_DATE" || "$TUTORIAL_FILE_DATE" -gt "$MC_FILE_DATE" ]]; then
+            echo "Error: Please re-export live script \"$BASENAME.mlx\"." >&2
+            exit 1
+        fi
+    done
+fi
+
+# Flag to track if any files are outdated
+OUTDATED_FOUND=0
+
+# Loop through each script file
+for SCRIPT_FILE in "${SCRIPT_FILES[@]}"; do
+    # Check if the script file has been modified
+    if echo "$MODIFIED_FILES" | grep -q "^$SCRIPT_FILE$"; then
+        # Get the corresponding HTML file in the doc folder
+        HTML_FILE="$DOC_FOLDER/${SCRIPT_FILE%.m}.html"
+
+        # Get modification dates (default to 0 if file doesn't exist)
+        SCRIPT_FILE_DATE=$(stat -c "%Y" "$SCRIPT_FILE")
+        HTML_FILE_DATE=$(stat -c "%Y" "$HTML_FILE" 2>/dev/null || echo 0)
+
+        # Check if the script is newer than the HTML file
+        if [[ "$SCRIPT_FILE_DATE" -gt "$HTML_FILE_DATE" ]]; then
+            echo "Error: Please re-export documentation for \"$SCRIPT_FILE\"." >&2
+            OUTDATED_FOUND=1
+        fi
+    fi
+done
+
+# Exit with error if any files are outdated
+if [[ $OUTDATED_FOUND -eq 1 ]]; then
+    exit 1
+fi
+
+exit 0

tools/githooks/pre-commit-mac

@@ -0,0 +1,68 @@
+#!/bin/bash
+
+# Relative paths
+TUTORIAL_FOLDER="tutorials"
+PRIVATE_FOLDER="$TUTORIAL_FOLDER/private"
+
+# Define file mappings (script files and their corresponding documentation)
+SCRIPT_FILES=("generateCore.m" "generateExtension.m" "nwbRead.m" "nwbExport.m")
+DOC_FOLDER="doc"
+
+# Get modified files (staged)
+MODIFIED_FILES=$(git diff --cached --name-only)
+
+# Check for .mlx files in the tutorials folder
+TUTORIAL_FILES=$(echo "$MODIFIED_FILES" | grep "^$TUTORIAL_FOLDER" | grep "\.mlx$")
+
+# If there are tutorial files, check that they have been exported
+if [[ -n "$TUTORIAL_FILES" ]]; then
+
+    for TUTORIAL_FILE in $TUTORIAL_FILES; do
+        # Get the base name without extension
+        BASENAME=$(basename "$TUTORIAL_FILE" .mlx)
+
+        # Find corresponding .html and .m files
+        HTML_FILE=$(find "$TUTORIAL_FOLDER" -name "$BASENAME.html" -print -quit)
+        MC_FILE=$(find "$PRIVATE_FOLDER" -name "$BASENAME.m" -print -quit)
+
+        # Get modification dates (default to 0 if file doesn't exist)
+        TUTORIAL_FILE_DATE=$(stat -f "%m" "$TUTORIAL_FILE")
+        HTML_FILE_DATE=$(stat -f "%m" "$HTML_FILE" 2>/dev/null || echo 0)
+        MC_FILE_DATE=$(stat -f "%m" "$MC_FILE" 2>/dev/null || echo 0)
+
+        # Check if .html or .m file is outdated
+        if [[ "$TUTORIAL_FILE_DATE" -gt "$HTML_FILE_DATE" || "$TUTORIAL_FILE_DATE" -gt "$MC_FILE_DATE" ]]; then
+            echo "Error: Please re-export html/m-files for live script \"$BASENAME.mlx\"." >&2
+            exit 1
+        fi
+    done
+fi
+
+# Flag to track if any files are outdated
+OUTDATED_FOUND=0
+
+# Loop through each script file
+for SCRIPT_FILE in "${SCRIPT_FILES[@]}"; do
+    # Check if the script file has been modified
+    if echo "$MODIFIED_FILES" | grep -q "^$SCRIPT_FILE$"; then
+        # Get the corresponding HTML file in the doc folder
+        HTML_FILE="$DOC_FOLDER/${SCRIPT_FILE%.m}.html"
+
+        # Get modification dates (default to 0 if file doesn't exist)
+        SCRIPT_FILE_DATE=$(stat -f "%m" "$SCRIPT_FILE")
+        HTML_FILE_DATE=$(stat -f "%m" "$HTML_FILE" 2>/dev/null || echo 0)
+
+        # Check if the script is newer than the HTML file
+        if [[ "$SCRIPT_FILE_DATE" -gt "$HTML_FILE_DATE" ]]; then
+            echo "Error: Please re-export documentation for \"$SCRIPT_FILE\"." >&2
+            OUTDATED_FOUND=1
+        fi
+    fi
+done
+
+# Exit with error if any files are outdated
+if [[ $OUTDATED_FOUND -eq 1 ]]; then
+    exit 1
+fi
+
+exit 0

tools/maintenance/matnwb_checkTutorials.m

@@ -0,0 +1,32 @@
+function matnwb_checkTutorials()
+% matnwb_checkTutorials - Checks for modified MATLAB Live Script tutorial files 
+% in the repository and executes tests and html exports if found.
+%
+% This function determines whether any tutorial files in the `tutorials`
+% directory have been modified in the matnwb repository. If such files exist, 
+% the function performs the following actions:
+% 1. Runs unit tests matching the tutorial names.
+% 2. Exports the modified tutorial files using the `matnwb_exportTutorials` 
+% function.
+%
+% Usage:
+%   matnwb_checkTutorials()
+%
+%   See also matnwb_listModifiedFiles, matnwb_exportTutorials
+
+    tutorialFolder = fullfile(misc.getMatnwbDir, 'tutorials');
+
+    modifiedFiles = matnwb_listModifiedFiles("all");
+
+    isInTutorialFolder = startsWith(modifiedFiles, tutorialFolder);
+    isLivescript = endsWith(modifiedFiles, ".mlx");
+
+    tutorialFiles = modifiedFiles(isInTutorialFolder & isLivescript);
+
+    if ~isempty(tutorialFiles)
+        [~, fileNames] = fileparts(tutorialFiles);
+        fileNames = string(fileNames) + ".mlx";
+        nwbtest('Name', 'tests.unit.Tutorial*', 'ParameterName', fileNames')    
+        matnwb_exportTutorials("FilePaths", tutorialFiles)
+    end
+end