fix: test commenting of new baselines

.github/workflows/regression_tests.yml

@@ -2,9 +2,9 @@
 name: Regression Tests
 
 on:
-#   pull_request:
-#     branches:
-#       - main
+  # pull_request:
+  #   branches:
+  #     - main
 
 jobs:
   regression_tests:

.github/workflows/update_regression_baseline.yml

@@ -17,7 +17,7 @@ jobs:
       contents: write
       pull-requests: write
     env:
-      username: ${{ github.event.pull_request.user.login }}
+      username: ${{ github.event.pull_request.user.login }} # ${{ github.event.comment.user.login }}
 
     steps:
     # - name: Get PR branch
@@ -46,7 +46,25 @@ jobs:
       run: |
         git config --global user.name '$username'
         git config --global user.email '[email protected]'
+        mv tests/regression_test_baselines.json tests/regression_test_baselines.json.bak
         NEW_BASELINE=1 pytest -m regression
+
+    - name: Add comment to PR
+      uses: actions/github-script@v7
+      if: always()
+      with:
+        github-token: ${{ secrets.GITHUB_TOKEN }}
+        script: |
+          const fs = require('fs');
+          const TestReport = fs.readFileSync('regression_test_report.txt', 'utf8');
+          
+          await github.rest.issues.createComment({
+            issue_number: context.issue.number,
+            owner: context.repo.owner,
+            repo: context.repo.repo,
+            body: `## New Baselines \n\`\`\`\n${TestReport}\n\`\`\``
+          });
+        
         
     - name: Commit and push
       if: github.event.pull_request.base.ref == 'main'

tests/test_regression.py

@@ -28,6 +28,13 @@
 #   t3 = time.time()
 #   return {"sth": t2-t1, sth_else: t3-t2}
 
+def load_json(fpath):
+    dct = {}
+    if os.path.exists(fpath):
+        with open(fpath, "r") as f:
+            dct = json.load(f)
+    return dct
+
 
 pytestmark = pytest.mark.regression  # mark all tests as regression tests in this file
 NEW_BASELINE = os.environ["NEW_BASELINE"] if "NEW_BASELINE" in os.environ else 0
@@ -37,10 +44,8 @@
 
 tolerance = 0.2
 
-baselines = {}
-if os.path.exists(fpath_baselines):
-    with open(fpath_baselines) as f:
-        baselines = json.load(f)
+baselines = load_json(fpath_baselines)
+
 
 
 def generate_regression_report(base_results, new_results):
@@ -99,12 +104,9 @@ def append_to_json(fpath, test_name, input_kwargs, runtimes):
     data = {generate_unique_key(header): {**header, "runtimes": runtimes}}
 
     # Save data to a JSON file
-    if os.path.exists(fpath):
-        with open(fpath, "r") as f:
-            result_data = json.load(f)
-            result_data.update(data)
-    else:
-        result_data = data
+    result_data = load_json(fpath)
+    result_data.update(data)
+
     with open(fpath, "w") as f:
         json.dump(result_data, f, indent=2)
 
@@ -120,6 +122,8 @@ def test_wrapper(**kwargs):
             header = {"test_name": func.__name__, "input_kwargs": kwargs}
             key = generate_unique_key(header)
 
+            new_data, old_data = None, None
+
             runs = []
             num_iters = self.baseline_iters if NEW_BASELINE else self.test_iters
             for _ in range(num_iters):
@@ -143,6 +147,18 @@ def test_wrapper(**kwargs):
                         1 + tolerance
                     ), f"{key} is {diff:.2%} slower than the baseline."
 
+            # save report
+            if NEW_BASELINE:
+                new_data = load_json(fpath_baselines)
+                old_data = load_json(fpath_baselines + ".bak")
+            else:
+                new_data = load_json(fpath_results)
+                old_data = load_json(fpath_baselines)
+            report = generate_regression_report(old_data, new_data)
+
+            with open(dirname + "/regression_test_report.txt", "w") as f:
+                f.write(report)
+
         return test_wrapper
 
 
@@ -186,13 +202,13 @@ def build_net(num_cells, artificial=True, connect=True, connection_prob=0.0):
     (
         # Test a single SWC cell with both solvers.
         pytest.param(1, False, False, 0.0, "jaxley.stone"),
-        pytest.param(1, False, False, 0.0, "jax.sparse"),
-        # Test a network of SWC cells with both solvers.
-        pytest.param(10, False, True, 0.1, "jaxley.stone"),
-        pytest.param(10, False, True, 0.1, "jax.sparse"),
-        # Test a larger network of smaller neurons with both solvers.
-        pytest.param(1000, True, True, 0.001, "jaxley.stone"),
-        pytest.param(1000, True, True, 0.001, "jax.sparse"),
+        # pytest.param(1, False, False, 0.0, "jax.sparse"),
+        # # Test a network of SWC cells with both solvers.
+        # pytest.param(10, False, True, 0.1, "jaxley.stone"),
+        # pytest.param(10, False, True, 0.1, "jax.sparse"),
+        # # Test a larger network of smaller neurons with both solvers.
+        # pytest.param(1000, True, True, 0.001, "jaxley.stone"),
+        # pytest.param(1000, True, True, 0.001, "jax.sparse"),
     ),
 )
 @compare_to_baseline(baseline_iters=3)