Reduce number of CI jobs run on PRs

.ci/scripts/calculate_trial_jobs.py → .ci/scripts/calculate_jobs.py

@@ -20,7 +20,11 @@
 
 IS_PR = os.environ["GITHUB_REF"].startswith("refs/pull/")
 
-sqlite_tests = [
+# First calculate the various trial jobs.
+#
+# For each type of test we only run on Py3.7 on PRs
+
+trial_sqlite_tests = [
     {
         "python-version": "3.7",
         "database": "sqlite",
@@ -29,7 +33,7 @@
 ]
 
 if not IS_PR:
-    sqlite_tests.extend(
+    trial_sqlite_tests.extend(
         {
             "python-version": version,
             "database": "sqlite",
@@ -39,7 +43,7 @@
     )
 
 
-postgres_tests = [
+trial_postgres_tests = [
     {
         "python-version": "3.7",
         "database": "postgres",
@@ -49,7 +53,7 @@
 ]
 
 if not IS_PR:
-    postgres_tests.append(
+    trial_postgres_tests.append(
         {
             "python-version": "3.10",
             "database": "postgres",
@@ -58,17 +62,67 @@
         }
     )
 
-no_extra_tests = [
+trial_no_extra_tests = [
     {
         "python-version": "3.7",
         "database": "sqlite",
         "extras": "",
     }
 ]
 
-print("::group::Calculated jobs")
-print(json.dumps(sqlite_tests + postgres_tests + no_extra_tests, indent=4))
+print("::group::Calculated trial jobs")
+print(
+    json.dumps(
+        trial_sqlite_tests + trial_postgres_tests + trial_no_extra_tests, indent=4
+    )
+)
+print("::endgroup::")
+
+test_matrix = json.dumps(
+    trial_sqlite_tests + trial_postgres_tests + trial_no_extra_tests
+)
+print(f"::set-output name=trial_test_matrix::{test_matrix}")
+
+
+# First calculate the various sytest jobs.
+#
+# For each type of test we only run on focal on PRs
+
+
+sytest_tests = [
+    {
+        "sytest-tag": "focal",
+    },
+    {
+        "sytest-tag": "focal",
+        "postgres": "postgres",
+    },
+    {
+        "sytest-tag": "focal",
+        "postgres": "mulit-postgres",
+        "workers": "workers",
+    },
+]
+
+if not IS_PR:
+    sytest_tests.extend(
+        [
+            {
+                "sytest-tag": "testing",
+                "postgres": "postgres",
+            },
+            {
+                "sytest-tag": "buster",
+                "postgres": "mulit-postgres",
+                "workers": "workers",
+            },
+        ]
+    )
+
+
+print("::group::Calculated sytest jobs")
+print(json.dumps(sytest_tests, indent=4))
 print("::endgroup::")
 
-test_matrix = json.dumps(sqlite_tests + postgres_tests + no_extra_tests)
-print(f"::set-output name=test_matrix::{test_matrix}")
+test_matrix = json.dumps(sytest_tests)
+print(f"::set-output name=sytest_test_matrix::{test_matrix}")

.github/workflows/tests.yml

@@ -73,7 +73,7 @@ jobs:
     steps:
       - run: "true"
 
-  trial-calculate-jobs:
+  calculate-test-jobs:
     if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail
     needs: linting-done
     runs-on: ubuntu-latest
@@ -83,15 +83,16 @@ jobs:
       - id: get-matrix
         run: .ci/scripts/calculate_trial_jobs.py
     outputs:
-      test_matrix: ${{ steps.get-matrix.outputs.test_matrix }}
+      trial_test_matrix: ${{ steps.get-matrix.outputs.trial_test_matrix }}
+      sytest_test_matrix: ${{ steps.get-matrix.outputs.sytest_test_matrix }}
 
   trial:
     if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail
-    needs: trial-calculate-jobs
+    needs: calculate-test-jobs
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        job:  ${{ fromJson(needs.trial-calculate-jobs.outputs.test_matrix) }}
+        job:  ${{ fromJson(needs.calculate-test-jobs.outputs.trial_test_matrix) }}
 
     steps:
       - uses: actions/checkout@v2
@@ -192,45 +193,26 @@ jobs:
 
   sytest:
     if: ${{ !failure() && !cancelled() }}
-    needs: linting-done
+    needs: calculate-test-jobs
     runs-on: ubuntu-latest
     container:
-      image: matrixdotorg/sytest-synapse:${{ matrix.sytest-tag }}
+      image: matrixdotorg/sytest-synapse:${{ matrix.job.sytest-tag }}
       volumes:
         - ${{ github.workspace }}:/src
       env:
         SYTEST_BRANCH: ${{ github.head_ref }}
-        POSTGRES: ${{ matrix.postgres && 1}}
-        MULTI_POSTGRES: ${{ (matrix.postgres == 'multi-postgres') && 1}}
-        WORKERS: ${{ matrix.workers && 1 }}
-        REDIS: ${{ matrix.redis && 1 }}
-        BLACKLIST: ${{ matrix.workers && 'synapse-blacklist-with-workers' }}
+        POSTGRES: ${{ matrix.job.postgres && 1}}
+        MULTI_POSTGRES: ${{ (matrix.job.postgres == 'multi-postgres') && 1}}
+        WORKERS: ${{ matrix.job.workers && 1 }}
+        REDIS: 1
+        BLACKLIST: ${{ matrix.job.workers && 'synapse-blacklist-with-workers' }}
         TOP: ${{ github.workspace }}
 
     strategy:
       fail-fast: false
       matrix:
         include:
-          - sytest-tag: focal
-
-          - sytest-tag: focal
-            postgres: postgres
-
-          - sytest-tag: testing
-            postgres: postgres
-
-          - sytest-tag: focal
-            postgres: multi-postgres
-            workers: workers
-
-          - sytest-tag: buster
-            postgres: multi-postgres
-            workers: workers
-
-          - sytest-tag: buster
-            postgres: postgres
-            workers: workers
-            redis: redis
+          job: ${{ fromJson(needs.calculate-test-jobs.outputs.sytest_test_matrix) }}
 
     steps:
       - uses: actions/checkout@v2
@@ -246,7 +228,7 @@ jobs:
         uses: actions/upload-artifact@v2
         if: ${{ always() }}
         with:
-          name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.*, ', ') }})
+          name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.job.*, ', ') }})
           path: |
             /logs/results.tap
             /logs/**/*.log*