GaloisInc · bboston7 · Sep 15, 2023 · Jul 12, 2023 · Jul 12, 2023 · Jul 12, 2023
diff --git a/.github/ci.sh b/.github/ci.sh
@@ -62,7 +62,10 @@ build() {
   else
     pkgs=(saw crux-mir-comp saw-remote-api)
   fi
-  tee -a cabal.project.local > /dev/null < cabal.project.ci
+  cat cabal.project.ci >> cabal.project.local
+  if [[ "$ENABLE_HPC" == "true" ]]; then
+    cat cabal.project.ci-hpc >> cabal.project.local
+  fi
   if ! retry cabal v2-build "$@" "${pkgs[@]}"; then
     if [[ "$RUNNER_OS" == "macOS" ]]; then
       echo "Working around a dylib issue on macos by removing the cache and trying again"
@@ -74,6 +77,20 @@ build() {
   fi
 }
 
+# Gather and tar up all HPC coverage files
+collect_hpc_files() {
+  local MIX_FILES=$(find dist-newstyle -name "*.mix")
+  local GENERATED_HS_FILES=$(find dist-newstyle/build -name "*.hs")
+  tar cvf hpc.tar.gz ${MIX_FILES} ${GENERATED_HS_FILES}
+}
+
+# Download HTML coverage reports and generate an index file linking to them
+collect_all_html() {
+  local HTML_DIR=all-html
+  mkdir -p ${HTML_DIR}
+  (cd ${HTML_DIR} && gh run download -p "coverage-html-*" && python3 ../.github/generate_index.py)
+}
+
 install_system_deps() {
   (cd $BIN && curl -o bins.zip -sL "https://github.com/GaloisInc/what4-solvers/releases/download/$SOLVER_PKG_VERSION/$BUILD_TARGET_OS-bin.zip" && unzip -o bins.zip && rm bins.zip)
   chmod +x $BIN/*

diff --git a/.github/generate_index.py b/.github/generate_index.py
@@ -0,0 +1,30 @@
+#!/usr/bin/env python3
+
+# This script generates an HTML index file for all coverage reports
+
+import glob
+
+HEADER = """
+<!DOCTYPE html>
+<head>
+  <title>SAWScript Test Coverage Results</title>
+</head>
+<body>
+  <h1>SAWScript Test Coverage Results</h1>
+  <p>SAWScript coverage results by pull request number:</p>
+  <ul>
+"""
+
+FOOTER = """
+  </ul>
+</body>
+"""
+
+if __name__ == "__main__":
+  with open("index.html", "w") as f:
+    f.write(HEADER)
+    for dir in sorted(glob.glob("coverage-html-*")):
+      pr_num = dir[14:]
+      link_dest = f"{dir}/hpc_index.html"
+      f.write(f"    <li><a href={link_dest}>{pr_num}</a></li>")
+    f.write(FOOTER)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -71,6 +71,7 @@ jobs:
         cabal: ["3.10.1.0"]
         ghc: ["8.10.7", "9.2.7", "9.4.4"]
         run-tests: [true]
+        hpc: [false]
         include:
           # We include one job from an older Ubuntu LTS release to increase our
           # coverage of possible Linux configurations. Since we already run the
@@ -79,6 +80,13 @@ jobs:
             ghc: "8.10.7"
             cabal: "3.10.1.0"
             run-tests: false
+            hpc: false
+          # Include one job with HPC enabled
+          - os: ubuntu-22.04
+            ghc: "9.4.4"
+            cabal: "3.10.1.0"
+            run-tests: true
+            hpc: true
     outputs:
       cabal-test-suites-json: ${{ steps.cabal-test-suites.outputs.targets-json }}
     steps:
@@ -134,6 +142,9 @@ jobs:
 
       - shell: bash
         run: .github/ci.sh build
+        env:
+          ENABLE_HPC: ${{ matrix.hpc }}
+
 
       - shell: bash
         env:
@@ -182,28 +193,44 @@ jobs:
           SIGNING_KEY: ${{ secrets.SIGNING_KEY }}
         run: .github/ci.sh sign $NAME-with-solvers.tar.gz
 
-      - if: matrix.ghc == '8.10.7'
+      - if: matrix.ghc == '8.10.7' && matrix.hpc == false
         uses: actions/upload-artifact@v2
         with:
           name: ${{ steps.config.outputs.name }} (GHC ${{ matrix.ghc }})
           path: "${{ steps.config.outputs.name }}.tar.gz*"
           if-no-files-found: error
           retention-days: ${{ needs.config.outputs.retention-days }}
 
-      - if: matrix.ghc == '8.10.7'
+      - if: matrix.ghc == '8.10.7' && matrix.hpc == false
         uses: actions/upload-artifact@v2
         with:
           name: ${{ steps.config.outputs.name }}-with-solvers (GHC ${{ matrix.ghc }})
           path: "${{ steps.config.outputs.name }}-with-solvers.tar.gz*"
           if-no-files-found: error
           retention-days: ${{ needs.config.outputs.retention-days }}
 
-      - if: matrix.ghc == '8.10.7' && matrix.run-tests
+      - if: matrix.ghc == '8.10.7' && matrix.run-tests && matrix.hpc == false
         uses: actions/upload-artifact@v2
         with:
           path: dist/bin
           name: ${{ runner.os }}-bins
 
+      - if: matrix.run-tests && matrix.hpc == true
+        uses: actions/upload-artifact@v2
+        with:
+          path: dist/bin
+          name: ${{ runner.os }}-with-hpc-bins
+
+      - if: matrix.run-tests && matrix.hpc == true
+        shell: bash
+        run: .github/ci.sh collect_hpc_files
+
+      - if: matrix.run-tests && matrix.hpc == true
+        uses: actions/upload-artifact@v2
+        with:
+          path: hpc.tar.gz
+          name: ${{ runner.os }}-hpc.tar.gz
+
       - uses: actions/cache/save@v3
         name: Save cabal store cache
         if: always()
@@ -360,6 +387,9 @@ jobs:
 
   cabal-test:
     runs-on: ${{ matrix.os }}
+    permissions:
+      pages: write
+      id-token: write
     needs: [build]
     strategy:
       fail-fast: false
@@ -382,6 +412,9 @@ jobs:
           - suite: integration_tests
             os: windows-2019
             continue-on-error: false
+          - suite: integration_tests
+            os: ubuntu-22.04
+            continue-on-error: false
     steps:
       - uses: actions/checkout@v2
         with:
@@ -428,6 +461,84 @@ jobs:
           export PATH="$PWD/bin:$PWD/dist/bin:$PATH"
           dist-tests/${{ matrix.suite }}
 
+  coverage:
+    name: "Run integration tests with coverage reporting"
+    needs: build
+    runs-on: ${{ matrix.os }}
+    if: github.event_name == 'pull_request'
+    strategy:
+      matrix:
+        os: [ubuntu-22.04]
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: true
+
+      - shell: bash
+        run: .github/ci.sh install_system_deps
+        env:
+          BUILD_TARGET_OS: ${{ matrix.os }}
+
+      - uses: actions/download-artifact@v2
+        with:
+          name: "${{ runner.os }}-with-hpc-bins"
+          path: dist/bin
+
+      - uses: actions/download-artifact@v2
+        with:
+          name: "${{ runner.os }}-hpc.tar.gz"
+          path: dist/bin
+
+      - uses: actions/download-artifact@v2
+        with:
+          name: dist-tests-${{ matrix.os }}
+          path: dist-tests
+
+      - name: Update permissions
+        shell: bash
+        run: |
+          chmod +x dist/bin/*
+          chmod +x bin/*
+          chmod +x dist-tests/*
+
+      - uses: actions/setup-java@v1
+        with:
+          java-version: "8"
+          java-package: jdk
+          architecture: x64
+
+      - name: Run integration tests
+        shell: bash
+        run: |
+          export PATH="$PWD/bin:$PWD/dist/bin:$PATH"
+          dist-tests/integration_tests
+
+      - name: Compute coverage
+        shell: bash
+        run: |
+          tar xvf hpc.tar.gz
+          ./compute-coverage.sh
+
+      - uses: actions/upload-artifact@v2
+        with:
+          path: hpc-html
+          name: coverage-html-${{ github.event.number }}
+
+      - name: Gather HPC coverage files
+        shell: bash
+        run: .github/ci.sh collect_all_html
+        env:
+          GH_TOKEN: ${{ github.token }}
+
+      - name: Upload pages artifact
+        uses: actions/upload-pages-artifact@v1
+        with:
+          path: all-html
+
+      - name: Deploy to github pages
+        id: deployment
+        uses: actions/deploy-pages@v2
+
   build-push-image:
     runs-on: ubuntu-22.04
     needs: [config]

diff --git a/cabal.project.ci-hpc b/cabal.project.ci-hpc
@@ -0,0 +1,3 @@
+-- concatenated with cabal.project when run in the CI HPC build
+package saw-script
+  coverage: true
diff --git a/compute-coverage.sh b/compute-coverage.sh
@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+set -Eeuxo pipefail
+
+# This script generates an HTML coverage report for any tests run within the
+# saw-script repo. It uses HPC, which is a tool in the standard GHC
+# distribution. Follow these steps to use this script:
+# 1. Build with coverage enabled. One way to do this is to add "coverage: true"
+#    to the saw-script package in cabal.project.
+# 2. Run whatever tests you want. It is important that you use the saw binary
+#    built in step (1), and that your current working directory be somewhere at
+#    or underneath this top level-directory.
+# 3. Run this script in the top-level directory (where this script is found).
+# 4. You'll find the HPC HTML report in the "hpc-html" directory beneath the
+#    directory containing this script.
+
+# Combine .tix files
+SUM_TIX="all.tix"
+hpc sum --output=$SUM_TIX --union --exclude=Main --exclude=GitRev $(find . -name "*.tix")
+
+# Generate report
+HPC_ROOT=$(find dist-newstyle -name "hpc")
+HPC_ARGS=""
+for dir in ${HPC_ROOT}/vanilla/mix/*; do
+  HPC_ARGS="${HPC_ARGS} --hpcdir=${dir}"
+done
+hpc markup --destdir=hpc-html ${HPC_ARGS} ${SUM_TIX}
diff --git a/coverage.sh b/coverage.sh