diff --git a/.github/ci.sh b/.github/ci.sh index 099380f318..0cd6b5ce16 100755 --- a/.github/ci.sh +++ b/.github/ci.sh @@ -197,7 +197,6 @@ zip_dist() { name="${name:-"saw-$VERSION-$RUNNER_OS-x86_64"}" mv dist "$name" tar -czf "$name".tar.gz "$name" - sign "$name".tar.gz [[ -f "$name".tar.gz.sig ]] && [[ -f "$name".tar.gz ]] } diff --git a/.github/workflows/build.yml b/.github/workflows/ci.yml similarity index 60% rename from .github/workflows/build.yml rename to .github/workflows/ci.yml index 2585992276..fd9f42f647 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/ci.yml @@ -1,82 +1,88 @@ name: SAWScript on: push: + tags: ["?[0-9]+.[0-9]+(.[0-9]+)?"] branches: [master, "release-**"] pull_request: + schedule: + - cron: "0 10 * * *" # 10am UTC -> 2/3am PST workflow_dispatch: - inputs: - releaseArtifacts: - description: 'Set to 1 to upload the release bundles as workflow artifacts' - required: true - default: '0' env: - CACHE_VERSION: 1 - # Solver versions - also update in the following locations: - # ./saw/Dockerfile - # ./saw-remote-api/Dockerfile - # ./s2nTests/docker/saw.dockerfile - Z3_VERSION: "4.8.10" + Z3_VERSION: "4.8.8" CVC4_VERSION: "4.1.8" YICES_VERSION: "2.6.2" jobs: - outputs: - runs-on: ubuntu-18.04 + config: + runs-on: ubuntu-latest outputs: - changed: ${{ steps.outputs.outputs.changed-files }} - saw-version: ${{ steps.outputs.outputs.saw-version }} - name: ${{ steps.outputs.outputs.name }} - release: ${{ steps.env.outputs.release }} + name: ${{ steps.config.outputs.name }} + version: ${{ steps.config.outputs.version }} + event-tag: ${{ steps.config.outputs.tag }} + event-schedule: ${{ steps.config.outputs.schedule }} + publish: ${{ steps.config.outputs.publish }} + release: ${{ steps.config.outputs.release }} + retention-days: ${{ steps.config.outputs.retention-days }} steps: - uses: actions/checkout@v2 with: fetch-depth: 0 - - id: env - if: | - startsWith(github.ref, 'refs/heads/release-') - || (github.event_name == 'pull_request' && startsWith(github.head_ref, 'actions/')) - || (github.event_name == 'workflow_dispatch' && github.event.inputs.releaseArtifacts == '1') - run: .github/ci.sh output release 1 - - id: outputs + + - name: config + id: config + env: + EVENT_TAG: ${{ github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/') }} + EVENT_SCHEDULE: ${{ github.event_name == 'schedule' }} + EVENT_DISPATCH: ${{ github.event_name == 'workflow_dispatch' }} + RELEASE: ${{ startsWith(github.event.ref, 'refs/heads/release-') }} run: | - .github/ci.sh set_files ${{ github.sha }} - .github/ci.sh set_version + set -x .github/ci.sh output name saw-$(.github/ci.sh ver) + .github/ci.sh output version $(.github/ci.sh ver) + .github/ci.sh output tag $EVENT_TAG + .github/ci.sh output schedule $EVENT_SCHEDULE + .github/ci.sh output publish $({ + $EVENT_TAG || + $EVENT_SCHEDULE || + $EVENT_DISPATCH && ${{ github.event.inputs.publish }}; + } && echo true || echo false) + .github/ci.sh output release $RELEASE + .github/ci.sh output retention-days $($RELEASE && echo 90 || echo 5) build: runs-on: ${{ matrix.os }} - needs: [outputs] + needs: [config] strategy: fail-fast: false matrix: os: [ubuntu-18.04] ghc: ["8.6.5", "8.8.4", "8.10.3"] - continue-on-error: [false] include: - os: macos-latest ghc: "8.6.5" - continue-on-error: true - os: macos-latest ghc: "8.8.4" - continue-on-error: true - os: windows-latest ghc: "8.6.5" - continue-on-error: true exclude: - os: windows-latest ghc: "8.8.4" - env: - RELEASE: ${{ needs.outputs.outputs.release }} - continue-on-error: ${{ matrix.continue-on-error }} outputs: cabal-test-suites-json: ${{ steps.cabal-test-suites.outputs.targets-json }} steps: + - uses: actions/checkout@v2 - run: | git submodule update --init git -C deps/abcBridge submodule update --init + - id: config + run: | + NAME="${{ needs.config.outputs.name }}-${{ runner.os }}-x86_64" + .github/ci.sh output name $NAME + echo "NAME=${{ needs.config.outputs.name }}-${{ runner.os }}-x86_64" >> $GITHUB_ENV + - uses: haskell/actions/setup@v1 id: setup-haskell with: @@ -99,7 +105,6 @@ jobs: - shell: bash run: .github/ci.sh build - continue-on-error: false - uses: GaloisInc/.github/actions/cabal-collect-bins@v1 id: cabal-test-suites @@ -109,7 +114,7 @@ jobs: cryptol-saw-core-tc-test dest: dist-tests - - name: "Upload Artifact: bin/abc" + - name: "actions/upload-artifact: bin/abc" if: | matrix.ghc == '8.6.5' && runner.os != 'Windows' @@ -118,7 +123,7 @@ jobs: path: ${{ steps.abc.outputs.path }} name: abc-${{ runner.os }} - - name: "Upload Artifact: dist-tests" + - name: "actions/upload-artifact: dist-tests" uses: actions/upload-artifact@v2 if: "matrix.ghc == '8.6.5'" with: @@ -128,20 +133,36 @@ jobs: - shell: bash run: .github/ci.sh setup_dist_bins - - if: >- - env.RELEASE && matrix.ghc == '8.6.5' - run: | - .github/ci.sh build_cryptol - .github/ci.sh extract_exe "cryptol" "dist/bin" "deps/cryptol/dist-newstyle" + - shell: bash + run: .github/ci.sh bundle_files + + - shell: bash + run: .github/ci.sh zip_dist $NAME + + - if: matrix.ghc == '8.6.5' && needs.config.outputs.release == 'true' + shell: bash + env: + SIGNING_PASSPHRASE: ${{ secrets.SIGNING_PASSPHRASE }} + SIGNING_KEY: ${{ secrets.SIGNING_KEY }} + run: .github/ci.sh sign $NAME.tar.gz - - name: "Upload Artifact: dist/bin" + - if: matrix.ghc == '8.6.5' + name: "actions/upload-artifact: ${{ steps.config.outputs.name }}.tar.gz" + uses: actions/upload-artifact@v2 + with: + name: ${{ steps.config.outputs.name }} (GHC ${{ matrix.ghc }}) + path: "${{ steps.config.outputs.name }}.tar.gz*" + if-no-files-found: error + retention-days: ${{ needs.config.outputs.retention-days }} + + - name: "actions/upload-artifact: dist/bin" if: "matrix.ghc == '8.6.5'" uses: actions/upload-artifact@v2 with: path: dist/bin name: ${{ runner.os }}-bins - - name: "Upload Artifact: dist/bin/saw" + - name: "actions/upload-artifact: dist/bin/saw" if: "matrix.os == 'ubuntu-18.04'" uses: actions/upload-artifact@v2 with: @@ -169,7 +190,7 @@ jobs: - shell: bash run: .github/ci.sh install_system_deps - - name: "Download Artifact: dist/bin" + - name: "actions/download-artifact: dist/bin" uses: actions/download-artifact@v2 with: name: "${{ runner.os }}-bins" @@ -198,21 +219,16 @@ jobs: matrix: suite: ${{ fromJson(needs.build.outputs.cabal-test-suites-json) }} os: [ubuntu-18.04] - continue-on-error: [false] include: - suite: integration_tests os: macos-latest - continue-on-error: true - suite: integration_tests os: windows-latest - continue-on-error: true exclude: - suite: integration_tests os: macos-latest - continue-on-error: false - suite: integration_tests os: windows-latest - continue-on-error: false steps: - uses: actions/checkout@v2 with: @@ -221,7 +237,7 @@ jobs: - shell: bash run: .github/ci.sh install_system_deps - - name: "Download Artifact: dist/bin" + - name: "actions/download-artifact: dist/bin" uses: actions/download-artifact@v2 with: name: "${{ runner.os }}-bins" @@ -231,7 +247,7 @@ jobs: if: "runner.os != 'Windows'" run: chmod +x dist/bin/* - - name: "Download Artifact: bin/abc" + - name: "actions/download-artifact: bin/abc" if: | runner.os != 'Windows' && matrix.suite == 'integration_tests' @@ -246,7 +262,7 @@ jobs: matrix.suite == 'integration_tests' run: chmod +x bin/* - - name: "Download Artifact: dist-tests" + - name: "actions/download-artifact: dist-tests" uses: actions/download-artifact@v2 with: name: dist-tests-${{ matrix.os }} @@ -264,46 +280,96 @@ jobs: architecture: x64 - name: ${{ matrix.suite }} - continue-on-error: ${{ matrix.continue-on-error }} shell: bash run: | export PATH="$PWD/bin:$PWD/dist/bin:$PATH" dist-tests/${{ matrix.suite }} - docker: - runs-on: ubuntu-18.04 - needs: [outputs] - if: contains(needs.outputs.outputs.changed, 'Dockerfile') + build-push-image: + runs-on: ubuntu-latest + needs: [config] + strategy: + fail-fast: false + matrix: + include: + - file: Dockerfile + image: ghcr.io/galoisinc/saw + cache: ghcr.io/galoisinc/cache-saw + - file: saw-remote-api/Dockerfile + image: ghcr.io/galoisinc/saw-remote-api + cache: ghcr.io/galoisinc/cache-saw-remote-api steps: - uses: actions/checkout@v2 - run: | git submodule update --init git -C deps/abcBridge submodule update --init - - name: Publish to Registry - uses: docker/build-push-action@v1 + + - uses: rlespinasse/github-slug-action@v3.x + + - run: echo "COMMON_TAG=${{ steps.prefix.outputs.prefix }}$GITHUB_REF_SLUG" >> $GITHUB_ENV + + - uses: docker/setup-buildx-action@v1 + + - uses: crazy-max/ghaction-docker-meta@v1 + name: Labels + id: labels with: - repository: galoisinc/saw - push: false + images: ${{ matrix.image }} - docker-server-test: - runs-on: ubuntu-latest - needs: [outputs] - strategy: - fail-fast: false - name: Docker SAW Remote API Test - steps: - - uses: actions/checkout@v2 + - uses: docker/login-action@v1 with: - submodules: true - - run: git -C deps/abcBridge submodule update --init - - uses: actions/setup-python@v2 + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - uses: docker/build-push-action@v2 + with: + context: . + tags: ${{ matrix.image }}:${{ steps.common-tag.outputs.common-tag }} + labels: ${{ steps.labels.outputs.labels }} + load: true + push: false + file: ${{ matrix.file }} + build-args: ${{ matrix.build-args }} + cache-from: | + type=registry,ref=${{ matrix.cache }}:${{ steps.prefix.outputs.prefix }}master + type=registry,ref=${{ matrix.cache }}:${{ steps.common-tag.outputs.common-tag }} + + - name: Cache image build + uses: docker/build-push-action@v2 + continue-on-error: true # Tolerate cache upload failures - this should be handled better + with: + file: ${{ matrix.file }} + build-args: ${{ matrix.build-args }} + cache-to: type=registry,ref=${{ matrix.cache }}:${{ steps.common-tag.outputs.common-tag }},mode=max + + - if: matrix.image == 'ghcr.io/galoisinc/saw-remote-api' + uses: actions/setup-python@v2 with: python-version: '3.9' - - uses: abatilo/actions-poetry@v2.0.0 + + - if: matrix.image == 'ghcr.io/galoisinc/saw-remote-api' + uses: abatilo/actions-poetry@v2.0.0 with: poetry-version: 1.1.5 - - shell: bash - run: saw-remote-api/scripts/build_and_test_docker.sh + + - if: matrix.image == 'ghcr.io/galoisinc/saw-remote-api' + name: Test saw-remote-api + run: ./saw-remote-api/scripts/test_docker.sh ${{ matrix.image }}:$COMMON_TAG + + - if: needs.config.outputs.event-schedule == 'true' + name: ${{ matrix.image }}:nightly + run: | + docker tag ${{ matrix.image }}:$COMMON_TAG ${{ matrix.image }}:nightly + docker push ${{ matrix.image }}:nightly + + - if: needs.config.outputs.event-tag == 'true' + name: ${{ matrix.image }}:${{ needs.config.outputs.version }} + run: | + docker tag ${{ matrix.image }}:$COMMON_TAG ${{ matrix.image }}:${{ needs.config.outputs.version }} + docker push ${{ matrix.image }}:${{ needs.config.outputs.version }} + docker tag ${{ matrix.image }}:$COMMON_TAG ${{ matrix.image }}:latest + docker push ${{ matrix.image }}:latest s2n-tests: name: "Test s2n proofs" @@ -355,45 +421,6 @@ jobs: chmod +x bin/saw make ${{ matrix.s2n-target }} - bundle: - runs-on: ${{ matrix.os }} - if: needs.outputs.outputs.release - strategy: - fail-fast: false - matrix: - os: [ubuntu-18.04, macos-latest, windows-latest] - ghc: ["8.6.5"] - needs: [outputs, build] - env: - VERSION: ${{ needs.outputs.outputs.saw-version }} - RELEASE: ${{ needs.outputs.outputs.release }} - steps: - - uses: actions/checkout@v2 - - run: | - git submodule update --init - git -C deps/abcBridge submodule update --init - - - name: "Download Artifact: dist/bin" - uses: actions/download-artifact@v2 - with: - path: dist/bin - name: ${{ runner.os }}-bins - - - shell: bash - run: .github/ci.sh bundle_files - - - shell: bash - env: - SIGNING_PASSPHRASE: ${{ secrets.SIGNING_PASSPHRASE }} - SIGNING_KEY: ${{ secrets.SIGNING_KEY }} - run: .github/ci.sh zip_dist - - - name: "Upload Artifact: ${{ needs.outputs.outputs.name }}-${{ runner.os }}-x86_64.tar.gz" - uses: actions/upload-artifact@v2 - with: - name: ${{ needs.outputs.outputs.name }}-${{ runner.os }}-x86_64 - path: "${{ needs.outputs.outputs.name }}-${{ runner.os }}-x86_64.tar.gz*" - # Indicates sufficient CI success for the purposes of mergify merging the pull # request, see .github/mergify.yml. This is done instead of enumerating each # instance of each job in the mergify configuration for a number of reasons: @@ -406,7 +433,7 @@ jobs: - build - saw-remote-api-tests - cabal-test - - docker-server-test + - build-push-image - s2n-tests steps: - run: "true" diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml deleted file mode 100644 index af583a89d9..0000000000 --- a/.github/workflows/docker.yml +++ /dev/null @@ -1,45 +0,0 @@ -name: Docker -on: - push: - tags: ["v?[0-9]+.[0-9]+.([0-9]+)?"] - workflow_dispatch: - -jobs: - saw: - runs-on: ubuntu-18.04 - steps: - - uses: actions/checkout@v2 - - run: | - git submodule update --init - git -C deps/abcBridge submodule update --init - - run: .github/ci.sh set_version - id: outputs - - name: Publish to Registry - uses: docker/build-push-action@v1 - with: - username: ${{ secrets.GITHUBSAW_USERNAME }} - password: ${{ secrets.GITHUBSAW }} - repository: galoisinc/saw - tags: "latest,${{ steps.outputs.outputs.saw-version }}" - dockerfile: saw/Dockerfile - add_git_labels: true - push: true - saw-remote-api: - runs-on: ubuntu-18.04 - steps: - - uses: actions/checkout@v2 - - run: | - git submodule update --init - git -C deps/abcBridge submodule update --init - - run: .github/ci.sh set_version - id: outputs - - name: Publish to Registry - uses: docker/build-push-action@v1 - with: - username: ${{ secrets.GITHUBSAW_USERNAME }} - password: ${{ secrets.GITHUBSAW }} - repository: galoisinc/saw-remote-api - tags: "latest,${{ steps.outputs.outputs.saw-version }}" - dockerfile: saw-remote-api/Dockerfile - add_git_labels: true - push: true diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml deleted file mode 100644 index 5dbdc67ddc..0000000000 --- a/.github/workflows/nightly.yml +++ /dev/null @@ -1,132 +0,0 @@ -name: Nightly Builds -on: - push: - branches: ["actions/**"] - schedule: - - cron: "0 0 * * *" - workflow_dispatch: - inputs: - publishNightly: - description: "If true, upload the docker images if they build successfully (with tag: nightly)" - required: false - default: "false" - -env: - CACHE_VERSION: 1 - -jobs: - outputs: - runs-on: ubuntu-18.04 - outputs: - saw-version: ${{ steps.saw.outputs.saw-version }} - name: ${{ steps.saw.outputs.name }} - publish: ${{ steps.publish.outputs.publish }} - steps: - - uses: actions/checkout@v2 - - id: saw - run: | - .github/ci.sh set_version - .github/ci.sh output name saw-$(.github/ci.sh ver)-$(date -I) - - id: publish - if: | - github.event_name == 'schedule' || - (github.event_name == 'workflow_dispatch' && github.event.inputs.publishNightly == 'true') - run: .github/ci.sh output publish 1 - - docker-saw: - runs-on: ubuntu-18.04 - needs: [outputs] - steps: - - uses: actions/checkout@v2 - with: - submodules: true - - run: git -C deps/abcBridge submodule update --init - - name: Publish to Registry - uses: docker/build-push-action@v1 - with: - username: ${{ secrets.GITHUBSAW_USERNAME }} - password: ${{ secrets.GITHUBSAW }} - repository: galoisinc/saw - tags: "nightly" - dockerfile: saw/Dockerfile - add_git_labels: true - push: ${{ github.event_name == 'schedule' || needs.outputs.outputs.publish }} - - docker-saw-remote-api: - runs-on: ubuntu-18.04 - needs: [outputs] - steps: - - uses: actions/checkout@v2 - with: - submodules: true - - run: git -C deps/abcBridge submodule update --init - - name: Publish to Registry - uses: docker/build-push-action@v1 - with: - username: ${{ secrets.GITHUBSAW_USERNAME }} - password: ${{ secrets.GITHUBSAW }} - repository: galoisinc/saw-remote-api - tags: "nightly" - dockerfile: saw-remote-api/Dockerfile - add_git_labels: true - push: ${{ github.event_name == 'schedule' || needs.outputs.outputs.publish }} - - build: - needs: [outputs] - runs-on: ${{ matrix.os }} - env: - VERSION: ${{ needs.outputs.outputs.saw-version }} - strategy: - matrix: - os: [ubuntu-18.04, macos-latest, windows-latest] - ghc: [8.8.4] - steps: - - uses: actions/checkout@v2 - with: - submodules: true - - run: git -C deps/abcBridge submodule update --init - - - uses: haskell/actions/setup@v1 - id: setup-haskell - with: - ghc-version: ${{ matrix.ghc }} - - - uses: actions/cache@v2 - name: Cache cabal store - with: - path: | - ${{ steps.setup-haskell.outputs.cabal-store }} - dist-newstyle - key: ${{ env.CACHE_VERSION }}-cabal-${{ runner.os }}-${{ matrix.ghc }}-${{ hashFiles(format('cabal.GHC-{0}.config', matrix.ghc)) }}-${{ github.sha }} - restore-keys: | - ${{ env.CACHE_VERSION }}-cabal-${{ runner.os }}-${{ matrix.ghc }}-${{ hashFiles(format('cabal.GHC-{0}.config', matrix.ghc)) }}- - - - shell: bash - run: .github/ci.sh build_abc - if: runner.os != 'Windows' - - - shell: bash - run: .github/ci.sh build - - - shell: bash - run: .github/ci.sh setup_dist_bins - - - shell: bash - run: | - .github/ci.sh build_cryptol - .github/ci.sh extract_exe "cryptol" "dist/bin" "deps/cryptol/dist-newstyle" - - - shell: bash - run: .github/ci.sh bundle_files - - - shell: bash - env: - SIGNING_PASSPHRASE: ${{ secrets.SIGNING_PASSPHRASE }} - SIGNING_KEY: ${{ secrets.SIGNING_KEY }} - name: ${{ needs.outputs.outputs.name }}-${{ runner.os }}-x86_64 - run: .github/ci.sh zip_dist - - - uses: actions/upload-artifact@v2 - with: - name: ${{ needs.outputs.outputs.name }}-${{ runner.os }}-x86_64 - path: "${{ needs.outputs.outputs.name }}-${{ runner.os }}-x86_64.tar.gz*" diff --git a/saw-remote-api/scripts/build_and_test_docker.sh b/saw-remote-api/scripts/test_docker.sh similarity index 97% rename from saw-remote-api/scripts/build_and_test_docker.sh rename to saw-remote-api/scripts/test_docker.sh index 1c91ed3ac3..79ec22a607 100755 --- a/saw-remote-api/scripts/build_and_test_docker.sh +++ b/saw-remote-api/scripts/test_docker.sh @@ -4,6 +4,8 @@ echo "Building and testing saw-remote-api docker image..." DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +TAG=${1:-saw-remote-api} + pushd $DIR/../.. docker build -t saw-remote-api --file saw-remote-api/Dockerfile . @@ -14,7 +16,7 @@ pushd $DIR/.. docker run --name=saw-remote-api -d \ -v $PWD/python/tests/saw/test-files:/home/saw/tests/saw/test-files \ -p 8080:8080 \ - saw-remote-api + "$TAG" if (( $? != 0 )); then echo "Failed to launch docker container"