azure-pipelines.yml

variables:
  DOCKER_CI_BUILD_TAG: monorepo.$(Build.BuildID)
  CORE_IMAGE_BUILD: augurproject/augur-core:$(DOCKER_CI_BUILD_TAG)
  CORE_IMAGE_LATEST: augurproject/augur-core:monorepo
  PRIMARY_NODE_VERSION: 12.18.x
  GETH_VERSION: v1.9.19
  SOLC_VERSION: v0.5.15
  SOLC_MD5: 884dbc8c8ed01cc34799a6ffab11bc3a
  VM_IMAGE: 'ubuntu-18.04'
  YARN_REGISTRY: https://pkgs.dev.azure.com/augurproject/_packaging/npmPackages/npm/registry/
  System.debug: true

trigger:
  batch: false
  branches:
    include:
      - dev
      - v2
      - release/*
      - azure/*
      - refs/tags/*

jobs:
  - job: PreFlightCheck
    displayName: 'Pre-Flight Check'
    pool:
      vmImage: $(VM_IMAGE)
    steps:
      - checkout: self
      - bash: |
          set -euo pipefail
          echo "Build Source Branch: $(Build.SourceBranchName)"
          echo "commit message: $(Build.SourceVersionMessage)"
          if [[ "$(Build.SourceBranchName)" == "dev" || "$(Build.SourceBranchName)" == "v2" ]]; then
            GIT_COMMAND='git log -m -1 --name-only --pretty="format:" $(Build.SourceVersion)'
          else
            GIT_COMMAND='git diff --name-only origin/$(System.PullRequest.TargetBranch)..$(System.SourceVersion)'
          fi
          echo "running git command: $GIT_COMMAND"
          for f in $(eval "$GIT_COMMAND");
            do
              echo "$f"
              if [[ $f == packages/* ]] && [[ ! $f =~ .*packages/augur-ui/* ]] && [[ ! $f =~ .*packages/augur-artifacts/* ]]
              then
                echo "found a non-ui change: $f"
                echo "##vso[task.setvariable variable=JOB_TRIGGER_TEST;isOutput=true]true"
              fi
              if [[ $f =~ .*augur-core/.* ]]
              then
                echo "found a core change: $f"
                echo "##vso[task.setvariable variable=JOB_TRIGGER_CORE;isOutput=true]true"
              fi
              if [[ $f =~ .*augur-artifacts/src/contracts.json ]]
              then
                echo "found an artifacts change: $f"
                echo "##vso[task.setvariable variable=JOB_TRIGGER_POP_GETH;isOutput=true]true"
              fi
              if [[ $f =~ .*augur-tools/docker-builder/* ]]
              then
                echo "found a docker-builder change: $f"
                echo "##vso[task.setvariable variable=JOB_TRIGGER_POP_GETH;isOutput=true]true"
              fi
              if [[ $f =~ .*augur-tools/ethereum-nodes/geth-poa/.* ]]
              then
                echo "found a change in geth-poa: $f"
                echo "##vso[task.setvariable variable=JOB_TRIGGER_GETH_POA;isOutput=true]true"
                echo "##vso[task.setvariable variable=JOB_TRIGGER_POP_GETH;isOutput=true]true"
              fi
              if [[ $f =~ .*augur-ui.* ]]
              then
                echo "found a ui change: $f"
                echo "##vso[task.setvariable variable=JOB_TRIGGER_UI;isOutput=true]true"
              fi
              if [[ $f =~ .*augur-sdk.* ]]
              then
                echo "found a sdk change: $f"
                echo "##vso[task.setvariable variable=JOB_TRIGGER_SDK;isOutput=true]true"
              fi
            done
        displayName: check git for changes
        name: setVars

  - job: TestAndLint
    dependsOn:
      - PreFlightCheck
    condition: |
      and
      (
        succeeded(),
        not(eq(variables['Build.sourceBranch'], 'refs/heads/dev')),
        not(eq(variables['Build.sourceBranch'], 'refs/heads/demo')),
        not(eq(variables['Build.sourceBranch'], 'refs/heads/v2')),
        not(eq(variables['Build.sourceBranch'], 'refs/tags')),
        eq(dependencies.PreFlightCheck.outputs['setVars.JOB_TRIGGER_SDK'], 'true')
      )
    variables:
      NORMAL_TIME: true
      PRODUCTION: false
    pool: 'Default'
    workspace:
      clean: resources
    strategy:
      matrix:
        non_specific_tests:
          node_version: $(PRIMARY_NODE_VERSION)
          TEST_ARGS: --testPathIgnorePatterns="(Universe.test.ts|(getter/(markets|users)/))"
        market_getters_tests:
          node_version: $(PRIMARY_NODE_VERSION)
          TEST_ARGS: --testPathPattern 'getter/markets/'
        user_getter_tests:
          node_version: $(PRIMARY_NODE_VERSION)
          TEST_ARGS: --testPathPattern 'getter/users/'
        universe_tests:
          node_version: $(PRIMARY_NODE_VERSION)
          TEST_ARGS: --testPathPattern 'Universe.test.ts'
    timeoutInMinutes: 120
    steps:
      - template: .azure-templates/node-setup.yml
      - task: Cache@2
        inputs:
          key: 'contract | "$(Agent.OS)" | packages/augur-artifacts/src/contracts.json'
          path: /tmp/augur
        displayName: Cache seed file
      - script: docker system prune -f
      - script: rm -rf /tmp/augur/seed.json
      - script: yarn test $(TEST_ARGS) --verbose --ci --reporters=default --reporters=jest-junit
        displayName: yarn test
        env:
          NODE_OPTIONS: "--max_old_space_size=4096"
      - task: PublishTestResults@2
        condition: succeededOrFailed()
        inputs:
          testRunner: JUnit
          testResultsFiles: '**/junit.xml'

  - job: BuildUI
    pool:
      vmImage: $(VM_IMAGE)
    dependsOn:
      - PreFlightCheck
    variables:
      JOB_TRIGGER_UI: $[dependencies.PreFlightCheck.outputs['setVars.JOB_TRIGGER_UI']]
    condition:  eq(variables['Build.Reason'], 'PullRequest')
    steps:
      - template: .azure-templates/node-setup.yml
      - bash: |
          set -euo pipefail
          export ETHEREUM_NETWORK=v2
          ( cd packages/orbit-web && ./node_modules/.bin/webpack)
          ( cd packages/augur-ui && ./node_modules/.bin/webpack)
          #yarn workspace @augurproject/ui build:kovan
        displayName: ui build
        env:
          NODE_OPTIONS: "--max_old_space_size=6144"

  - job: ContractSanityCheck
    dependsOn:
      - PreFlightCheck
    condition: |
      and
      (
        succeeded(),
        eq(dependencies.PreFlightCheck.outputs['setVars.JOB_TRIGGER_CORE'], 'true')
      )
    pool:
      vmImage: $(VM_IMAGE)
    timeoutInMinutes: 120
    steps:
      - template: .azure-templates/node-setup.yml
      - task: UsePythonVersion@0
        inputs:
          versionSpec: '3.6'
      - bash: |
          set -euo pipefail
          python3 -m venv venv && \
          source venv/bin/activate && \
          python3 -m pip install --no-cache-dir -r packages/augur-core/requirements.txt
        displayName: Python3 requirements
      - bash: |
          set -exuo pipefail
          echo "$(SOLC_MD5) */usr/local/bin/solc" > solc.md5 && \
          sudo curl -sL -o /usr/local/bin/solc https://github.com/ethereum/solidity/releases/download/${SOLC_VERSION}/solc-static-linux && \
          md5sum -c solc.md5 && \
          sudo chmod a+x /usr/local/bin/solc
        displayName: install solc
      - bash: |
          set -euo pipefail
          source venv/bin/activate
          yarn workspace @augurproject/core build
        displayName: compile contracts
      - bash: |
          set -euo pipefail
          git update-index -q --ignore-submodules --refresh
          err=0
          if ! git diff-files --quiet --ignore-submodules --
          then
              echo >&2 "unstaged changes detected"
              git diff-files --name-status -r --ignore-submodules -- >&2
              git diff
              err=1
          fi
          if ! git diff-index --cached --quiet HEAD --ignore-submodules --
          then
              echo >&2 "uncommitted changes detected"
              git diff-index --cached --name-status -r --ignore-submodules HEAD -- >&2
              err=1
          fi
          if [ $err = 1 ]
          then
              echo >&2 "Contract changes detected"
              echo >&2 "Please rebuild augur-core and commit the generated files:"
              echo >&2 "yarn workspace @augurproject/core build"
              #exit 1
          fi
        displayName: check for contract changes

  - job: DockerGethDev
    dependsOn:
      - PreFlightCheck
    pool:
      vmImage: $(VM_IMAGE)
    condition: |
      and
      (
        succeeded(),
        eq(dependencies.PreFlightCheck.outputs['setVars.JOB_TRIGGER_GETH_POA'], 'true')
      )
    steps:
      - checkout: self
      - task: Docker@1
        displayName: docker login
        inputs:
          command: login
          containerRegistryType: Container Registry
          dockerRegistryEndpoint: dockerhub-augurproject
      - script: |
          set -exuo pipefail
          sudo modprobe -r overlay && sudo modprobe overlay redirect_dir=off && sudo systemctl restart docker
          sudo systemctl status docker
          cd packages/augur-tools/ethereum-nodes/geth-poa && docker image build --tag augurproject/dev-node-geth:$(GETH_VERSION) .
          docker tag augurproject/dev-node-geth:$(GETH_VERSION) augurproject/dev-node-geth:latest
          docker push augurproject/dev-node-geth:$(GETH_VERSION)
          #docker push augurproject/dev-node-geth:latest
        displayName: 'build and push dev-node-geth image'
      - task: Docker@1
        displayName: docker logout
        inputs:
          command: logout
          containerRegistryType: Container Registry
          dockerRegistryEndpoint: dockerhub-augurproject

  - job: TestAugurCore
    dependsOn:
      - ContractSanityCheck
      - DockerGethDev
    condition: |
      and
      (
        in(dependencies.ContractSanityCheck.result, 'Succeeded'),
        in(dependencies.DockerGethDev.result, 'Succeeded', 'Skipped')
      )
    pool:
      vmImage: $(VM_IMAGE)
    timeoutInMinutes: 120
    strategy:
      maxParallel: 5
      matrix:
        trading_and_libs:
          TESTS: tests/test*.py tests/libraries tests/trading
        reporting:
          TESTS: tests/reporting
        fuzzy:
          TESTS: tests/fuzzy
        unit:
          TESTS: tests/unit
        geth_integration:
          TESTS: integration:geth
        # TODO parity_integration:
        #  TESTS: integration:parity
        security_mithril:
          TESTS: security:mythril
    steps:
      - template: .azure-templates/node-setup.yml
      - task: UsePythonVersion@0
        inputs:
          versionSpec: '3.6'
      - bash: |
          set -exuo pipefail
          echo "$(SOLC_MD5) */usr/local/bin/solc" > solc.md5 && \
          sudo curl -sL -o /usr/local/bin/solc https://github.com/ethereum/solidity/releases/download/${SOLC_VERSION}/solc-static-linux && \
          md5sum -c solc.md5 && \
          sudo chmod a+x /usr/local/bin/solc
        displayName: install solc
      - bash: |
          set -euo pipefail
          python3 -m venv venv && \
          source venv/bin/activate && \
          python3 -m pip install --no-cache-dir -r packages/augur-core/requirements.txt
          python3 -m pip install pytest
          echo "tests: $(TESTS)"
          export ETHEREUM_HTTP=http://localhost:8545
          export ETHEREUM_GAS_PRICE_IN_NANOETH=1
          export ETHEREUM_PRIVATE_KEY=0xfae42052f82bed612a724fec3632f325f377120592c75bb78adfcceae6470c5a
          if [[ "$(TESTS)" == "integration:geth" ]]; then
            docker run -p 8545:8545 -p 47625:8546 --name geth --detach augurproject/dev-node-geth:v1.9.9
            yarn workspace @augurproject/core test:integration
            docker stop geth
          elif [[ "$(TESTS)" == "integration:parity" ]]; then
            docker run -p 8180:8180 -p 8545:8545 -p 47623:8546 --name parity --detach augurproject/dev-node-parity:latest
            yarn workspace @augurproject/core test:integration
            docker stop parity
          elif [[ "$(TESTS)" == "security:mythril" ]]; then
            yarn workspace @augurproject/core docker:run:test:security:mythril;
            cat $(find . -name test-results.log)
          elif [[ "$(TESTS)" == "security:maian" ]]; then
            docker run -v `pwd`/contracts:/augur/packages/augur-core/output/contracts/ cryptomental/maian-augur-ci python /scripts/test_runner.py
          elif [[ "$(TESTS)" == "security:smt" ]]; then
            yarn workspace @augurproject/core docker:run:test:security:smt;
          elif [[ "$(TESTS)" == "security:oyente" ]]; then
            yarn workspace @augurproject/core docker:run:test:security:oyente;
          else
            (cd packages/augur-core && yarn test:unit $(TESTS))
          fi
        displayName: integration tests

  - job: Docker_populated_geth_images
    dependsOn:
      - PreFlightCheck
      - TestAugurCore
      - TestAndLint
      - ContractSanityCheck
    condition: |
      and
      (
        eq(dependencies.PreFlightCheck.outputs['setVars.JOB_TRIGGER_POP_GETH'], 'true'),
        in(dependencies.PreFlightCheck.result, 'Succeeded'),
        in(dependencies.TestAugurCore.result, 'Succeeded', 'Skipped'),
        in(dependencies.TestAndLint.result, 'Succeeded', 'Skipped'),
        in(dependencies.ContractSanityCheck.result, 'Succeeded', 'Skipped'),
        not(eq(variables['Build.sourceBranch'], 'refs/heads/dev')),
        not(eq(variables['Build.sourceBranch'], 'refs/heads/demo')),
        not(eq(variables['Build.sourceBranch'], 'refs/heads/v2')),
        not(eq(variables['Build.sourceBranch'], 'refs/tags'))
      )
    pool: 'Default'
    workspace:
      clean: resources
    timeoutInMinutes: 240
    strategy:
      matrix:
        dev-pop-geth-v2:
          IMAGE_NAME: dev-pop-geth-v2
          NETWORK_ID: 102
          NORMAL_TIME: false
        dev-pop-normtime-geth-v2:
          IMAGE_NAME: dev-pop-normtime-geth-v2
          NETWORK_ID: 103
          NORMAL_TIME: true
    variables:
      ZEROX_RPC_ENABLED: false
      ZEROX_MESH_ENABLED: false
      GSN_ENABLED: false
    steps:
      - template: .azure-templates/node-setup.yml
      - script: |
          set -exuo pipefail
          export DOCKER_BUILDKIT=1
          docker system prune -f
          docker build --progress=plain --no-cache=true -f packages/augur-tools/docker-builder/Dockerfile --build-arg normal_time=$(NORMAL_TIME) --build-arg network_id=$(NETWORK_ID) -t $(IMAGE_NAME) .
        displayName: 'build $(IMAGE_NAME) image'
      - task: Docker@1
        displayName: docker login
        inputs:
          command: login
          containerRegistryType: Container Registry
          dockerRegistryEndpoint: dockerhub-augurproject
      - bash: |
          set -exuo pipefail
          CONTRACT_SHA=$(cd packages/augur-tools/ && node scripts/get-contract-hashes.js)
          docker images
          docker tag $(IMAGE_NAME) augurproject/$(IMAGE_NAME):"${CONTRACT_SHA}"
          docker push augurproject/$(IMAGE_NAME):"${CONTRACT_SHA}"
        displayName: Push Docker Images to hub
      - task: Docker@1
        displayName: docker logout
        inputs:
          command: logout
          containerRegistryType: Container Registry
          dockerRegistryEndpoint: dockerhub-augurproject

  - job: test_3rd_party_integrations
    dependsOn:
      - Docker_populated_geth_images
      - PreFlightCheck
      - TestAugurCore
      - TestAndLint
      - ContractSanityCheck
    condition: |
      and
      (
        eq(dependencies.PreFlightCheck.outputs['setVars.JOB_TRIGGER_POP_GETH'], 'true'),
        in(dependencies.PreFlightCheck.result, 'Succeeded'),
        in(dependencies.TestAugurCore.result, 'Succeeded', 'Skipped'),
        in(dependencies.TestAndLint.result, 'Succeeded', 'Skipped'),
        in(dependencies.ContractSanityCheck.result, 'Succeeded', 'Skipped'),
        in(dependencies.Docker_populated_geth_images.result, 'Succeeded', 'Skipped'),
        not(eq(variables['Build.sourceBranch'], 'refs/heads/dev')),
        not(eq(variables['Build.sourceBranch'], 'refs/heads/demo')),
        not(eq(variables['Build.sourceBranch'], 'refs/heads/v2')),
        not(eq(variables['Build.sourceBranch'], 'refs/tags'))
      )
    pool:
      vmImage: $(VM_IMAGE)
    steps:
      - template: .azure-templates/node-setup.yml
      - bash: |
          set -exuo pipefail
          yarn docker:all --detach
          sleep 60
        displayName: Run 3rd Party Dockers
      - bash: |
          set -exuo pipefail
          free -m
          yarn test --config=packages/augur-test/src/tests/3rd-party/jest.config.js --runInBand --verbose --ci --reporters=default --reporters=jest-junit
        displayName: Test 3rd party integratios
      - bash: |
          set -exuo pipefail
          docker kill geth
        displayName: Stop 3rd Party Dockers

  - template: .azure-templates/ui-build-to-s3.yml
    parameters:
      name: deploy_v2_to_staging
      environment: v2
      deployBucketName: staging.augur.net
      triggerBranch: refs/heads/v2
      cloudfrontDistributionId: E3RQRY8KA1PEUL

  - template: .azure-templates/ui-build-to-s3.yml
    parameters:
      name: deploy_v2_to_reporting
      environment: v2
      reportingOnly: true
      deployBucketName: v2-reporting.augur.net
      triggerBranch: refs/heads/v2
      cloudfrontDistributionId: E34VY4MHHEDAH9

  - template: .azure-templates/ui-build-to-s3.yml
    parameters:
      name: deploy_dev_to_dev
      environment: dev
      deployBucketName: dev.augur.net
      triggerBranch: refs/heads/dev
      cloudfrontDistributionId: E2K6239JF3WYG6

  - job: github_release
    displayName: github release
    dependsOn: PreFlightCheck
    condition: |
      and
      (
        succeeded(),
        startsWith(variables['build.sourceBranch'], 'refs/tags')
      )
    pool:
      vmImage: $(VM_IMAGE)
    steps:
      - template: .azure-templates/node-setup.yml
      - bash: |
          set -euo pipefail
          yarn build
        displayName: yarn build
      - bash: |
          set -euo pipefail
          export ETHEREUM_NETWORK=v2
          ( cd packages/orbit-web && ./node_modules/.bin/webpack)
          ( cd packages/augur-ui && REPORTING_ONLY=true ./node_modules/.bin/webpack)
          mv packages/augur-ui/build reporting-build
          ( cd packages/augur-ui && ./node_modules/.bin/webpack)
          mv packages/augur-ui/build trading-build
        displayName: ui build
        env:
          NODE_OPTIONS: "--max_old_space_size=6144"
      - bash: |
          set -euo pipefail
          sudo apt update
          sudo apt install golang-go -y
          curl -l https://dist.ipfs.io/go-ipfs/v0.5.1/go-ipfs_v0.5.1_linux-amd64.tar.gz | tar xvzf -
          sudo mv go-ipfs/ipfs /usr/local/bin/ipfs
        displayName: install ipfs
      - bash: |
          set -euo pipefail
          ipfs init > /dev/null 2>&1
          IPFS_HASH_REPORTING_V0=$(ipfs add -r -Q -n --pin=false --cid-version=0 reporting-build)
          IPFS_HASH_REPORTING_V1=$(ipfs add -r -Q -n --pin=false --cid-version=1 reporting-build)
          IPFS_HASH_TRADING_V0=$(ipfs add -r -Q -n --pin=false --cid-version=0 trading-build)
          IPFS_HASH_TRADING_V1=$(ipfs add -r -Q -n --pin=false --cid-version=1 trading-build)
          (
          cat <<-EOF
          # Augur V2 Release

          ## IPFS Hashes
          * Reporting UI
            * CIDv0: ${IPFS_HASH_REPORTING_V0}
            * CIDv1: ${IPFS_HASH_REPORTING_V1}
          * Trading UI
            * CIDv0: ${IPFS_HASH_TRADING_V0}
            * CIDv1: ${IPFS_HASH_TRADING_V1}
          ---
          EOF
          ) > release-notes.txt
        displayName: Generate Release Notes
      - task: CopyFiles@2
        inputs:
          contents: |
            release-notes.txt
          targetFolder: $(Build.ArtifactStagingDirectory)
          cleanTargetFolder: true
      - task: GitHubRelease@1
        displayName: 'GitHub release (create)'
        inputs:
          gitHubConnection: 'Github OAuth'
          tagSource: 'userSpecifiedTag'
          tag: '$(Build.SourceBranchName)'
          title: 'Augur $(Build.SourceBranchName)'
          assets: '$(Build.ArtifactStagingDirectory)/*'
          releaseNotesSource: filePath
          releaseNotesFile: '$(Build.ArtifactStagingDirectory)/release-notes.txt'
          isDraft: true
          isPreRelease: true