Force rustup update on CI for building polkadot

.github/workflows/build.yml

@@ -195,6 +195,28 @@ jobs:
             ./scripts/build-alphanet-relay-image.sh
             docker push $DOCKER_TAG
           fi
+      - name: Check & prepare para-test docker image
+        run: |
+          POLKADOT_REPO=${{ needs.set-tags.outputs.polkadot_repo }}
+          POLKADOT_COMMIT=${{ needs.set-tags.outputs.polkadot_commit }}
+          DOCKER_TAG="purestake/polkadot-para-tests:sha-$POLKADOT_COMMIT"
+          docker rmi --force $DOCKER_TAG
+          POLKADOT_EXISTS=$(docker manifest inspect $DOCKER_TAG > /dev/null && \
+            echo "true" || echo "false")
+
+          if [[ "$POLKADOT_EXISTS" == "false" ]]; then
+            mkdir -p build
+            MOONBEAM_DOCKER_TAG="purestake/moonbase-relay-testnet:sha-$POLKADOT_COMMIT"
+            docker create --pull always -ti --name dummy $MOONBEAM_DOCKER_TAG bash
+            docker cp dummy:/usr/local/bin/polkadot build/polkadot
+            docker rm -f dummy
+
+            docker build . --pull --no-cache -f docker/polkadot-para-tests.Dockerfile \
+              --network=host \
+              --build-arg HOST_UID="$UID" \
+              -t $DOCKER_TAG
+            docker push $DOCKER_TAG
+          fi
 
   ####### Building and Testing binaries #######
 
@@ -219,8 +241,6 @@ jobs:
       RUSTFLAGS: "-C opt-level=3 -D warnings"
       # MOONBEAM_LOG: info
       # DEBUG: "test*"
-    outputs:
-      RUSTC: ${{ steps.get-rust-versions.outputs.rustc }}
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -251,21 +271,37 @@ jobs:
           fi
           $SCCACHE_BIN -s
           echo "RUSTC_WRAPPER=$SCCACHE_BIN" >> $GITHUB_ENV
-      - id: get-rust-versions
-        run: |
-          echo "::set-output name=rustc::$(rustc --version)"
-      - name: Build Node
-        run: |
-          env
-          cargo build --release --all
-      - name: Verify node version
-        run: |
-          GIT_COMMIT=`git log -1 --format="%H" | cut -c1-7`
-          MB_VERSION=`./target/release/moonbeam --version`
-          echo "Checking $MB_VERSION contains $GIT_COMMIT"
-          echo "$MB_VERSION" | grep $GIT_COMMIT
-      - name: Unit tests
-        run: cargo test --release --all
+      # - name: Build Node
+      #   run: |
+      #     env
+      #     cargo build --release --all
+      - name: TODO remove - Download artifact moonbeam
+        uses: dawidd6/action-download-artifact@v2
+        with:
+          github_token: ${{secrets.GITHUB_TOKEN}}
+          # Required, workflow file name or ID
+          workflow: build.yml
+          name: moonbeam
+          run_id: 2296555362
+          path: build
+      - name: TODO remove - Download artifact runtime
+        uses: dawidd6/action-download-artifact@v2
+        with:
+          github_token: ${{secrets.GITHUB_TOKEN}}
+          # Required, workflow file name or ID
+          workflow: build.yml
+          name: runtimes
+          run_id: 2296555362
+          path: runtimes
+
+      # - name: Verify node version
+      #   run: |
+      #     GIT_COMMIT=`git log -1 --format="%H" | cut -c1-7`
+      #     MB_VERSION=`./target/release/moonbeam --version`
+      #     echo "Checking $MB_VERSION contains $GIT_COMMIT"
+      #     echo "$MB_VERSION" | grep $GIT_COMMIT
+      # - name: Unit tests
+      #   run: cargo test --release --all
 
       # We determine whether there are unmodified Cargo.lock files by:
       # 1. Asking git for a list of all modified files
@@ -289,21 +325,21 @@ jobs:
             echo $DIFF_INDEX | grep Cargo.lock
             false
           fi
-      - name: Ensure benchmarking compiles
-        run: cargo check --release --features=runtime-benchmarks
-      - name: Save moonbeam binary
-        run: |
-          mkdir -p build
-          cp target/release/moonbeam build/moonbeam;
+      # - name: Ensure benchmarking compiles
+      #   run: cargo check --release --features=runtime-benchmarks
+      # - name: Save moonbeam binary
+      #   run: |
+      #     mkdir -p build
+      #     cp target/release/moonbeam build/moonbeam;
       - name: Upload binary
         uses: actions/upload-artifact@v2
         with:
           name: moonbeam
           path: build
-      - name: Save runtime wasm
-        run: |
-          mkdir -p runtimes
-          cp target/release/wbuild/moon*/moon*_runtime.compact.compressed.wasm runtimes/;
+      # - name: Save runtime wasm
+      #   run: |
+      #     mkdir -p runtimes
+      #     cp target/release/wbuild/moon*/moon*_runtime.compact.compressed.wasm runtimes/;
       - name: Upload runtimes
         uses: actions/upload-artifact@v2
         with:
@@ -312,7 +348,7 @@ jobs:
 
   typescript-tests:
     runs-on: self-hosted
-    needs: ["set-tags", "build"]
+    needs: ["set-tags", "build", "prepare-polkadot"]
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -330,13 +366,74 @@ jobs:
         env:
           BINARY_PATH: ../build/moonbeam
         run: |
+          #### Compute CPUS & CONCURRENCY
+          CPU_PER_TEST=3;
+          CPUS=$(lscpu  | grep '^CPU(s):' | grep -o '[0-9]*')
+          CONCURRENCY=$((CPUS/CPU_PER_TEST))
+          echo "Preparing $CONCURRENCY groups (CPUs/test: $CPU_PER_TEST, Total: $CPUS)"
+
+          ####  Preparing the repository
           chmod uog+x build/moonbeam
           cd moonbeam-types-bundle
           npm install
           npm run build
           cd ../tests
           npm install
-          node_modules/.bin/mocha --parallel -j 4 -r ts-node/register 'tests/**/test-*.ts'
+
+          #### Splitting test files in groups (for concurrent execution)
+          mapfile -t SPLIT < <(find tests/ -name "test-*.ts" |\
+            while read -r file; do \
+              L[$((COUNT%CONCURRENCY))]+="$file "; \
+              echo ${L[$((COUNT%CONCURRENCY))]}; \
+              COUNT=$(($COUNT + 1)); \
+            done | tail -$CONCURRENCY)
+
+          cd ..
+
+          #### Cleaning possible previous data
+          rm -f fail.txt docker*.txt
+
+          #### Retrieving docker image as base to run tests
+          POLKADOT_REPO=${{ needs.set-tags.outputs.polkadot_repo }}
+          POLKADOT_COMMIT=${{ needs.set-tags.outputs.polkadot_commit }}
+          DOCKER_TAG="purestake/polkadot-para-tests:sha-$POLKADOT_COMMIT"
+
+          #### Script function to launch concurrent tests
+          launch_as() {
+            local cmd_name=$1
+            shift
+            (time $@ || echo $cmd_name >> fail.txt) 2>&1 > $cmd_name.txt
+            echo "Done $cmd_name"
+          }
+
+          echo "Running tests inside docker"
+          for i in $(eval echo "{0..$((CONCURRENCY - 1))}"); do
+            launch_as docker$i docker run \
+              --cpus $((CPUS / $CONCURRENCY)) \
+              -e BINARY_PATH='../build/moonbeam' \
+              -e DEBUG='test:*' \
+              -v $(pwd):/moonbeam:z \
+              -u $UID \
+              -w /moonbeam/tests \
+              $DOCKER_TAG \
+                node node_modules/.bin/mocha --exit -r ts-node/register "${SPLIT[$i]}" &
+          done
+
+          #### Waiting for all the group to finish
+          wait
+          echo "Done executing tests"
+
+          #### Displaying output of each group
+          for i in $(eval echo "{0..$((CONCURRENCY - 1))}"); do \
+            cat docker$i.txt
+          done
+          egrep "[0-9][0-9]* (passing|pending|failing)" docker*.txt
+
+          #### Forcing error if file is present
+          if [ -f fail.txt ]; then
+            cat fail.txt
+            exit 1
+          fi
 
       # We determine whether there are unmodified package-lock.json files by:
       # 1. Asking git for a list of all modified files
@@ -398,7 +495,7 @@ jobs:
           npm run build
           cd ../tests
           npm install
-          node_modules/.bin/mocha --parallel -j 2 -r ts-node/register 'tracing-tests/**/test-*.ts'
+          node_modules/.bin/mocha --exit --parallel -j 2 -r ts-node/register 'tracing-tests/**/test-*.ts'
 
   typescript-para-tests:
     runs-on: self-hosted
@@ -417,31 +514,62 @@ jobs:
         with:
           name: runtimes
           path: tests/runtimes
-      - name: Retrieve polkadot
-        run: |
-          POLKADOT_COMMIT=${{ needs.set-tags.outputs.polkadot_commit }}
-          DOCKER_TAG="purestake/moonbase-relay-testnet:sha-$POLKADOT_COMMIT"
-          docker create -ti --name dummy $DOCKER_TAG bash
-          docker cp dummy:/usr/local/bin/polkadot build/polkadot
-          docker rm -f dummy
       - name: Use Node.js 14.x
         uses: actions/setup-node@v2
         with:
           node-version: 14.x
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
       - name: Typescript integration tests (against dev service)
-        env:
-          OVERRIDE_RUNTIME_PATH: ./runtimes
-          BINARY_PATH: ../build/moonbeam
-          MOONBEAM_LOG: info
-          RELAY_BINARY_PATH: ../build/polkadot
         run: |
-          chmod uog+x build/polkadot
+          ####  Preparing the repository
           chmod uog+x build/moonbeam
           cd moonbeam-types-bundle
           npm install
           cd ../tests
           npm install
-          node_modules/.bin/mocha -r ts-node/register 'para-tests/**/test-*.ts'
+          cd ..
+
+          POLKADOT_REPO=${{ needs.set-tags.outputs.polkadot_repo }}
+          POLKADOT_COMMIT=${{ needs.set-tags.outputs.polkadot_commit }}
+          DOCKER_TAG="purestake/polkadot-para-tests:sha-$POLKADOT_COMMIT"
+
+          #### Retrieve binary to avoid para-node script to rely on docker
+          mkdir -p build
+          localVersion=$(grep 'spec_version: [0-9]*' runtime/moonbase/src/lib.rs | grep -o '[0-9]*')
+          baseRuntime=$(git tag -l -n 'runtime-[0-9]*' | \
+            cut -d' ' -f 1 | cut -d'-' -f 2 | \
+            sed "1 i ${localVersion}" | \
+            sort -n -r | \
+            uniq | \
+            grep -A1 "${localVersion}" | \
+            tail -1)
+          rev=$(git rev-list -1 runtime-$baseRuntime)
+          sha8=${rev:0:8}
+          dockerImage=purestake/moonbeam:sha-${sha8}
+          binaryPath=build/moonbeam-${sha8}
+          docker create --pull always --name moonbeam-tmp ${dockerImage} && \
+          docker cp moonbeam-tmp:/moonbeam/moonbeam ${binaryPath} && \
+          docker rm moonbeam-tmp
+
+          echo "Running para-tests inside docker"
+          docker run \
+            -e OVERRIDE_RUNTIME_PATH='./runtimes' \
+            -e BINARY_PATH='../build/moonbeam' \
+            -e MOONBEAM_LOG='info' \
+            -e RELAY_BINARY_PATH='/binaries/polkadot' \
+            -e RUNTIME_DIRECTORY='/tmp/runtimes' \
+            -e BINARY_DIRECTORY='../build/' \
+            -e SPECS_DIRECTORY='/tmp/specs' \
+            -v $(pwd):/moonbeam:z \
+            -v $(pwd)/${binaryPath}:/binaries/moonbeam:z \
+            -u $UID \
+            -w /moonbeam/tests \
+            $DOCKER_TAG \
+              node node_modules/.bin/mocha --exit -r ts-node/register 'para-tests/**/test-*.ts'
 
   docker-parachain:
     runs-on: ubuntu-latest

docker/polkadot-para-tests.Dockerfile

@@ -0,0 +1,27 @@
+# Node for Moonbase Parachains.
+#
+# Requires to run from repository root and to copy the binary in the build folder (part of the release workflow)
+
+FROM node
+LABEL maintainer "alan@purestake.com"
+LABEL description="Node image use to run Moonbeam para-tests"
+
+ARG HOST_UID=1001
+
+RUN ((getent passwd $HOST_UID > /dev/null) || \
+	  useradd -m -u $HOST_UID -U -s /bin/sh -d /polkadot polkadot) && \
+	mkdir -p /polkadot/.local/share/polkadot && \
+	chown -R $HOST_UID /polkadot && \
+	ln -s /polkadot/.local/share/polkadot /data
+
+RUN mkdir -p /binaries
+COPY build/polkadot /binaries/
+RUN chmod -R uog+rwX /binaries
+
+USER $HOST_UID
+
+WORKDIR /polkadot
+
+ENTRYPOINT ["docker-entrypoint.sh"]
+CMD [ "node" ]
+

moonbeam-types-bundle/package.json

@@ -2,7 +2,7 @@
   "name": "moonbeam-types-bundle",
   "version": "2.0.4",
   "description": "Bundled types to instantiate the Polkadot JS api with a Moonbeam network",
-  "main": "dist/index.js",
+  "main": "index.js",
   "types": "dist/index.d.ts",
   "prepublish": "tsc",
   "scripts": {

scripts/build-alphanet-relay-image.sh

@@ -13,7 +13,7 @@ fi
 
 echo "Using Polkadot from $POLKADOT_REPO revision #${POLKADOT_COMMIT}"
 
-docker build . -f docker/polkadot-relay.Dockerfile --network=host \
+docker build . --pull --no-cache -f docker/polkadot-relay.Dockerfile --network=host \
   --build-arg POLKADOT_COMMIT="$POLKADOT_COMMIT" \
   --build-arg POLKADOT_REPO="$POLKADOT_REPO" \
   -t purestake/moonbase-relay-testnet:sha-$POLKADOT_COMMIT