Updating dev branches with latest version from upstream

.github/mergify.yml

@@ -21,3 +21,14 @@ pull_request_rules:
         assignees:
           - "{{ author }}"
         label_conflicts: backport-conflicts
+  - name: backport patches to dev
+    conditions:
+      - base=master
+      - label=backport-dev
+    actions:
+      backport:
+        branches:
+          - dev
+        assignees:
+          - "{{ author }}"
+        label_conflicts: backport-conflicts

.github/scripts/Make.user

@@ -1,2 +1,3 @@
 FORCE_ASSERTIONS=1
-LLVM_ASSERTIONS=1
+LLVM_ASSERTIONS=1
+USE_BINARYBUILDER_MMTK_JULIA=0

.github/scripts/ci-build.sh

@@ -27,15 +27,18 @@ fi
 plan_feature=${plan,,}
 moving_feature=${is_moving,,}
 
-cd $MMTK_JULIA_DIR/mmtk
-cargo build --features $plan_feature,$moving_feature $build_args
+if [ "$is_moving" == "moving" ]; then
+    MOVING=1
+else
+    MOVING=0
+fi
 
 cd $JULIA_PATH
-
 # Clean first
 make cleanall
-# Build
+# This will build the binding in MMTK_JULIA_DIR (defined in common.sh), and link it
+# when building Julia, instead of using the set version defined in Julia itself 
 cp $BINDING_PATH/.github/scripts/Make.user $JULIA_PATH/
-MMTK_PLAN=$plan MMTK_BUILD=$build_type make
+MMTK_MOVING=$MOVING MMTK_PLAN=$plan MMTK_BUILD=$build_type make
 # Run hello world
 $JULIA_PATH/julia $HELLO_WORLD_JL

.github/scripts/ci-checkout.sh

@@ -2,8 +2,10 @@ set -ex
 
 . $(dirname "$0")/common.sh
 
-JULIA_URL=`cargo read-manifest --manifest-path=$BINDING_PATH/mmtk/Cargo.toml | python -c 'import json,sys; print(json.load(sys.stdin)["metadata"]["julia"]["julia_repo"])'`
-JULIA_VERSION=`cargo read-manifest --manifest-path=$BINDING_PATH/mmtk/Cargo.toml | python -c 'import json,sys; print(json.load(sys.stdin)["metadata"]["julia"]["julia_version"])'`
+# We may later allow setting up a specific version of Julia using comments
+# in the PR, but for now we just use the latest master from JuliaLang
+JULIA_URL=https://github.com/$1.git
+JULIA_VERSION=$2
 
 rm -rf $JULIA_PATH
 git clone $JULIA_URL $JULIA_PATH

.github/scripts/ci-ffi.sh

@@ -0,0 +1,10 @@
+set -ex
+
+. $(dirname "$0")/common.sh
+
+pushd $MMTK_JULIA_DIR
+
+make regen-bindgen-ffi
+
+popd
+

.github/scripts/ci-style.sh

@@ -6,11 +6,8 @@ export RUSTFLAGS="-D warnings"
 
 pushd $BINDING_PATH/mmtk
 
-# Currently we have many warnings from clippy, and we have PRs sitting there to be merged.
-# I am concerned that enabling this will cause merge conflicts everywhere.
-# However, we should enable this as soon as we can.
-# cargo clippy
-# cargo clippy --release
+cargo clippy
+cargo clippy --release
 
 cargo fmt -- --check
 popd

.github/scripts/ci-test-LinearAlgebra.sh

@@ -13,4 +13,4 @@ num_workers=2
 export JULIA_TEST_MAXRSS_MB=$((total_mem_restricted/ num_workers))
 
 echo "-> Run single threaded"
-ci_run_jl_test "LinearAlgebra" 2
+ci_run_jl_test "LinearAlgebra" 2

.github/scripts/ci-test-gc-core.sh

@@ -27,5 +27,3 @@ do
         ci_run_jl_test $test
     fi
 done
-
-

.github/scripts/ci-test-stdlib.sh

@@ -22,6 +22,8 @@ declare -a tests_to_skip=(
     "SparseArrays"
     # Running LinearAlgebra in a separate job
     "LinearAlgebra"
+    # Skipping distributed tests because of https://github.com/JuliaLang/Distributed.jl/issues/132
+    "Distributed"
 )
 # These tests need multiple workers.
 declare -a tests_with_multi_workers=(

.github/scripts/ci-test.sh

@@ -15,4 +15,4 @@ cd $cur
 
 # Use release build to run tests
 cd $cur
-./ci-test-gc-core.sh
+./ci-test-gc-core.sh

.github/workflows/binding-tests.yml

@@ -9,6 +9,12 @@ on:
       moving:
         required: true
         type: string
+      julia_repo:
+        required: true
+        type: string
+      julia_ref:
+        required: true
+        type: string
 
 jobs:
   build-debug:
@@ -17,26 +23,76 @@ jobs:
       - uses: actions/checkout@v2
       - name: Setup environments
         run: |
-          ./.github/scripts/ci-checkout.sh
+          ./.github/scripts/ci-checkout.sh ${{ inputs.julia_repo }} ${{ inputs.julia_ref }}
           ./.github/scripts/ci-setup.sh
           # removing these as they cause a conflict within bindgen
-          sudo rm -rf /usr/lib/llvm-14  
+          sudo rm -rf /usr/lib/llvm-14
+          sudo rm -rf /usr/lib/llvm-13
+      - name: Build Julia (Debug)
+        run: |
+          ./.github/scripts/ci-build.sh debug ${{ inputs.gc_plan }} ${{ inputs.moving }}
+      - name: Print Julia version info
+        run: |
+          JULIA_PATH=$(realpath $(dirname "$0"))/../mmtk-julia/mmtk-julia/vm/julia
+          $JULIA_PATH/julia --project=. -e "using InteractiveUtils; versioninfo()"
+      - name: Style check
+        run: |
+          ./.github/scripts/ci-style.sh
+
+  build-debug-regen-ffi:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v2
+      - name: Setup environments
+        run: |
+          ./.github/scripts/ci-checkout.sh ${{ inputs.julia_repo }} ${{ inputs.julia_ref }}
+          ./.github/scripts/ci-setup.sh
+          # removing these as they cause a conflict within bindgen
+          sudo rm -rf /usr/lib/llvm-14
           sudo rm -rf /usr/lib/llvm-13
+      - name: Check FFI bindings
+        run: |
+          ./.github/scripts/ci-ffi.sh
       - name: Build Julia (Debug)
         run: |
           ./.github/scripts/ci-build.sh debug ${{ inputs.gc_plan }} ${{ inputs.moving }}
+      - name: Print Julia version info
+        run: |
+          JULIA_PATH=$(realpath $(dirname "$0"))/../mmtk-julia/mmtk-julia/vm/julia
+          $JULIA_PATH/julia --project=. -e "using InteractiveUtils; versioninfo()"
       - name: Style check
         run: |
           ./.github/scripts/ci-style.sh
 
+  check-ffi:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v2
+      - name: Setup environments
+        run: |
+          ./.github/scripts/ci-checkout.sh ${{ inputs.julia_repo }} ${{ inputs.julia_ref }}
+          ./.github/scripts/ci-setup.sh
+          # removing these as they cause a conflict within bindgen
+          sudo rm -rf /usr/lib/llvm-14
+          sudo rm -rf /usr/lib/llvm-13
+      - name: Regenerate FFI bindings
+        run: |
+          ./.github/scripts/ci-ffi.sh
+      - run: |
+          MMTK_JULIA_DIR=$(realpath $(dirname "$0"))/../mmtk-julia/mmtk-julia
+          if ! git diff --exit-code $MMTK_JULIA_DIR/mmtk/src/julia_types.rs; then
+            echo "Rust FFI bindings in \`julia_types.rs\` are outdated. Run \`make regen-bindgen-ffi\` from the mmtk-julia directory and make sure to include the updated file in the pull request."
+            exit 1
+          fi
+
   build-test-other:
     runs-on: ubuntu-22.04
     timeout-minutes: 360
     steps:
       - uses: actions/checkout@v2
       - name: Setup environments
         run: |
-          ./.github/scripts/ci-checkout.sh
+          ./.github/scripts/ci-checkout.sh ${{ inputs.julia_repo }} ${{ inputs.julia_ref }}
           ./.github/scripts/ci-setup.sh
           # removing these as they cause a conflict within bindgen
           sudo rm -rf /usr/lib/llvm-14  
@@ -47,6 +103,10 @@ jobs:
       - name: Build Julia (Release)
         run: |
           ./.github/scripts/ci-build.sh release ${{ inputs.gc_plan }} ${{ inputs.moving }}
+      - name: Print Julia version info
+        run: |
+          JULIA_PATH=$(realpath $(dirname "$0"))/../mmtk-julia/mmtk-julia/vm/julia
+          $JULIA_PATH/julia --project=. -e "using InteractiveUtils; versioninfo()"
       - name: Test Julia
         run: |
           ./.github/scripts/ci-test-other.sh
@@ -58,7 +118,7 @@ jobs:
       - uses: actions/checkout@v2
       - name: Setup environments
         run: |
-          ./.github/scripts/ci-checkout.sh
+          ./.github/scripts/ci-checkout.sh ${{ inputs.julia_repo }} ${{ inputs.julia_ref }}
           ./.github/scripts/ci-setup.sh
           # removing these as they cause a conflict within bindgen
           sudo rm -rf /usr/lib/llvm-14  
@@ -69,6 +129,10 @@ jobs:
       - name: Build Julia (Release)
         run: |
           ./.github/scripts/ci-build.sh release ${{ inputs.gc_plan }} ${{ inputs.moving }}
+      - name: Print Julia version info
+        run: |
+          JULIA_PATH=$(realpath $(dirname "$0"))/../mmtk-julia/mmtk-julia/vm/julia
+          $JULIA_PATH/julia --project=. -e "using InteractiveUtils; versioninfo()"
       - name: Test Julia
         run: |
           ./.github/scripts/ci-test-stdlib.sh
@@ -80,14 +144,18 @@ jobs:
       - uses: actions/checkout@v2
       - name: Setup environments
         run: |
-          ./.github/scripts/ci-checkout.sh
+          ./.github/scripts/ci-checkout.sh ${{ inputs.julia_repo }} ${{ inputs.julia_ref }}
           ./.github/scripts/ci-setup.sh
           # removing these as they cause a conflict within bindgen
           sudo rm -rf /usr/lib/llvm-14  
           sudo rm -rf /usr/lib/llvm-13
       - name: Build Julia (Release)
         run: |
           ./.github/scripts/ci-build.sh release ${{ inputs.gc_plan }} ${{ inputs.moving }}
+      - name: Print Julia version info
+        run: |
+          JULIA_PATH=$(realpath $(dirname "$0"))/../mmtk-julia/mmtk-julia/vm/julia
+          $JULIA_PATH/julia --project=. -e "using InteractiveUtils; versioninfo()"
       - name: Test Julia
         run: |
           ./.github/scripts/ci-test-LinearAlgebra.sh

.github/workflows/ci.yml

@@ -14,7 +14,28 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  julia-version:
+    runs-on: ubuntu-latest
+    env:
+      JULIA_REPO_DEFAULT: JuliaLang/julia
+      JULIA_REF_DEFAULT: master
+    outputs:
+      julia_repo: ${{ steps.print.outputs.julia_repo }}
+      julia_ref: ${{ steps.print.outputs.julia_ref }}
+    steps:
+      - name: Check Julia revisions
+        uses: qinsoon/[email protected]
+        with:
+          pull_request: ${{ github.event.pull_request.number }}
+          token: ${{ secrets.GITHUB_TOKEN }}
+          default_env: 'JULIA_REPO=${{ env.JULIA_REPO_DEFAULT }},JULIA_REF=${{ env.JULIA_REF_DEFAULT }}'
+      - id: print
+        run: |
+          echo "julia_repo=${{ env.JULIA_REPO }}" >> $GITHUB_OUTPUT
+          echo "julia_ref=${{ env.JULIA_REF }}" >> $GITHUB_OUTPUT
+
   binding-tests:
+    needs: julia-version
     strategy:
       fail-fast: false
       matrix:
@@ -23,4 +44,6 @@ jobs:
     uses: ./.github/workflows/binding-tests.yml
     with:
       gc_plan: ${{ matrix.gc_plan }}
-      moving: ${{ matrix.moving }}
+      moving: ${{ matrix.moving }}
+      julia_repo: ${{ needs.julia-version.outputs.julia_repo }}
+      julia_ref: ${{ needs.julia-version.outputs.julia_ref }}