[JSC] Ahead-of-time compilation

.github/workflows/ci.yml

@@ -53,7 +53,7 @@ jobs:
       # The release the lanes are published as.
       tag: ${{ steps.plan.outputs.tag }}
       # Matrices ({"include": [lane, ...]}) of the lanes that are only built, of the lanes that are also tested, and
-      # of the test job; and the labels of all lanes.
+      # of the test job; and the names of the release's assets: the labels of all lanes, and `SOURCES` of lanes.mjs.
       build: ${{ steps.plan.outputs.build }}
       build_tested: ${{ steps.plan.outputs.build_tested }}
       test: ${{ steps.plan.outputs.test }}
@@ -248,6 +248,10 @@ jobs:
           REPOSITORY: ${{ github.repository }}
           IMAGE: ${{ matrix.image }}
           TEST: ${{ matrix.test }}
+          PROGRAMS: ${{ matrix.programs }}
+          AOT: ${{ matrix.aot }}
+          DEFINES: ${{ matrix.defines }}
+          SOURCES: ${{ matrix.sources }}
           GH_TOKEN: ${{ github.token }}
           GH_REPO: ${{ github.repository }}
           TAG: ${{ needs.plan.outputs.tag }}
@@ -262,12 +266,36 @@ jobs:
           node .github/scripts/lanes.mjs build "$LABEL" --output "$RUNNER_TEMP/bun-webkit" --base-image "$IMAGE"
           cd "$RUNNER_TEMP"
           echo "#define BUN_WEBKIT_VERSION \"$BUILD_REF\"" >> bun-webkit/include/cmakeconfig.h
+          # ENABLE_AOT stands on the lane's command line alone (lanes.mjs), and it changes the layout of classes in the
+          # headers. Whoever links these libraries and says nothing reads the headers as they were compiled here; bun
+          # compares BUN_WEBKIT_ENABLE_AOT with what it was told to build.
+          printf '#define BUN_WEBKIT_ENABLE_AOT %s\n#if !defined(ENABLE_AOT)\n#define ENABLE_AOT %s\n#endif\n' "$AOT" "$AOT" >> bun-webkit/include/cmakeconfig.h
+          # So do these (`defines` in lanes.mjs).
+          for define in $DEFINES; do printf '#if !defined(%s)\n#define %s %s\n#endif\n' "${define%%=*}" "${define%%=*}" "${define#*=}" >> bun-webkit/include/cmakeconfig.h; done
           echo "{ \"name\": \"$LABEL\", \"version\": \"0.0.1-$BUILD_REF\", \"os\": [\"$PACKAGE_OS\"], \"cpu\": [\"$PACKAGE_CPU\"], \"repository\": \"https://github.com/$REPOSITORY\" }" > bun-webkit/package.json
           rm -rf bun-webkit/lib/*.so bun-webkit/lib/*.so.*
+          # Which programs this lane ships: `programs` in lanes.mjs.
+          for program in jsc testFFI testaot; do
+            case " $PROGRAMS " in *" $program "*) ;; *) rm -f "bun-webkit/bin/$program" "bun-webkit/bin/$program.exe" ;; esac
+          done
+          for objects in jsc-objects testaot-objects; do
+            case " $PROGRAMS " in *" $objects "*) ls -l bun-webkit/lib/*"$objects".* ;; *) rm -f "bun-webkit/lib/lib$objects.a" "bun-webkit/lib/$objects.lib" ;; esac
+          done
+          # For `test` alone: not part of what ships.
+          rm -rf test-bin && if [ -d bun-webkit/test-bin ]; then mv bun-webkit/test-bin test-bin; fi
           tar -czf bun-webkit.tar.gz bun-webkit
+          du -sh bun-webkit/lib bun-webkit/bin/* && ls -l bun-webkit.tar.gz
+          # `SOURCES` in lanes.mjs: one lane packs it.
+          if [ -n "$SOURCES" ]; then
+            tar -czf "$SOURCES.tar.gz" --exclude=bun-webkit/Source/JavaScriptCore/aot/tests bun-webkit/Source/JavaScriptCore/aot
+            tar -tzf "$SOURCES.tar.gz" | grep -c . && ls -l "$SOURCES.tar.gz"
+          fi
           # What `test` runs.
-          if [ "$TEST" = true ]; then tar -cf jsc-shell.tar bun-webkit/bin; fi
-          rm -rf bun-webkit
+          if [ "$TEST" = true ]; then
+            if [ -d test-bin ]; then mv test-bin/* bun-webkit/bin/; fi
+            tar -cf jsc-shell.tar bun-webkit/bin
+          fi
+          rm -rf bun-webkit test-bin
       - name: Upload the jsc shell for the tests
         if: matrix.test
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
@@ -282,15 +310,19 @@ jobs:
           GH_REPO: ${{ github.repository }}
           TAG: ${{ needs.plan.outputs.tag }}
           LABEL: ${{ matrix.label }}
+          SOURCES: ${{ matrix.sources }}
           DIR: ${{ runner.temp }}
         run: |
           set -euo pipefail
           mv "$DIR/bun-webkit.tar.gz" "$DIR/$LABEL.tar.gz"
-          for attempt in 1 2 3 4 5; do
-            gh release upload "$TAG" "$DIR/$LABEL.tar.gz" --clobber && { echo "uploaded $LABEL"; exit 0; }
-            echo "upload of $LABEL failed (attempt $attempt)"; sleep $((attempt * 10))
+          for asset in "$LABEL" $SOURCES; do
+            uploaded=false
+            for attempt in 1 2 3 4 5; do
+              gh release upload "$TAG" "$DIR/$asset.tar.gz" --clobber && { echo "uploaded $asset"; uploaded=true; break; }
+              echo "upload of $asset failed (attempt $attempt)"; sleep $((attempt * 10))
+            done
+            [ "$uploaded" = true ] || exit 1
           done
-          exit 1
 
   build-tested:
     name: ${{ matrix.label }}
@@ -326,7 +358,7 @@ jobs:
       fail-fast: false
       matrix: ${{ fromJSON(needs.plan.outputs.test) }}
     # A quick lane is one tested on a runner with few cores (lanes.mjs); there is none at the moment.
-    timeout-minutes: ${{ matrix.quick && 300 || 90 }}
+    timeout-minutes: ${{ matrix.quick && 315 || 105 }}
     env:
       QUICK: ${{ matrix.quick }}
       LABEL: ${{ matrix.label }}
@@ -387,7 +419,8 @@ jobs:
           cd "$RUNNER_TEMP"
           tar -xf jsc-shell.tar
           ls -la bun-webkit/bin
-      # testFFI and the JavaScriptCore tests each run whatever the other did, and either failing fails the job.
+      # testFFI, TestWTF and the JavaScriptCore tests each run whatever the others did, and any of them failing fails
+      # the job.
       - name: testFFI
         if: &have_jsc ${{ !cancelled() && steps.download.outcome == 'success' }}
         env: &test_env
@@ -402,6 +435,15 @@ jobs:
           JSCTEST_memoryLimit: 4294967296
         run: |
           "$RUNNER_TEMP/bun-webkit/bin/testFFI"
+      - name: TestWTF
+        if: *have_jsc
+        env: *test_env
+        run: |
+          # Line by line: what a program that crashes still holds in its buffer is lost, and with it the name of the test.
+          # Not StackTraceTest: the lanes are compiled without unwind tables, so backtrace() sees one frame.
+          program="$RUNNER_TEMP/bun-webkit/bin/TestWTF"
+          if [ ! -x "$program" ]; then echo "this lane has no TestWTF"; exit 0; fi
+          stdbuf -oL "$program" --gtest_filter=-StackTraceTest.*
       - name: run-javascriptcore-tests
         if: *have_jsc
         env: *test_env
@@ -430,13 +472,16 @@ jobs:
           esac
           # --quick on the small runners: see lanes.mjs.
           if [ "$QUICK" = true ]; then platform+=(--quick); fi
-          # --root: <root>/bin/jsc. The C++ test binaries (testapi, testmasm, testb3, testair, testdfg) are not built
-          # by the release Dockerfiles, hence the --no-test*.
+          for program in testair testb3 testmasm; do [ -x "$root/bin/$program" ] || platform+=("--no-$program"); done
+          # A lane built with ENABLE_AOT=0 ships no testaot (`programs` in lanes.mjs).
+          [ -x "$root/bin/testaot" ] || platform+=(--without-aot)
+          # --root: <root>/bin/jsc. Of the C++ test programs a lane has testaot, and testair, testb3 and testmasm where lanes.mjs
+          # says so (testPrograms); the others are not built, hence the --no-test*.
           # --memory-limited: what upstream's Linux bots pass; skips the tests marked as needing a lot of memory.
           # --no-fail-fast: run everything, report every failure.
           perl Tools/Scripts/run-javascriptcore-tests \
             --no-build --root="$root" --release --jsc-only ${platform[@]+"${platform[@]}"} \
-            --no-testmasm --no-testair --no-testb3 --no-testdfg --no-testapi --no-testwasmdebugger --no-testlibjsctools \
+            --no-testdfg --no-testapi --no-testwasmdebugger --no-testlibjsctools \
             --no-fail-fast --memory-limited \
             --json-output="$results" 2>&1 | tee "$RUNNER_TEMP/jsc-tests.log" | awk '
               # The full output (a line per test run, plus what each test prints) is jsc-tests.log in the artifact; it
@@ -458,13 +503,24 @@ jobs:
               }'
           status=${PIPESTATUS[0]}
           echo "run-javascriptcore-tests exited with $status"
+          # AOTNativeTestsOfHostEntries.inc: bun's CI runs them.
+          left_out=$(grep -a -E '^[0-9]+ passed, [0-9]+ failed, [0-9]+ todo' "$RUNNER_TEMP/jsc-tests.log" | tail -n 1 | grep -o -E '[0-9]+ left out' || echo "0 left out")
+          if [ -x "$root/bin/testaot" ] && [ "$left_out" != "33 left out" ]; then echo "::error::testaot: $left_out, and 33 are expected"; status=1; fi
           # What each failing run printed (exception, stack, exit code), grouped under its name: the part of the full
           # log that is needed to fix a test.
           awk '
             FNR == NR { if ($0 ~ /^FAIL: /) { name = $2; failing[name] = 1; order[++n] = name } next }
             { i = index($0, ": "); if (i > 0) { name = substr($0, 1, i - 1); if (name in failing) out[name] = out[name] substr($0, i + 2) "\n" } }
             END { for (k = 1; k <= n; k++) { print "=== " order[k]; printf "%s\n", out[order[k]] } }
           ' "$RUNNER_TEMP/jsc-tests.log" "$RUNNER_TEMP/jsc-tests.log" > "$RUNNER_TEMP/jsc-failures.log" || true
+          # Each of testaot's failing tests once more, alone, with all that it prints.
+          grep -a '^FAIL ' "$RUNNER_TEMP/jsc-tests.log" | sed -E 's/^FAIL //; s/(\.ir):.*/\1/; s/: .*//' | sort -u | while read -r name; do
+            echo "=== testaot $name"
+            case "$name" in
+              *.ir) timeout 300 "$root/bin/testaot" run "$name" --show-output ;;
+              *) timeout 300 "$root/bin/testaot" native "$name" ;;
+            esac
+          done >> "$RUNNER_TEMP/jsc-failures.log" 2>&1
           exit "$status"
       - name: Summary
         if: always()
@@ -501,7 +557,9 @@ jobs:
   release:
     name: release
     # Runs whatever happened to the builds (also when the run was cancelled), so the draft never outlives the run: it
-    # is published when every build job succeeded and every listed asset is on it, and deleted otherwise.
+    # is published when every build job succeeded and every listed asset is on it. A pull request's is also published
+    # when lanes failed, with the lanes that were built, and the job fails: whoever tries the preview is not held up by
+    # a lane that they do not use. Otherwise, and when the run was cancelled, it is deleted.
     if: always() && needs.plan.result == 'success'
     runs-on: ubuntu-latest
     timeout-minutes: 15
@@ -510,7 +568,8 @@ jobs:
       - build
       - build-tested
     steps:
-      # Every lane uploads <label>.tar.gz. The lanes are the release: a label missing from it fails the job.
+      # Every lane uploads <label>.tar.gz, and one of them `SOURCES` of lanes.mjs too. Those are the release: one missing
+      # from it fails the job.
       - name: Publish or delete the draft
         env:
           GH_TOKEN: ${{ github.token }}
@@ -523,21 +582,22 @@ jobs:
           set -euo pipefail
           ok=true
           failed=$(jq -r 'to_entries[] | select(.value.result != "success") | "\(.key): \(.value.result)"' <<< "$NEEDS")
+          cancelled=$(jq -r 'to_entries[] | select(.value.result == "cancelled") | .key' <<< "$NEEDS")
           if [ -n "$failed" ]; then
             echo "jobs that did not succeed:"; echo "$failed"; ok=false
-          else
-            jq -r '.[]' <<< "$LABELS" | sort > expected.txt
-            gh api "repos/$GH_REPO/releases/$RELEASE_ID" --jq '.assets[].name' | sed 's/\.tar\.gz$//' | sort > uploaded.txt
-            missing=$(comm -23 expected.txt uploaded.txt)
-            if [ -n "$missing" ]; then
-              echo "assets missing from $TAG:"; echo "$missing"; ok=false
-            fi
           fi
-          if [ "$ok" = true ]; then
+          jq -r '.[]' <<< "$LABELS" | sort > expected.txt
+          gh api "repos/$GH_REPO/releases/$RELEASE_ID" --jq '.assets[].name' | sed 's/\.tar\.gz$//' | sort > uploaded.txt || : > uploaded.txt
+          missing=$(comm -23 expected.txt uploaded.txt)
+          if [ -n "$missing" ]; then
+            echo "assets missing from $TAG:"; echo "$missing"; ok=false
+          fi
+          if [ "$ok" = true ] || { [ -n "$PR" ] && [ -z "$cancelled" ] && [ -s uploaded.txt ]; }; then
             url=$(gh api -X PATCH "repos/$GH_REPO/releases/$RELEASE_ID" -F draft=false --jq .html_url)
             echo "published $TAG with $(wc -l < uploaded.txt) assets: $url"
             # A pull request gets a comment linking its release, edited in place by the next build.
             [ -z "$PR" ] || gh pr comment "$PR" --edit-last --create-if-none --body "Preview build of ${REF:0:8}: [\`$TAG\`]($url)" || echo "::warning::could not comment on #$PR"
+            [ "$ok" = true ] || exit 1
             exit 0
           fi
           # Only the draft this run created is deleted: not a published release (a re-run of a run that succeeded),