diff --git a/.github/workflows/unit-test-on-pull-request.yml b/.github/workflows/unit-test-on-pull-request.yml index 50563e0a7..a8a3446a1 100644 --- a/.github/workflows/unit-test-on-pull-request.yml +++ b/.github/workflows/unit-test-on-pull-request.yml @@ -142,7 +142,7 @@ jobs: integration-tests: name: Integration tests (v${{ matrix.kernel }} ${{ matrix.target_arch }}) - runs-on: ubuntu-24.04 + runs-on: ${{ matrix.target_arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} needs: build-integration-test-binaries timeout-minutes: 10 strategy: @@ -233,10 +233,50 @@ jobs: uname -a sudo go test ./interpreter/... -v -run "TestIntegration/(node-local-nightly|node-latest)" + build-distro-qemu-initramfs: + name: Build distro QEMU initramfs (${{ matrix.target_arch }}) + runs-on: ${{ matrix.target_arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} + timeout-minutes: 10 + strategy: + matrix: + target_arch: [amd64, arm64] + steps: + - name: Clone code + uses: actions/checkout@v4 + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache-dependency-path: go.sum + - name: Install dependencies + run: | + sudo apt-get update -y + sudo apt-get install -y --no-install-recommends busybox-static systemtap-sdt-dev + - name: Cache parcagpu library + uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 + with: + path: test/distro-qemu/parcagpu-lib + key: parcagpu-${{ matrix.target_arch }} + - name: Build initramfs + run: | + cd test/distro-qemu + case "${{ matrix.target_arch }}" in + amd64) export QEMU_ARCH=x86_64;; + arm64) export QEMU_ARCH=aarch64;; + *) echo >&2 "bug: bad arch selected"; exit 1;; + esac + ./build-initramfs.sh initramfs-${{ matrix.target_arch }}.gz + - name: Upload initramfs + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: distro-qemu-initramfs-${{ matrix.target_arch }} + path: test/distro-qemu/initramfs-${{ matrix.target_arch }}.gz + distro-qemu-tests: name: Distro QEMU tests (${{ matrix.kernel }} ${{ matrix.target_arch }}) - runs-on: ubuntu-24.04 - timeout-minutes: 15 + runs-on: ${{ matrix.target_arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} + needs: build-distro-qemu-initramfs + timeout-minutes: 10 strategy: matrix: include: @@ -261,13 +301,6 @@ jobs: steps: - name: Clone code uses: actions/checkout@v4 - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version-file: go.mod - cache-dependency-path: go.sum - - name: Set up environment - uses: ./.github/workflows/env - name: Install dependencies run: | sudo tee /etc/apt/sources.list.d/ubuntu.sources < /dev/null @@ -294,17 +327,16 @@ jobs: arm64) sudo apt-get -y install qemu-system-arm ipxe-qemu gcc-aarch64-linux-gnu libc6-arm64-cross libc6:arm64 binutils-aarch64-linux-gnu musl-dev:arm64 systemtap-sdt-dev:arm64;; *) echo >&2 "bug: bad arch selected"; exit 1;; esac - sudo apt-get install -y debootstrap systemtap-sdt-dev - - name: Get parcagpu image digest - id: parcagpu-digest - run: | - digest=$(docker buildx imagetools inspect ghcr.io/parca-dev/parcagpu:latest --format '{{.Digest}}' 2>/dev/null || echo "unknown") - echo "digest=${digest}" >> "$GITHUB_OUTPUT" - - name: Cache parcagpu library + - name: Fetch initramfs + uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + with: + name: distro-qemu-initramfs-${{ matrix.target_arch }} + path: test/distro-qemu + - name: Cache kernel image uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 with: - path: test/distro-qemu/parcagpu-lib - key: parcagpu-${{ matrix.target_arch }}-${{ steps.parcagpu-digest.outputs.digest }} + path: test/distro-qemu/ci-kernels/${{ matrix.kernel }} + key: ci-kernel-${{ matrix.target_arch }}-${{ matrix.kernel }} - name: Download kernel run: | cd test/distro-qemu @@ -314,7 +346,7 @@ jobs: *) echo >&2 "bug: bad arch selected"; exit 1;; esac ./download-kernel.sh ${{ matrix.kernel }} - - name: Run Full Distro tests in QEMU + - name: Run tests in QEMU run: | cd test/distro-qemu case "${{ matrix.target_arch }}" in @@ -322,4 +354,4 @@ jobs: arm64) export QEMU_ARCH=aarch64;; *) echo >&2 "bug: bad arch selected"; exit 1;; esac - ./build-and-run.sh ${{ matrix.kernel }} + ./run-qemu.sh ${{ matrix.kernel }} initramfs-${{ matrix.target_arch }}.gz diff --git a/support/run-tests.sh b/support/run-tests.sh index cced106bb..90058f883 100755 --- a/support/run-tests.sh +++ b/support/run-tests.sh @@ -41,8 +41,7 @@ done < <(find . -name '*.test' -print0) additionalQemuArgs="" -supportKVM=$(grep -E 'vmx|svm' /proc/cpuinfo || true) -if [ ! "$supportKVM" ] && [ "$qemu_arch" = "$(uname -m)" ]; then +if [ -e /dev/kvm ] && [ "$qemu_arch" = "$(uname -m)" ]; then additionalQemuArgs="-enable-kvm" fi diff --git a/test/distro-qemu/build-and-run.sh b/test/distro-qemu/build-and-run.sh index f746e1b71..612eeb613 100755 --- a/test/distro-qemu/build-and-run.sh +++ b/test/distro-qemu/build-and-run.sh @@ -1,314 +1,18 @@ #!/bin/bash set -ex -# Configuration +# Convenience wrapper that builds the initramfs and runs QEMU in one step. +# In CI, these are split into separate jobs for efficiency. +# +# Usage: QEMU_ARCH=x86_64 ./build-and-run.sh + KERNEL_VERSION="${1:-5.10.217}" -# Auto-detect host architecture if QEMU_ARCH not set. -case "$(uname -m)" in - x86_64) _default_arch="x86_64" ;; - aarch64) _default_arch="aarch64" ;; - *) _default_arch="x86_64" ;; -esac -QEMU_ARCH="${QEMU_ARCH:-$_default_arch}" -DISTRO="${DISTRO:-ubuntu}" # debian or ubuntu -RELEASE="${RELEASE:-jammy}" # jammy/noble for ubuntu (with USDT probes), bullseye for debian -ROOTFS_DIR=$(mktemp -d /tmp/distro-qemu-rootfs.XXXXXX) -OUTPUT_DIR=$(mktemp -d /tmp/distro-qemu-output.XXXXXX) -KERN_DIR="${KERN_DIR:-ci-kernels}" -PARCAGPU_DIR="${PARCAGPU_DIR:-parcagpu-lib}" -CACHE_DIR="${CACHE_DIR:-/tmp/debootstrap-cache}" +INITRAMFS=$(mktemp /tmp/distro-qemu-initramfs.XXXXXX.gz) cleanup() { - if [ -d "$ROOTFS_DIR" ]; then - findmnt -o TARGET -n -l | grep "^${ROOTFS_DIR}" | sort -r | while read -r mp; do - sudo umount "$mp" || sudo umount -l "$mp" || true - done - sudo rm -rf "$ROOTFS_DIR" - fi - rm -rf "$OUTPUT_DIR" + rm -f "$INITRAMFS" } trap cleanup EXIT -# Download parcagpu library -QEMU_ARCH="${QEMU_ARCH}" PARCAGPU_DIR="${PARCAGPU_DIR}" ./download-parcagpu.sh - -echo "Building rootfs with $DISTRO $RELEASE..." - -mkdir -p "$CACHE_DIR" - -# Determine debootstrap architecture -DEBOOTSTRAP_ARCH="amd64" -case "$QEMU_ARCH" in - x86_64) - DEBOOTSTRAP_ARCH="amd64" - ;; - aarch64) - DEBOOTSTRAP_ARCH="arm64" - ;; - *) - echo "Unsupported QEMU_ARCH: $QEMU_ARCH" - exit 1 - ;; -esac - -GOARCH=$DEBOOTSTRAP_ARCH - -# Choose mirror based on distro and architecture -if [[ "$DISTRO" == "ubuntu" ]]; then - # Ubuntu ARM64 packages are on ports.ubuntu.com - if [[ "$DEBOOTSTRAP_ARCH" == "arm64" ]]; then - MIRROR="http://ports.ubuntu.com/ubuntu-ports/" - else - MIRROR="https://archive.ubuntu.com/ubuntu/" - fi -else - MIRROR="http://deb.debian.org/debian/" -fi - -# Create minimal rootfs with debootstrap (requires sudo for chroot operations) -echo "Running debootstrap to create $DISTRO $RELEASE rootfs for $DEBOOTSTRAP_ARCH..." -if ! sudo debootstrap --variant=minbase \ - --arch="$DEBOOTSTRAP_ARCH" \ - --cache-dir="$CACHE_DIR" \ - --foreign \ - --include=libstdc++6 \ - "$RELEASE" "$ROOTFS_DIR" "$MIRROR" ; then - echo "Debootstrap failed, log follows." - cat "$ROOTFS_DIR/debootstrap/debootstrap.log" - exit 1 -fi - -# Change ownership of rootfs to current user to avoid needing sudo for subsequent operations -sudo chown -R "$(id -u):$(id -g)" "$ROOTFS_DIR" - -# Build the test binary (must be dynamic for dlopen to work) -echo "Building test binary for $DISTRO $RELEASE $DEBOOTSTRAP_ARCH..." - -REPO_ROOT="$(cd ../.. && pwd)" -TEST_PKGS="./interpreter/rtld ./support/usdt/test ./test/cudaverify" - -# For cross-compilation or Ubuntu jammy/noble, local build works (host has compatible or newer glibc) -# For older distros, would need Docker build (disabled by default for speed) -if [[ "${USE_DOCKER}" == "1" ]] && command -v docker &> /dev/null; then - # Determine base image - if [[ "$DISTRO" == "ubuntu" ]]; then - BASE_IMAGE="ubuntu:${RELEASE}" - else - BASE_IMAGE="debian:${RELEASE}" - fi - - # Build in container to match target glibc (slow, downloads Go) - echo "Using Docker to build with matching glibc version..." - docker run --rm \ - -v "${REPO_ROOT}:/workspace" \ - -v "${OUTPUT_DIR}:/output" \ - -w /workspace \ - --platform "linux/${DEBOOTSTRAP_ARCH}" \ - "$BASE_IMAGE" \ - bash -c "apt-get update -qq && apt-get install -y -qq wget libc6-dev gcc > /dev/null 2>&1 && \ - wget -q https://go.dev/dl/go1.24.7.linux-${GOARCH}.tar.gz && \ - tar -C /usr/local -xzf go1.24.7.linux-${GOARCH}.tar.gz && \ - export PATH=/usr/local/go/bin:\$PATH && \ - CGO_ENABLED=1 go test -c -o /output/ ${TEST_PKGS}" -else - # Local build with cross-compilation if needed - echo "Building locally for ${GOARCH}..." - ( - cd "${REPO_ROOT}" - if [ "$GOARCH" = "arm64" ]; then - CGO_ENABLED=1 GOARCH=${GOARCH} CC=aarch64-linux-gnu-gcc \ - go test -c -o "${OUTPUT_DIR}/" ${TEST_PKGS} - else - CGO_ENABLED=1 GOARCH=${GOARCH} \ - go test -c -o "${OUTPUT_DIR}/" ${TEST_PKGS} - fi - ) -fi - -# Copy test binaries and parcagpu .so into rootfs -cp "${OUTPUT_DIR}"/*.test "$ROOTFS_DIR/" -cp "${PARCAGPU_DIR}/libparcagpucupti.so" "$ROOTFS_DIR/" - -# Copy stub libcupti .so into the RUNPATH (/usr/local/cuda/lib64) so the -# dynamic linker resolves the DT_NEEDED entry without a real CUDA install. -mkdir -p "$ROOTFS_DIR/usr/local/cuda/lib64" -for stub in "${PARCAGPU_DIR}"/libcupti.so*; do - [ -f "$stub" ] && cp "$stub" "$ROOTFS_DIR/usr/local/cuda/lib64/" -done - -# Copy libstdc++ into the RUNPATH so the dynamic linker finds it. -# With --foreign debootstrap the .deb is downloaded but not extracted, so we -# pull the .so directly from the .deb archive. -LIBSTDCXX_DEB=$(find "$ROOTFS_DIR" -name 'libstdc++6_*.deb' -type f | head -1) -if [ -n "$LIBSTDCXX_DEB" ]; then - EXTRACT_TMP=$(mktemp -d) - dpkg-deb -x "$LIBSTDCXX_DEB" "$EXTRACT_TMP" - # Copy the real .so file (not the symlink) and create the soname symlink. - LIBSTDCXX_REAL=$(find "$EXTRACT_TMP" -name 'libstdc++.so.6.*' ! -name '*.py' -type f | head -1) - if [ -n "$LIBSTDCXX_REAL" ]; then - cp "$LIBSTDCXX_REAL" "$ROOTFS_DIR/usr/local/cuda/lib64/" - ln -sf "$(basename "$LIBSTDCXX_REAL")" "$ROOTFS_DIR/usr/local/cuda/lib64/libstdc++.so.6" - echo "Copied $(basename "$LIBSTDCXX_REAL") + symlink to RUNPATH from deb" - fi - rm -rf "$EXTRACT_TMP" -else - # Fallback: check if already extracted (non-foreign debootstrap) - LIBSTDCXX_REAL=$(find "$ROOTFS_DIR" -name 'libstdc++.so.6.*' ! -name '*.py' -type f | head -1) - if [ -n "$LIBSTDCXX_REAL" ]; then - cp "$LIBSTDCXX_REAL" "$ROOTFS_DIR/usr/local/cuda/lib64/" - ln -sf "$(basename "$LIBSTDCXX_REAL")" "$ROOTFS_DIR/usr/local/cuda/lib64/libstdc++.so.6" - echo "Copied $(basename "$LIBSTDCXX_REAL") + symlink to RUNPATH" - fi -fi - -# List dynamic dependencies for debugging -echo "Test binary dependencies:" -ldd "${OUTPUT_DIR}/rtld.test" || true - -# Create init script -cat << 'EOF' > "$ROOTFS_DIR/init" -#!/bin/sh -echo "===== Test Environment =====" -echo "Kernel: $(uname -r)" -echo "Hostname: $(hostname)" - -# Find and display ld.so info -LDSO=$(find /lib* /usr/lib* -name 'ld-linux*' -o -name 'ld-*.so*' 2>/dev/null | head -1) -echo "ld.so location: $LDSO" -if [ -n "$LDSO" ]; then - echo "ld.so version: $($LDSO --version | head -1)" -fi - -# Find libm for dlopen test -LIBM=$(find /lib* /usr/lib* -name 'libm.so*' 2>/dev/null | head -1) -echo "libm.so location: $LIBM" - -echo "=================================" - -# Mount required filesystems -mount -t proc proc /proc 2>/dev/null || true -mount -t sysfs sys /sys 2>/dev/null || true -mount -t debugfs debugfs /sys/kernel/debug 2>/dev/null || true - -# Rebuild ld.so cache so the linker finds libraries in multiarch paths -# (e.g. /usr/lib/aarch64-linux-gnu/libstdc++.so.6). -ldconfig 2>/dev/null || true - -# Enable debug logging -export DEBUG_TEST=1 - -# Run the tests -echo "" -/rtld.test -test.v && /test.test -test.v && /cudaverify.test -test.v -so-path=/libparcagpucupti.so -RESULT=$? - -if [ $RESULT -eq 0 ]; then - echo "" - echo "===== TEST PASSED =====" -elif [ $RESULT -eq 137 ] || [ $RESULT -eq 124 ]; then - echo "" - echo "===== TEST TIMED OUT =====" -else - echo "" - echo "===== BPF dmesg =====" - dmesg | tail -60 - echo "===== TEST FAILED (exit code: $RESULT) =====" -fi - -# Give time to see output before shutdown -sleep 1 - -# Try to cleanly shutdown QEMU -# The sysrq 'o' trigger will power off the system -echo o > /proc/sysrq-trigger 2>/dev/null - -# If sysrq doesn't work, force halt -sleep 1 -poweroff -f 2>/dev/null || halt -f -EOF -chmod +x "$ROOTFS_DIR/init" - -# Create initramfs -echo "Creating initramfs..." -(cd "$ROOTFS_DIR" && find . | cpio -o -H newc | gzip > "$OUTPUT_DIR/initramfs.gz") - -echo "Rootfs created: $OUTPUT_DIR/initramfs.gz ($(du -h $OUTPUT_DIR/initramfs.gz | cut -f1))" - -# Check if kernel exists -if [[ ! -f "${KERN_DIR}/${KERNEL_VERSION}/vmlinuz" ]]; then - echo "" - echo "ERROR: Kernel ${KERNEL_VERSION} not found at ${KERN_DIR}/${KERNEL_VERSION}/vmlinuz" - echo "" - echo "To download kernel images:" - echo " mkdir -p ci-kernels" - echo " docker pull ghcr.io/cilium/ci-kernels:${KERNEL_VERSION}" - echo " docker create --name kernel-extract ghcr.io/cilium/ci-kernels:${KERNEL_VERSION}" - echo " docker cp kernel-extract:/boot ci-kernels/${KERNEL_VERSION}" - echo " docker rm kernel-extract" - echo "" - exit 1 -fi - -# Use sudo if /dev/kvm isn't accessible by the current user -sudo="" -if [[ ! -r /dev/kvm || ! -w /dev/kvm ]]; then - sudo="sudo" -fi - -# Determine additional QEMU args based on architecture -additionalQemuArgs="" -supportKVM=$(grep -E 'vmx|svm' /proc/cpuinfo || true) -if [ "$supportKVM" ] && [ "$QEMU_ARCH" = "$(uname -m)" ]; then - additionalQemuArgs="-enable-kvm" -fi - -case "$QEMU_ARCH" in - x86_64) - CONSOLE_ARG="console=ttyS0" - ;; - aarch64) - additionalQemuArgs+=" -machine virt -cpu max" - CONSOLE_ARG="console=ttyAMA0" - ;; -esac - -echo "" -echo "===== Starting QEMU with kernel ${KERNEL_VERSION} on ${QEMU_ARCH} =====" -echo "" - -# Run QEMU and capture output -QEMU_OUTPUT=$(mktemp) -${sudo} qemu-system-${QEMU_ARCH} ${additionalQemuArgs} \ - -nographic \ - -monitor none \ - -serial mon:stdio \ - -m 2G \ - -kernel "${KERN_DIR}/${KERNEL_VERSION}/vmlinuz" \ - -initrd "$OUTPUT_DIR/initramfs.gz" \ - -append "${CONSOLE_ARG} init=/init quiet loglevel=3" \ - -no-reboot \ - -display none \ - | tee "$QEMU_OUTPUT" - -# Parse output for test result -if grep -q "===== TEST PASSED =====" "$QEMU_OUTPUT"; then - rm -f "$QEMU_OUTPUT" - echo "" - echo "Test completed successfully" - exit 0 -elif grep -q "===== TEST FAILED" "$QEMU_OUTPUT"; then - rm -f "$QEMU_OUTPUT" - echo "" - echo "Test failed" - exit 1 -elif grep -q "===== TEST TIMED OUT =====" "$QEMU_OUTPUT"; then - rm -f "$QEMU_OUTPUT" - echo "" - echo "Test timed out" - exit 124 -else - rm -f "$QEMU_OUTPUT" - echo "" - echo "Could not determine test result (QEMU may have crashed)" - exit 2 -fi +./build-initramfs.sh "$INITRAMFS" +./run-qemu.sh "$KERNEL_VERSION" "$INITRAMFS" diff --git a/test/distro-qemu/build-initramfs.sh b/test/distro-qemu/build-initramfs.sh new file mode 100755 index 000000000..7bb7a3ea7 --- /dev/null +++ b/test/distro-qemu/build-initramfs.sh @@ -0,0 +1,183 @@ +#!/bin/bash +set -ex + +# Build an initramfs containing test binaries, shared libraries, and busybox. +# The resulting initramfs is arch-specific but kernel-independent. +# Must be run on the same architecture as the target (no cross-compilation). +# +# Usage: QEMU_ARCH=x86_64 ./build-initramfs.sh [output-path] + +QEMU_ARCH="${QEMU_ARCH:-$(uname -m)}" +OUTPUT="${1:-initramfs.gz}" +# Make output path absolute (cpio runs in a subshell with different cwd) +[[ "$OUTPUT" != /* ]] && OUTPUT="$(pwd)/$OUTPUT" +PARCAGPU_DIR="${PARCAGPU_DIR:-parcagpu-lib}" + +ROOTFS_DIR=$(mktemp -d /tmp/distro-qemu-rootfs.XXXXXX) +BUILD_DIR=$(mktemp -d /tmp/distro-qemu-build.XXXXXX) + +cleanup() { + rm -rf "$ROOTFS_DIR" "$BUILD_DIR" +} +trap cleanup EXIT + +timer_start() { TIMER_START=$(date +%s); } +timer_end() { echo "::> $1 took $(( $(date +%s) - TIMER_START ))s"; } + +# Determine architecture +case "$QEMU_ARCH" in + x86_64) GOARCH="amd64";; + aarch64) GOARCH="arm64";; + *) echo "Unsupported QEMU_ARCH: $QEMU_ARCH"; exit 1;; +esac + +# Download parcagpu library +timer_start +PARCAGPU_DIR="${PARCAGPU_DIR}" ./download-parcagpu.sh +timer_end "download parcagpu" + +# Build test binaries (must be dynamic for dlopen to work) +timer_start +echo "Building test binaries for $GOARCH..." +REPO_ROOT="$(cd ../.. && pwd)" +TEST_PKGS="./interpreter/rtld ./support/usdt/test ./test/cudaverify" +( + cd "${REPO_ROOT}" + CGO_ENABLED=1 GOARCH=${GOARCH} \ + go test -c -o "${BUILD_DIR}/" ${TEST_PKGS} +) +timer_end "go test -c (build test binaries)" + +# Create minimal rootfs (busybox + shared libs only) +timer_start +echo "Creating minimal rootfs..." +mkdir -p "$ROOTFS_DIR"/{bin,proc,sys,dev,tmp} + +# Install busybox for shell and basic utilities +BUSYBOX=$(command -v busybox 2>/dev/null || true) +if [ -z "$BUSYBOX" ]; then + echo "ERROR: busybox not found. Install busybox-static." + exit 1 +fi +cp "$BUSYBOX" "$ROOTFS_DIR/bin/busybox" +chmod +x "$ROOTFS_DIR/bin/busybox" +for cmd in sh mount umount dmesg poweroff halt reboot hostname uname find head tail sleep cat grep; do + ln -sf busybox "$ROOTFS_DIR/bin/$cmd" +done + +# Copy shared libraries needed by test binaries using ldd +copy_lib_deps() { + local binary="$1" + # Copy ELF interpreter + local interp + interp=$(readelf -l "$binary" 2>/dev/null | grep -oP 'Requesting program interpreter: \K[^\]]+' || true) + if [ -n "$interp" ] && [ -f "$interp" ]; then + install -Dm755 "$interp" "$ROOTFS_DIR$interp" + fi + # Copy all shared library dependencies + ldd "$binary" 2>/dev/null | while read -r line; do + lib=$(echo "$line" | grep -oP '=> \K/\S+' || true) + if [ -n "$lib" ] && [ -f "$lib" ] && [ ! -f "$ROOTFS_DIR$lib" ]; then + real=$(readlink -f "$lib") + install -Dm755 "$real" "$ROOTFS_DIR$real" + if [ "$real" != "$lib" ]; then + mkdir -p "$ROOTFS_DIR$(dirname "$lib")" + ln -sf "$real" "$ROOTFS_DIR$lib" + fi + fi + done +} + +for binary in "${BUILD_DIR}"/*.test; do + copy_lib_deps "$binary" +done + +# Ensure libm.so is available for rtld test (it dlopens libm at runtime, +# so it won't appear in ldd output) +LIBM=$(find /lib* /usr/lib* -name 'libm.so.6' -type f 2>/dev/null | head -1) +if [ -n "$LIBM" ] && [ ! -f "$ROOTFS_DIR$LIBM" ]; then + real=$(readlink -f "$LIBM") + install -Dm755 "$real" "$ROOTFS_DIR$real" + if [ "$real" != "$LIBM" ]; then + mkdir -p "$ROOTFS_DIR$(dirname "$LIBM")" + ln -sf "$real" "$ROOTFS_DIR$LIBM" + fi +fi + +# Copy test binaries and parcagpu .so into rootfs +cp "${BUILD_DIR}"/*.test "$ROOTFS_DIR/" +cp "${PARCAGPU_DIR}/libparcagpucupti.so" "$ROOTFS_DIR/" +copy_lib_deps "${PARCAGPU_DIR}/libparcagpucupti.so" + +# Copy stub libcupti .so into the RUNPATH (/usr/local/cuda/lib64) so the +# dynamic linker resolves the DT_NEEDED entry without a real CUDA install. +mkdir -p "$ROOTFS_DIR/usr/local/cuda/lib64" +for stub in "${PARCAGPU_DIR}"/libcupti.so*; do + [ -f "$stub" ] && cp "$stub" "$ROOTFS_DIR/usr/local/cuda/lib64/" +done + +# Show what we have for debugging +echo "Test binary dependencies:" +ldd "${BUILD_DIR}/rtld.test" || true +echo "" +echo "Rootfs contents:" +find "$ROOTFS_DIR" -type f -o -type l | sort +echo "" + +# Create init script +cat << 'INIT_EOF' > "$ROOTFS_DIR/init" +#!/bin/sh +export PATH=/bin + +echo "===== Test Environment =====" +echo "Kernel: $(uname -r)" + +# Mount required filesystems +mount -t proc proc /proc 2>/dev/null || true +mount -t sysfs sys /sys 2>/dev/null || true +mount -t debugfs debugfs /sys/kernel/debug 2>/dev/null || true + +# Run the tests +echo "" +RESULT=0 +for test_bin in /rtld.test /test.test "/cudaverify.test -so-path=/libparcagpucupti.so"; do + name=$(echo "$test_bin" | cut -d' ' -f1) + T0=$(cat /proc/uptime | cut -d' ' -f1) + $test_bin -test.v + rc=$? + T1=$(cat /proc/uptime | cut -d' ' -f1) + echo "::> $name took ${T0}s-${T1}s (uptime)" + if [ $rc -ne 0 ]; then + RESULT=$rc + break + fi +done + +if [ $RESULT -eq 0 ]; then + echo "" + echo "===== TEST PASSED =====" +elif [ $RESULT -eq 137 ] || [ $RESULT -eq 124 ]; then + echo "" + echo "===== TEST TIMED OUT =====" +else + echo "" + echo "===== BPF dmesg =====" + dmesg | tail -60 + echo "===== TEST FAILED (exit code: $RESULT) =====" +fi + +# Shutdown +sleep 1 +echo o > /proc/sysrq-trigger 2>/dev/null +sleep 1 +poweroff -f 2>/dev/null || halt -f +INIT_EOF +chmod +x "$ROOTFS_DIR/init" +timer_end "create rootfs" + +# Create initramfs +timer_start +echo "Creating initramfs..." +(cd "$ROOTFS_DIR" && find . | cpio -o -H newc | gzip > "$OUTPUT") +echo "Initramfs created: $OUTPUT ($(du -h "$OUTPUT" | cut -f1))" +timer_end "create initramfs" diff --git a/test/distro-qemu/run-qemu.sh b/test/distro-qemu/run-qemu.sh new file mode 100755 index 000000000..ab2cfb777 --- /dev/null +++ b/test/distro-qemu/run-qemu.sh @@ -0,0 +1,88 @@ +#!/bin/bash +set -ex + +# Run tests in QEMU with a pre-built initramfs and a specific kernel. +# +# Usage: QEMU_ARCH=x86_64 ./run-qemu.sh + +KERNEL_VERSION="${1:?Usage: $0 }" +INITRAMFS="${2:?Usage: $0 }" +QEMU_ARCH="${QEMU_ARCH:-x86_64}" +KERN_DIR="${KERN_DIR:-ci-kernels}" + +# Check inputs +if [[ ! -f "$INITRAMFS" ]]; then + echo "ERROR: Initramfs not found at $INITRAMFS" + exit 1 +fi + +if [[ ! -f "${KERN_DIR}/${KERNEL_VERSION}/vmlinuz" ]]; then + echo "ERROR: Kernel ${KERNEL_VERSION} not found at ${KERN_DIR}/${KERNEL_VERSION}/vmlinuz" + echo "" + echo "To download kernel images:" + echo " QEMU_ARCH=$QEMU_ARCH ./download-kernel.sh $KERNEL_VERSION" + exit 1 +fi + +# Use sudo if /dev/kvm isn't accessible by the current user +sudo="" +if [[ ! -r /dev/kvm || ! -w /dev/kvm ]]; then + sudo="sudo" +fi + +# Determine additional QEMU args based on architecture +additionalQemuArgs="" +if [ -e /dev/kvm ] && [ "$QEMU_ARCH" = "$(uname -m)" ]; then + additionalQemuArgs="-enable-kvm" +fi + +case "$QEMU_ARCH" in + x86_64) + CONSOLE_ARG="console=ttyS0" + ;; + aarch64) + additionalQemuArgs+=" -machine virt -cpu max" + CONSOLE_ARG="console=ttyAMA0" + ;; +esac + +echo "" +echo "===== Starting QEMU with kernel ${KERNEL_VERSION} on ${QEMU_ARCH} =====" +echo "" + +# Run QEMU and capture output +QEMU_OUTPUT=$(mktemp) +${sudo} qemu-system-${QEMU_ARCH} ${additionalQemuArgs} \ + -nographic \ + -monitor none \ + -serial mon:stdio \ + -m 2G \ + -kernel "${KERN_DIR}/${KERNEL_VERSION}/vmlinuz" \ + -initrd "$INITRAMFS" \ + -append "${CONSOLE_ARG} init=/init quiet loglevel=3" \ + -no-reboot \ + -display none \ + | tee "$QEMU_OUTPUT" + +# Parse output for test result +if grep -q "===== TEST PASSED =====" "$QEMU_OUTPUT"; then + rm -f "$QEMU_OUTPUT" + echo "" + echo "Test completed successfully" + exit 0 +elif grep -q "===== TEST FAILED" "$QEMU_OUTPUT"; then + rm -f "$QEMU_OUTPUT" + echo "" + echo "Test failed" + exit 1 +elif grep -q "===== TEST TIMED OUT =====" "$QEMU_OUTPUT"; then + rm -f "$QEMU_OUTPUT" + echo "" + echo "Test timed out" + exit 124 +else + rm -f "$QEMU_OUTPUT" + echo "" + echo "Could not determine test result (QEMU may have crashed)" + exit 2 +fi