diff --git a/.github/actions/run-tests-newton-latest/action.yml b/.github/actions/run-tests-newton-latest/action.yml new file mode 100644 index 00000000000..42f45636d48 --- /dev/null +++ b/.github/actions/run-tests-newton-latest/action.yml @@ -0,0 +1,214 @@ +# Copyright (c) 2022-2025, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +name: 'Run Tests with Latest Newton' +description: 'Runs pytest tests in a Docker container with the latest Newton from main branch' + +inputs: + test-path: + description: 'Path to test directory or pytest arguments' + required: true + result-file: + description: 'Name of the result XML file' + required: true + container-name: + description: 'Name for the Docker container' + required: true + image-tag: + description: 'Docker image tag to use' + required: true + reports-dir: + description: 'Directory to store test results' + default: 'reports' + required: false + pytest-options: + description: 'Additional pytest options (e.g., -k filter)' + default: '' + required: false + filter-pattern: + description: 'Pattern to filter test files (e.g., isaaclab_tasks)' + default: '' + required: false + +runs: + using: composite + steps: + - name: Run Tests with Latest Newton in Docker Container + shell: bash + run: | + # Function to run tests in Docker container with latest Newton + run_tests() { + local test_path="$1" + local result_file="$2" + local container_name="$3" + local image_tag="$4" + local reports_dir="$5" + local pytest_options="$6" + local filter_pattern="$7" + + echo "Running tests with latest Newton in: $test_path" + if [ -n "$pytest_options" ]; then + echo "With pytest options: $pytest_options" + fi + if [ -n "$filter_pattern" ]; then + echo "With filter pattern: $filter_pattern" + fi + + # Create reports directory + mkdir -p "$reports_dir" + + # Clean up any existing container + docker rm -f $container_name 2>/dev/null || true + + # Build Docker environment variables + docker_env_vars="\ + -e OMNI_KIT_ACCEPT_EULA=yes \ + -e ACCEPT_EULA=Y \ + -e OMNI_KIT_DISABLE_CUP=1 \ + -e ISAAC_SIM_HEADLESS=1 \ + -e ISAAC_SIM_LOW_MEMORY=1 \ + -e PYTHONUNBUFFERED=1 \ + -e PYTHONIOENCODING=utf-8 \ + -e TEST_RESULT_FILE=$result_file" + + if [ -n "$filter_pattern" ]; then + if [[ "$filter_pattern" == not* ]]; then + # Handle "not pattern" case + exclude_pattern="${filter_pattern#not }" + docker_env_vars="$docker_env_vars -e TEST_EXCLUDE_PATTERN=$exclude_pattern" + echo "Setting exclude pattern: $exclude_pattern" + else + # Handle positive pattern case + docker_env_vars="$docker_env_vars -e TEST_FILTER_PATTERN=$filter_pattern" + echo "Setting include pattern: $filter_pattern" + fi + else + echo "No filter pattern provided" + fi + + echo "Docker environment variables: '$docker_env_vars'" + + # Run tests in container with error handling + echo "๐Ÿš€ Starting Docker container for tests with latest Newton..." + if docker run --name $container_name \ + --entrypoint bash --gpus all --network=host \ + --security-opt=no-new-privileges:true \ + --memory=$(echo "$(free -m | awk '/^Mem:/{print $2}') * 0.9 / 1" | bc)m \ + --cpus=$(echo "$(nproc) * 0.9" | bc) \ + --oom-kill-disable=false \ + --ulimit nofile=65536:65536 \ + --ulimit nproc=4096:4096 \ + $docker_env_vars \ + $image_tag \ + -c " + set -e + cd /workspace/isaaclab + mkdir -p tests + + echo '=== Uninstalling existing newton and mujoco-warp ===' + /isaac-sim/python.sh -m pip uninstall -y newton mujoco-warp mujoco || echo 'Some packages may not have been installed' + + echo '=== Cloning latest Newton from main branch ===' + git clone --depth 1 https://github.com/newton-physics/newton.git /tmp/newton + + echo '=== Installing Newton with dependencies from uv.lock ===' + cd /tmp/newton + + # Extract mujoco-warp and mujoco versions from uv.lock and install them + # Parse the uv.lock file to get the package versions + if [ -f uv.lock ]; then + echo 'Parsing uv.lock for mujoco-warp and mujoco versions...' + + # Set PIP_FIND_LINKS for mujoco packages + export PIP_FIND_LINKS=https://py.mujoco.org/ + + # Extract mujoco version from uv.lock + mujoco_version=\$(grep -A5 'name = \"mujoco\"' uv.lock | grep 'version = ' | head -1 | sed 's/.*version = \"\(.*\)\"/\1/') + if [ -n \"\$mujoco_version\" ]; then + echo \"Installing mujoco==\$mujoco_version\" + /isaac-sim/python.sh -m pip install \"mujoco==\$mujoco_version\" + else + echo 'mujoco version not found in uv.lock, installing latest' + /isaac-sim/python.sh -m pip install mujoco + fi + + # Extract mujoco-warp git URL from uv.lock (it's installed from git source) + mujoco_warp_git=\$(grep -A10 'name = \"mujoco-warp\"' uv.lock | grep 'git = ' | head -1 | sed 's/.*git = \"\([^\"]*\)\".*/\1/') + if [ -n \"\$mujoco_warp_git\" ]; then + # Parse the git URL - format is https://github.com/org/repo#commit_hash + git_url=\$(echo \"\$mujoco_warp_git\" | cut -d'#' -f1) + commit_hash=\$(echo \"\$mujoco_warp_git\" | cut -d'#' -f2) + echo \"Installing mujoco-warp from git: \$git_url at commit \$commit_hash\" + /isaac-sim/python.sh -m pip install \"git+\${git_url}@\${commit_hash}\" + else + echo 'mujoco-warp git source not found in uv.lock, installing from pip' + /isaac-sim/python.sh -m pip install mujoco-warp + fi + else + echo 'uv.lock not found, installing latest versions' + export PIP_FIND_LINKS=https://py.mujoco.org/ + /isaac-sim/python.sh -m pip install mujoco + /isaac-sim/python.sh -m pip install git+https://github.com/google-deepmind/mujoco_warp.git + fi + + # Install Newton from the cloned repo + echo '=== Installing Newton from source ===' + /isaac-sim/python.sh -m pip install -e . + + echo '=== Verifying installations ===' + /isaac-sim/python.sh -c \"import newton; print(f'Newton version: {newton.__version__}')\" || echo 'Newton import check' + /isaac-sim/python.sh -c \"import mujoco_warp; print('mujoco_warp imported successfully')\" || echo 'mujoco_warp not available' + /isaac-sim/python.sh -c \"import mujoco; print(f'mujoco version: {mujoco.__version__}')\" || echo 'mujoco not available' + + cd /workspace/isaaclab + echo '=== Starting pytest with path: $test_path ===' + /isaac-sim/python.sh -m pytest --ignore=tools/conftest.py $test_path $pytest_options -v -s --junitxml=tests/$result_file || echo 'Pytest completed with exit code: \$?' + "; then + echo "โœ… Docker container completed successfully" + else + echo "โš ๏ธ Docker container failed, but continuing to copy results..." + fi + + # Copy test results with error handling + echo "๐Ÿ“‹ Attempting to copy test results..." + if docker cp $container_name:/workspace/isaaclab/tests/$result_file "$reports_dir/$result_file" 2>/dev/null; then + echo "โœ… Test results copied successfully" + else + echo "โŒ Failed to copy specific result file, trying to copy all test results..." + if docker cp $container_name:/workspace/isaaclab/tests/ "$reports_dir/" 2>/dev/null; then + echo "โœ… All test results copied successfully" + # Look for any XML files and use the first one found + if [ -f "$reports_dir/full_report.xml" ]; then + mv "$reports_dir/full_report.xml" "$reports_dir/$result_file" + echo "โœ… Found and renamed full_report.xml to $result_file" + elif [ -f "$reports_dir/test-reports-"*".xml" ]; then + # Combine individual test reports if no full report exists + echo "๐Ÿ“Š Combining individual test reports..." + echo '' > "$reports_dir/$result_file" + for xml_file in "$reports_dir"/test-reports-*.xml; do + if [ -f "$xml_file" ]; then + echo " Processing: $xml_file" + sed '1d; /^> "$reports_dir/$result_file" 2>/dev/null || true + fi + done + echo '' >> "$reports_dir/$result_file" + echo "โœ… Combined individual test reports into $result_file" + else + echo "โŒ No test result files found, creating fallback" + echo "Container may have failed to generate any results" > "$reports_dir/$result_file" + fi + else + echo "โŒ Failed to copy any test results, creating fallback" + echo "Container may have failed to generate results" > "$reports_dir/$result_file" + fi + fi + + # Clean up container + echo "๐Ÿงน Cleaning up Docker container..." + docker rm $container_name 2>/dev/null || echo "โš ๏ธ Container cleanup failed, but continuing..." + } + + # Call the function with provided parameters + run_tests "${{ inputs.test-path }}" "${{ inputs.result-file }}" "${{ inputs.container-name }}" "${{ inputs.image-tag }}" "${{ inputs.reports-dir }}" "${{ inputs.pytest-options }}" "${{ inputs.filter-pattern }}" diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 05048cc03b1..56b875c0af2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -151,8 +151,110 @@ jobs: exit 1 fi + test-newton-latest-tasks: + runs-on: [self-hosted, gpu] + timeout-minutes: 180 + continue-on-error: true + + steps: + - name: Checkout Code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + lfs: true + + - name: Build Docker Image + uses: ./.github/actions/docker-build + with: + image-tag: ${{ env.DOCKER_IMAGE_TAG }} + isaacsim-base-image: ${{ env.ISAACSIM_BASE_IMAGE }} + isaacsim-version: ${{ env.ISAACSIM_BASE_VERSION }} + + - name: Run IsaacLab Tasks Tests with Latest Newton + uses: ./.github/actions/run-tests-newton-latest + with: + test-path: "tools" + result-file: "newton-latest-tasks-report.xml" + container-name: "isaac-lab-newton-latest-tasks-test-$$" + image-tag: ${{ env.DOCKER_IMAGE_TAG }} + pytest-options: "" + filter-pattern: "isaaclab_tasks" + + - name: Upload Newton Latest Tasks Test Results + uses: actions/upload-artifact@v4 + if: always() + with: + name: newton-latest-tasks-test-results + path: reports/newton-latest-tasks-report.xml + retention-days: 1 + compression-level: 9 + + - name: Check Test Results for Fork PRs + if: github.event.pull_request.head.repo.full_name != github.repository + run: | + if [ -f "reports/newton-latest-tasks-report.xml" ]; then + if grep -q 'failures="[1-9]' reports/newton-latest-tasks-report.xml || grep -q 'errors="[1-9]' reports/newton-latest-tasks-report.xml; then + echo "Tests failed for PR from fork. The test report is in the logs. Failing the job." + exit 1 + fi + else + echo "No test results file found. This might indicate test execution failed." + exit 1 + fi + + test-newton-latest-general: + runs-on: [self-hosted, gpu] + timeout-minutes: 180 + continue-on-error: true + + steps: + - name: Checkout Code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + lfs: true + + - name: Build Docker Image + uses: ./.github/actions/docker-build + with: + image-tag: ${{ env.DOCKER_IMAGE_TAG }} + isaacsim-base-image: ${{ env.ISAACSIM_BASE_IMAGE }} + isaacsim-version: ${{ env.ISAACSIM_BASE_VERSION }} + + - name: Run General Tests with Latest Newton + uses: ./.github/actions/run-tests-newton-latest + with: + test-path: "tools" + result-file: "newton-latest-general-report.xml" + container-name: "isaac-lab-newton-latest-general-test-$$" + image-tag: ${{ env.DOCKER_IMAGE_TAG }} + pytest-options: "" + filter-pattern: "not isaaclab_tasks" + + - name: Upload Newton Latest General Test Results + uses: actions/upload-artifact@v4 + if: always() + with: + name: newton-latest-general-test-results + path: reports/newton-latest-general-report.xml + retention-days: 1 + compression-level: 9 + + - name: Check Test Results for Fork PRs + if: github.event.pull_request.head.repo.full_name != github.repository + run: | + if [ -f "reports/newton-latest-general-report.xml" ]; then + if grep -q 'failures="[1-9]' reports/newton-latest-general-report.xml || grep -q 'errors="[1-9]' reports/newton-latest-general-report.xml; then + echo "Tests failed for PR from fork. The test report is in the logs. Failing the job." + exit 1 + fi + else + echo "No test results file found. This might indicate test execution failed." + exit 1 + fi + combine-results: - needs: [test-isaaclab-tasks, test-general] + needs: [test-isaaclab-tasks, test-general, test-newton-latest-tasks, test-newton-latest-general] runs-on: [self-hosted, gpu] if: always() @@ -179,6 +281,21 @@ jobs: with: name: general-test-results path: reports/ + continue-on-error: true + + - name: Download Newton Latest Tasks Test Results + uses: actions/download-artifact@v4 + with: + name: newton-latest-tasks-test-results + path: reports/ + continue-on-error: true + + - name: Download Newton Latest General Test Results + uses: actions/download-artifact@v4 + with: + name: newton-latest-general-test-results + path: reports/ + continue-on-error: true - name: Combine All Test Results uses: ./.github/actions/combine-results diff --git a/source/isaaclab/isaaclab/app/app_launcher.py b/source/isaaclab/isaaclab/app/app_launcher.py index 0eb449335a5..5ac74a940ff 100644 --- a/source/isaaclab/isaaclab/app/app_launcher.py +++ b/source/isaaclab/isaaclab/app/app_launcher.py @@ -991,6 +991,9 @@ def _create_app(self): # disable sys stdout and stderr to avoid printing the warning messages # this is mainly done to purge the print statements from the simulation app + # Note: We save the current stdout (not sys.__stdout__) to properly restore it + # when running under pytest or other tools that capture output + original_stdout = sys.stdout if "--verbose" not in sys.argv and "--info" not in sys.argv: sys.stdout = open(os.devnull, "w") # noqa: SIM115 # launch simulation app diff --git a/tools/test_settings.py b/tools/test_settings.py index 5b0664aaaf7..9af0df8367b 100644 --- a/tools/test_settings.py +++ b/tools/test_settings.py @@ -19,7 +19,7 @@ "test_articulation.py": 500, "test_rigid_object.py": 300, "test_environments.py": 1500, # This test runs through all the environments for 100 steps each - "test_environments_standalone.py": 1500, # This test runs through all the environments for 100 steps each + "test_environments_standalone.py": 2500, # This test runs through all the environments for 100 steps each "test_environment_determinism.py": 500, # This test runs through many the environments for 100 steps each "test_factory_environments.py": 300, # This test runs through Factory environments for 100 steps each "test_env_rendering_logic.py": 300,