diff --git a/.github/workflows/create-stale-docs-issues.yml b/.github/workflows/create-stale-docs-issues.yml index ff284c84..7dfdeaa5 100644 --- a/.github/workflows/create-stale-docs-issues.yml +++ b/.github/workflows/create-stale-docs-issues.yml @@ -47,9 +47,16 @@ jobs: path: logs - name: Create or Update Issues (One Per File) + env: + GH_TOKEN: ${{ github.token }} + RESULTS_FILE: ${{ inputs.results-file }} + THRESHOLD_DAYS: ${{ inputs.threshold-days }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_RUN_ID: ${{ github.run_id }} run: | # Extract stale files from JSON - $results = Get-Content '${{ inputs.results-file }}' | ConvertFrom-Json + $results = Get-Content $env:RESULTS_FILE | ConvertFrom-Json $staleFiles = $results | Where-Object { $_.IsStale -eq $true } if (-not $staleFiles) { @@ -73,9 +80,9 @@ jobs: # Check for existing open issue for this specific file using search # Search by automation marker in issue body - $searchQuery = "repo:${{ github.repository }} is:issue is:open in:body `"automation:stale-docs:$filePath`"" + $searchQuery = "repo:$env:GITHUB_REPOSITORY is:issue is:open in:body `"automation:stale-docs:$filePath`"" $existingIssueJson = gh issue list ` - --repo "${{ github.repository }}" ` + --repo $env:GITHUB_REPOSITORY ` --search "$searchQuery" ` --limit 1 ` --json number ` @@ -89,10 +96,10 @@ jobs: **File:** ``$filePath`` **Current ms.date:** ``$msDate`` - **Age:** $ageDays days (threshold: ${{ inputs.threshold-days }} days) + **Age:** $ageDays days (threshold: $env:THRESHOLD_DAYS days) --- - **Workflow Run:** ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + **Workflow Run:** $env:GITHUB_SERVER_URL/$env:GITHUB_REPOSITORY/actions/runs/$env:GITHUB_RUN_ID **Detection Date:** $(Get-Date -Format 'yyyy-MM-dd' -AsUTC) ### Action Required @@ -107,17 +114,17 @@ jobs: if ($existingIssue) { Write-Host "Updating existing issue #$existingIssue for $filePath" gh issue edit $existingIssue ` - --repo "${{ github.repository }}" ` + --repo $env:GITHUB_REPOSITORY ` --body $issueBody $updateDate = Get-Date -Format 'yyyy-MM-dd' -AsUTC gh issue comment $existingIssue ` - --repo "${{ github.repository }}" ` + --repo $env:GITHUB_REPOSITORY ` --body "🔄 **Weekly validation update:** Still stale ($ageDays days) as of $updateDate" } else { Write-Host "Creating new issue for $filePath" gh issue create ` - --repo "${{ github.repository }}" ` + --repo $env:GITHUB_REPOSITORY ` --title $issueTitle ` --body $issueBody ` --label "documentation,stale-docs,automated,needs-triage" @@ -125,5 +132,3 @@ jobs: } Write-Host "Issue creation/update complete" - env: - GH_TOKEN: ${{ github.token }} diff --git a/.github/workflows/msdate-freshness-check.yml b/.github/workflows/msdate-freshness-check.yml index d25519ac..93548aed 100644 --- a/.github/workflows/msdate-freshness-check.yml +++ b/.github/workflows/msdate-freshness-check.yml @@ -13,6 +13,11 @@ on: required: false type: boolean default: false + soft-fail: + description: 'Prevent PR failures when ms.date issues are detected' + required: false + type: boolean + default: false permissions: contents: read @@ -43,14 +48,18 @@ jobs: Install-Module -Name PowerShell-Yaml -Force -Scope CurrentUser - name: Run ms.date Freshness Check + env: + THRESHOLD_DAYS: ${{ inputs.staleness-threshold-days }} + CHANGED_FILES_ONLY: ${{ inputs.changed-files-only }} run: | - $args = @{ - ThresholdDays = ${{ inputs.staleness-threshold-days }} + $params = @{ + ThresholdDays = [int]$env:THRESHOLD_DAYS } - if ('${{ inputs.changed-files-only }}' -eq 'true') { - $args['ChangedFilesOnly'] = $true + if ($env:CHANGED_FILES_ONLY -eq 'true') { + $params['ChangedFilesOnly'] = $true } - & shared/ci/linting/Invoke-MsDateFreshnessCheck.ps1 @args + & shared/ci/linting/Invoke-MsDateFreshnessCheck.ps1 @params + continue-on-error: ${{ inputs.soft-fail }} - name: Upload freshness check results if: always() diff --git a/.github/workflows/pr-validation.yml b/.github/workflows/pr-validation.yml index 4eb54a61..af0f8014 100644 --- a/.github/workflows/pr-validation.yml +++ b/.github/workflows/pr-validation.yml @@ -50,6 +50,7 @@ jobs: with: staleness-threshold-days: 90 changed-files-only: true + soft-fail: false permissions: contents: read diff --git a/.github/workflows/weekly-validation.yml b/.github/workflows/weekly-validation.yml index ca0c35fc..95d18f99 100644 --- a/.github/workflows/weekly-validation.yml +++ b/.github/workflows/weekly-validation.yml @@ -26,6 +26,7 @@ jobs: with: staleness-threshold-days: 90 changed-files-only: false # Check all files + soft-fail: false permissions: contents: read diff --git a/shared/ci/linting/Invoke-MsDateFreshnessCheck.ps1 b/shared/ci/linting/Invoke-MsDateFreshnessCheck.ps1 index c8d536fd..f83559ca 100644 --- a/shared/ci/linting/Invoke-MsDateFreshnessCheck.ps1 +++ b/shared/ci/linting/Invoke-MsDateFreshnessCheck.ps1 @@ -201,6 +201,7 @@ function New-MsDateReport { | File | ms.date | Age (days) | |------|---------|------------| "@ + $markdown += "`n" $sortedStaleFiles = $staleFiles | Sort-Object -Property AgeDays -Descending diff --git a/shared/ci/tests/Fixtures/Frontmatter/empty-description.md b/shared/ci/tests/Fixtures/Frontmatter/empty-description.md index 6cf6f00b..877887cc 100644 --- a/shared/ci/tests/Fixtures/Frontmatter/empty-description.md +++ b/shared/ci/tests/Fixtures/Frontmatter/empty-description.md @@ -1,7 +1,7 @@ --- title: Empty Description Test description: "" -ms.date: 2026-03-16 +ms.date: 2026-03-20 --- # Empty Description diff --git a/shared/ci/tests/Fixtures/Frontmatter/footer-with-formatting.md b/shared/ci/tests/Fixtures/Frontmatter/footer-with-formatting.md index bce51a42..eac38d07 100644 --- a/shared/ci/tests/Fixtures/Frontmatter/footer-with-formatting.md +++ b/shared/ci/tests/Fixtures/Frontmatter/footer-with-formatting.md @@ -2,7 +2,7 @@ title: Formatted Footer Test description: A document with bold and italic formatting applied to the footer author: test-author -ms.date: 2026-03-16 +ms.date: 2026-03-20 --- Content with a formatted footer that should still match after normalization. diff --git a/shared/ci/tests/Fixtures/Frontmatter/footer-with-html-comment.md b/shared/ci/tests/Fixtures/Frontmatter/footer-with-html-comment.md index ddf7ce53..8fdbad1b 100644 --- a/shared/ci/tests/Fixtures/Frontmatter/footer-with-html-comment.md +++ b/shared/ci/tests/Fixtures/Frontmatter/footer-with-html-comment.md @@ -2,7 +2,7 @@ title: HTML Comment Footer Test description: A document with HTML comment wrapping around the footer author: test-author -ms.date: 2026-03-16 +ms.date: 2026-03-20 --- Content with HTML comments interspersed near the footer. diff --git a/shared/ci/tests/Fixtures/Frontmatter/valid-docs-with-footer.md b/shared/ci/tests/Fixtures/Frontmatter/valid-docs-with-footer.md index d705d5e1..160ae2ad 100644 --- a/shared/ci/tests/Fixtures/Frontmatter/valid-docs-with-footer.md +++ b/shared/ci/tests/Fixtures/Frontmatter/valid-docs-with-footer.md @@ -2,7 +2,7 @@ title: Azure ML Validation Job Debugging description: Troubleshooting guide for Azure Machine Learning validation job failures author: Robotics-AI Team -ms.date: 2026-03-16 +ms.date: 2026-03-20 ms.topic: troubleshooting --- diff --git a/shared/ci/tests/Fixtures/Frontmatter/valid-docs.md b/shared/ci/tests/Fixtures/Frontmatter/valid-docs.md index ea65c6b1..c96996da 100644 --- a/shared/ci/tests/Fixtures/Frontmatter/valid-docs.md +++ b/shared/ci/tests/Fixtures/Frontmatter/valid-docs.md @@ -2,7 +2,7 @@ title: Azure ML Validation Job Debugging description: Troubleshooting guide for Azure Machine Learning validation job failures author: Robotics-AI Team -ms.date: 2026-03-16 +ms.date: 2026-03-20 ms.topic: troubleshooting --- diff --git a/shared/ci/tests/Fixtures/Frontmatter/valid-root-community-with-footer.md b/shared/ci/tests/Fixtures/Frontmatter/valid-root-community-with-footer.md index 5ece808c..9f0cf654 100644 --- a/shared/ci/tests/Fixtures/Frontmatter/valid-root-community-with-footer.md +++ b/shared/ci/tests/Fixtures/Frontmatter/valid-root-community-with-footer.md @@ -2,7 +2,7 @@ title: Contributing Guide description: Guidelines for contributing to the Physical AI Toolchain author: Robotics-AI Team -ms.date: 2026-03-16 +ms.date: 2026-03-20 --- Thank you for your interest in contributing to this project. diff --git a/shared/ci/tests/Fixtures/Frontmatter/valid-root-community.md b/shared/ci/tests/Fixtures/Frontmatter/valid-root-community.md index bbc96eeb..6f627516 100644 --- a/shared/ci/tests/Fixtures/Frontmatter/valid-root-community.md +++ b/shared/ci/tests/Fixtures/Frontmatter/valid-root-community.md @@ -2,7 +2,7 @@ title: Contributing Guide description: Guidelines for contributing to the Physical AI Toolchain author: Robotics-AI Team -ms.date: 2026-03-16 +ms.date: 2026-03-20 --- Thank you for your interest in contributing to this project. diff --git a/shared/ci/tests/Fixtures/Frontmatter/warning-author-empty.md b/shared/ci/tests/Fixtures/Frontmatter/warning-author-empty.md index e7f3df07..e9dbc988 100644 --- a/shared/ci/tests/Fixtures/Frontmatter/warning-author-empty.md +++ b/shared/ci/tests/Fixtures/Frontmatter/warning-author-empty.md @@ -2,7 +2,7 @@ title: Warning Test Fixture description: File with empty author field to trigger a validation warning author: "" -ms.date: 2026-03-16 +ms.date: 2026-03-20 ms.topic: conceptual --- diff --git a/workflows/README.md b/workflows/README.md index b4025898..02acb711 100644 --- a/workflows/README.md +++ b/workflows/README.md @@ -2,7 +2,7 @@ title: Workflows description: AzureML and OSMO workflow templates for robotics training and validation jobs author: Edge AI Team -ms.date: 2026-03-16 +ms.date: 2026-03-20 ms.topic: reference --- @@ -43,32 +43,32 @@ workflows/ ```bash # Training job -./scripts/submit-azureml-training.sh --task Isaac-Velocity-Rough-Anymal-C-v0 +training/rl/scripts/submit-azureml-training.sh --task Isaac-Velocity-Rough-Anymal-C-v0 # LeRobot behavioral cloning (AzureML) -./scripts/submit-azureml-lerobot-training.sh -d lerobot/aloha_sim_insertion_human +training/il/scripts/submit-azureml-lerobot-training.sh -d lerobot/aloha_sim_insertion_human # Validation job (model name derived from task by default) -./scripts/submit-azureml-validation.sh --task Isaac-Velocity-Rough-Anymal-C-v0 +evaluation/sil/scripts/submit-azureml-validation.sh --task Isaac-Velocity-Rough-Anymal-C-v0 ``` ### OSMO Workflows ```bash # Base64 payload (< 1MB training code) -./scripts/submit-osmo-training.sh --task Isaac-Velocity-Rough-Anymal-C-v0 +training/rl/scripts/submit-osmo-training.sh --task Isaac-Velocity-Rough-Anymal-C-v0 # Dataset folder upload (unlimited size, versioned) -./scripts/submit-osmo-dataset-training.sh --task Isaac-Velocity-Rough-Anymal-C-v0 +training/il/scripts/submit-osmo-dataset-training.sh --task Isaac-Velocity-Rough-Anymal-C-v0 # LeRobot behavioral cloning (HuggingFace datasets) -./scripts/submit-osmo-lerobot-training.sh -d lerobot/aloha_sim_insertion_human +training/il/scripts/submit-osmo-lerobot-training.sh -d lerobot/aloha_sim_insertion_human # LeRobot inference/evaluation -./scripts/submit-osmo-lerobot-inference.sh --policy-repo-id user/trained-policy +evaluation/sil/scripts/submit-osmo-lerobot-eval.sh --policy-repo-id user/trained-policy # End-to-end pipeline: train → evaluate → register -./scripts/run-lerobot-pipeline.sh \ +training/pipelines/run-lerobot-pipeline.sh \ -d lerobot/aloha_sim_insertion_human \ --policy-repo-id user/my-policy \ -r my-model @@ -89,10 +89,10 @@ The `train-dataset.yaml` template uploads `training/rl/` as a versioned OSMO dat ```bash # Default configuration -./scripts/submit-osmo-dataset-training.sh --task Isaac-Velocity-Rough-Anymal-C-v0 +training/il/scripts/submit-osmo-dataset-training.sh --task Isaac-Velocity-Rough-Anymal-C-v0 # Custom dataset configuration -./scripts/submit-osmo-dataset-training.sh \ +training/il/scripts/submit-osmo-dataset-training.sh \ --dataset-bucket custom-bucket \ --dataset-name my-training-v1 \ --task Isaac-Velocity-Rough-Anymal-C-v0 @@ -135,11 +135,11 @@ The `lerobot-train.yaml` workflow trains behavioral cloning policies using the L ```bash # ACT training with WANDB -./scripts/submit-osmo-lerobot-training.sh \ +training/il/scripts/submit-osmo-lerobot-training.sh \ -d lerobot/aloha_sim_insertion_human # Diffusion policy with MLflow and model registration -./scripts/submit-osmo-lerobot-training.sh \ +training/il/scripts/submit-osmo-lerobot-training.sh \ -d user/custom-dataset \ -p diffusion \ --mlflow-enable \ @@ -172,11 +172,11 @@ The `lerobot-infer.yaml` workflow evaluates trained LeRobot policies from Huggin ```bash # Evaluate trained policy -./scripts/submit-osmo-lerobot-inference.sh \ +evaluation/sil/scripts/submit-osmo-lerobot-eval.sh \ --policy-repo-id user/trained-act-policy # Evaluate with model registration -./scripts/submit-osmo-lerobot-inference.sh \ +evaluation/sil/scripts/submit-osmo-lerobot-eval.sh \ --policy-repo-id user/trained-act-policy \ -r my-evaluated-model \ --eval-episodes 50 @@ -208,7 +208,7 @@ The workflow accepts checkpoints from multiple sources: ### Basic Usage ```bash -./scripts/submit-osmo-inference.sh \ +evaluation/sil/scripts/submit-osmo-eval.sh \ --checkpoint-uri "runs:/abc123/checkpoints/final/model_999.pt" \ --task Isaac-Ant-v0 ``` @@ -228,7 +228,7 @@ The workflow accepts checkpoints from multiple sources: ```bash # ONNX-only inference with custom parameters -./scripts/submit-osmo-inference.sh \ +evaluation/sil/scripts/submit-osmo-eval.sh \ --checkpoint-uri "models:/my-model/1" \ --task Isaac-Velocity-Rough-Anymal-C-v0 \ --format onnx \ @@ -237,13 +237,13 @@ The workflow accepts checkpoints from multiple sources: --video-length 300 # TorchScript-only inference -./scripts/submit-osmo-inference.sh \ +evaluation/sil/scripts/submit-osmo-eval.sh \ --checkpoint-uri "runs:/abc123/checkpoints/final/model_99.pt" \ --task Isaac-Ant-v0 \ --format jit # With explicit Azure context -./scripts/submit-osmo-inference.sh \ +evaluation/sil/scripts/submit-osmo-eval.sh \ --checkpoint-uri "runs:/abc123/checkpoints/model_999.pt" \ --task Isaac-Ant-v0 \ --azure-subscription-id "00000000-0000-0000-0000-000000000000" \ diff --git a/workflows/azureml/README.md b/workflows/azureml/README.md index b70dc67f..6bf97531 100644 --- a/workflows/azureml/README.md +++ b/workflows/azureml/README.md @@ -2,7 +2,7 @@ title: AzureML Workflows description: Azure Machine Learning job templates for robotics training and validation author: Edge AI Team -ms.date: 2025-12-04 +ms.date: 2026-03-20 ms.topic: reference --- @@ -10,11 +10,11 @@ Azure Machine Learning job templates for Isaac Lab training and validation workl ## 📜 Available Templates -| Template | Purpose | Submission Script | -|------------------------------------------|---------------------------------------|----------------------------------------------| -| [train.yaml](train.yaml) | Training jobs with checkpoint support | `scripts/submit-azureml-training.sh` | -| [validate.yaml](validate.yaml) | Policy validation and inference | `scripts/submit-azureml-validation.sh` | -| [lerobot-train.yaml](lerobot-train.yaml) | LeRobot behavioral cloning training | `scripts/submit-azureml-lerobot-training.sh` | +| Template | Purpose | Submission Script | +|---------------------------------------------------------------------------------|---------------------------------------|----------------------------------------------------------| +| [train.yaml](../../training/rl/workflows/azureml/train.yaml) | Training jobs with checkpoint support | `training/rl/scripts/submit-azureml-training.sh` | +| [validate.yaml](../../evaluation/sil/workflows/azureml/validate.yaml) | Policy validation and inference | `evaluation/sil/scripts/submit-azureml-validation.sh` | +| [lerobot-train.yaml](../../training/il/workflows/azureml/lerobot-train.yaml) | LeRobot behavioral cloning training | `training/il/scripts/submit-azureml-lerobot-training.sh` | ## 🏋️ Training Job (`train.yaml`) @@ -35,10 +35,10 @@ Submits Isaac Lab reinforcement learning training to AKS GPU nodes via Azure ML. ```bash # Default configuration from Terraform outputs -./scripts/submit-azureml-training.sh +training/rl/scripts/submit-azureml-training.sh # Override specific parameters -./scripts/submit-azureml-training.sh \ +training/rl/scripts/submit-azureml-training.sh \ --resource-group rg-custom \ --workspace-name mlw-custom ``` @@ -60,10 +60,10 @@ Runs trained policy validation and generates inference metrics. ```bash # Default configuration -./scripts/submit-azureml-validation.sh +evaluation/sil/scripts/submit-azureml-validation.sh # With custom checkpoint -./scripts/submit-azureml-validation.sh \ +evaluation/sil/scripts/submit-azureml-validation.sh \ --checkpoint-path "azureml://datastores/checkpoints/paths/model.pt" ``` @@ -103,11 +103,11 @@ Submits LeRobot behavioral cloning training (ACT/Diffusion policies) to Azure ML ```bash # ACT policy training -./scripts/submit-azureml-lerobot-training.sh \ +training/il/scripts/submit-azureml-lerobot-training.sh \ -d lerobot/aloha_sim_insertion_human # Diffusion policy with model registration -./scripts/submit-azureml-lerobot-training.sh \ +training/il/scripts/submit-azureml-lerobot-training.sh \ -d user/custom-dataset \ -p diffusion \ -r my-diffusion-model \ diff --git a/workflows/osmo/README.md b/workflows/osmo/README.md index 626664bf..e016dab5 100644 --- a/workflows/osmo/README.md +++ b/workflows/osmo/README.md @@ -2,7 +2,7 @@ title: OSMO Workflows description: NVIDIA OSMO workflow templates for distributed robotics training author: Edge AI Team -ms.date: 2026-03-16 +ms.date: 2026-03-20 ms.topic: reference --- @@ -55,10 +55,10 @@ Parameters are passed as key=value pairs through the submission script: ```bash # Default configuration from Terraform outputs -./scripts/submit-osmo-training.sh +training/rl/scripts/submit-osmo-training.sh # Override parameters -./scripts/submit-osmo-training.sh \ +training/rl/scripts/submit-osmo-training.sh \ --azure-subscription-id "your-subscription-id" \ --azure-resource-group "rg-custom" ``` @@ -86,10 +86,10 @@ Submits Isaac Lab training using OSMO dataset folder injection instead of base64 ```bash # Default configuration -./scripts/submit-osmo-dataset-training.sh +training/il/scripts/submit-osmo-dataset-training.sh # Custom dataset bucket -./scripts/submit-osmo-dataset-training.sh \ +training/il/scripts/submit-osmo-dataset-training.sh \ --dataset-bucket custom-bucket \ --dataset-name my-training-code ``` @@ -124,18 +124,18 @@ Submits LeRobot behavioral cloning training for ACT and Diffusion policy archite ```bash # ACT training with WANDB logging -./scripts/submit-osmo-lerobot-training.sh \ +training/il/scripts/submit-osmo-lerobot-training.sh \ -d lerobot/aloha_sim_insertion_human # Diffusion policy with MLflow logging -./scripts/submit-osmo-lerobot-training.sh \ +training/il/scripts/submit-osmo-lerobot-training.sh \ -d user/custom-dataset \ -p diffusion \ --mlflow-enable \ -r my-diffusion-model # Fine-tune from existing policy -./scripts/submit-osmo-lerobot-training.sh \ +training/il/scripts/submit-osmo-lerobot-training.sh \ -d user/dataset \ --policy-repo-id user/pretrained-act \ --training-steps 50000 @@ -176,7 +176,7 @@ Trains LeRobot policies using OSMO dataset mounts instead of HuggingFace Hub dow ```bash # Train with local dataset uploaded via OSMO -./scripts/submit-osmo-lerobot-training.sh \ +training/il/scripts/submit-osmo-lerobot-training.sh \ -w workflows/osmo/lerobot-train-dataset.yaml \ -d user/fallback-dataset \ --dataset-bucket my-bucket \ @@ -209,16 +209,16 @@ Evaluates trained LeRobot policies from HuggingFace Hub repositories. Downloads ```bash # Evaluate a trained policy -./scripts/submit-osmo-lerobot-inference.sh \ +evaluation/sil/scripts/submit-osmo-lerobot-eval.sh \ --policy-repo-id user/trained-act-policy # Evaluate with Azure ML model registration -./scripts/submit-osmo-lerobot-inference.sh \ +evaluation/sil/scripts/submit-osmo-lerobot-eval.sh \ --policy-repo-id user/trained-act-policy \ -r my-evaluated-model # Diffusion policy with more episodes -./scripts/submit-osmo-lerobot-inference.sh \ +evaluation/sil/scripts/submit-osmo-lerobot-eval.sh \ --policy-repo-id user/diffusion-policy \ -p diffusion \ --eval-episodes 50