Add automated tests using Meta XR Simulator (#150)

dsnopek · web-flow · commit 87dff2b3cc88 · 2024-06-14T09:29:08.000-07:00
diff --git a/.github/workflows/build-addon-on-push.yml b/.github/workflows/build-addon-on-push.yml
@@ -180,3 +180,192 @@ jobs:
           omitPrereleaseDuringUpdate: true
           token: ${{ secrets.GITHUB_TOKEN }}
         if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
+
+  run_xr_simulator:
+   name: "Run XR Simulator"
+   #runs-on: windows-latest
+   runs-on: [Windows, self-hosted, gpu]
+   needs: build
+
+   steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Install Chocolatey
+        run: |
+          Set-ExecutionPolicy Bypass -Scope Process -Force; [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; iex ((New-Object System.Net.WebClient).DownloadString('https://community.chocolatey.org/install.ps1'))
+
+      - name: Install 7zip
+        run: |
+          choco install 7zip -y
+
+      - name: Download Godot
+        run: |
+          # @todo Download an actual Godot build!
+          Invoke-WebRequest -Uri "https://github.com/godotengine/godot-builds/releases/download/4.3-beta1/Godot_v4.3-beta1_win64.exe.zip" -OutFile "godot.zip"
+
+      - name: Extract Godot
+        run: |
+          Expand-Archive -Path "godot.zip" -DestinationPath .
+
+      #- name: Download SwiftShader vulkan-1.dll
+      #  run: |
+      #    Invoke-WebRequest -Uri "https://www.dropbox.com/scl/fi/r6radx6b125555xkxue20/vulkan-1.dll?rlkey=ffv3cgvqbytivm1zy119hss7n&st=frcettrf&dl=1" -OutFile "vulkan-1.dll"
+
+      - name: Download Meta XR Simulator
+        run: |
+          Invoke-WebRequest -Uri "https://npm.developer.oculus.com/com.meta.xr.simulator/-/com.meta.xr.simulator-65.0.0.tgz" -OutFile MetaXRSimulator.tgz
+
+      - name: Extract Meta XR Simulator
+        run: |
+          # Unzip.
+          New-Item -Path "C:\Meta" -ItemType Directory -Force
+          tar -xzvf "MetaXRSimulator.tgz" -C "C:\Meta\"
+          rm "MetaXRSimulator.tgz"
+
+          # Gunzip.
+          $tgzFile = Get-ChildItem -Path "C:\Meta" -Filter *.tgz
+          7z x $tgzFile.FullName -oC:\Meta -aoa
+
+          # Untar.
+          $tarFile = Get-ChildItem -Path "C:\Meta" -Filter *.tar
+          7z x $tarFile.FullName -oC:\Meta -aoa
+
+      - name: Configure the Meta XR Simulator as the active OpenXR runtime
+        run: |
+          New-Item -Path "HKLM:\SOFTWARE\Khronos\OpenXR\1" -Name "ActiveRuntime" -Force
+          Set-ItemProperty -Path "HKLM:\SOFTWARE\Khronos\OpenXR\1" -Name "ActiveRuntime" -Value "C:\Meta\package\MetaXRSimulator\meta_openxr_simulator.json"
+
+      - name: Download Windows build artifacts
+        uses: actions/download-artifact@v3
+        with:
+          name: build-files-windows
+          path: build-files-windows
+
+      - name: Copy Windows build of the addon into the demo project
+        run: |
+          mkdir -p demo/addons/godotopenxrvendors/.bin/windows/
+          cp -r build-files-windows/* demo/addons/godotopenxrvendors/.bin/windows/
+
+      - name: Import the demo project
+        run: |
+          $godot = "Godot_v4.3-beta1_win64.exe"
+          Start-Process -FilePath "$godot" -ArgumentList "--path demo --import --headless" -NoNewWindow -Wait
+
+      - name: Launch a synthetic environment
+        run: |
+          # Ensure a synthetic environment isn't already running.
+          try {
+            Get-Process -Name "synth_env_server" | Stop-Process
+          } catch {
+            # Do nothing if there is no existing process.
+          }
+
+          Start-Process -FilePath "C:\Meta\package\MetaXRSimulator\.synth_env_server\synth_env_server.exe" -ArgumentList "Bedroom" -PassThru
+
+      - name: Run tests
+        run: |
+          $jsonPath = "$env:AppData\MetaXR\MetaXrSimulator\persistent_data.json"
+
+          $vrsFiles = Get-ChildItem -Path tests/vrs -Filter *.vrs
+          foreach ($file in $vrsFiles) {
+            $replayPath = Join-Path -Path $file.DirectoryName -ChildPath ($file.BaseName + "-replay.vrs")
+            $jsonContent = @{
+              session_capture = @{
+                delay_start_ms = 5000
+                exec_state = "replay"
+                quit_buffer_ms = 5000
+                quit_when_complete = $true
+                record_path = $file.FullName
+                replay_path = $replayPath
+              }
+            } | ConvertTo-Json
+
+            New-Item -ItemType Directory -Force -Path (Split-Path $jsonPath)
+            Set-Content -Path $jsonPath -Value $jsonContent
+            echo "$jsonContent"
+
+            $godot = "Godot_v4.3-beta1_win64.exe"
+            $timeout = 300
+            $waitTime = 0
+
+            $process = Start-Process -FilePath "$godot" -ArgumentList "--path demo --rendering-method mobile --verbose -- --quit-with-openxr" -NoNewWindow -PassThru
+
+            while ($process.HasExited -eq $false -and $waitTime -lt $timeout) {
+                Start-Sleep -Seconds 1
+                $waitTime++
+            }
+
+            if ($process.HasExited -eq $false) {
+                Write-Output "Process is still running after $timeout seconds. Stopping the process."
+                $process.Kill()
+                Exit 1
+            } else {
+                Write-Output "Process completed within $waitTime seconds."
+            }
+
+            if (-Not (Test-Path $replayPath)) {
+              Write-Error "Replay file not found: $replayPath"
+              Exit 1
+            }
+          }
+
+      - name: Stop synthetic environment
+        run: |
+          Get-Process -Name "synth_env_server" | Stop-Process
+
+      - name: Copy vrs_pixmatch.py from Meta XR Simulator
+        run: |
+          $scriptsPath = "C:\Meta\package\MetaXRSimulator\scripts"
+          cp "$scriptsPath\requirements.txt" tests\vrs\
+          cp "$scriptsPath\vrs_pixmatch.py" tests\vrs\
+
+      - name: Upload VRS artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: ReplayVRS
+          path: tests/vrs/**/*
+
+  compare_vrs_replay:
+    name: "Compare VRS replay"
+    runs-on: ubuntu-latest
+    needs: run_xr_simulator
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Download VRS artifacts
+        uses: actions/download-artifact@v3
+        with:
+          name: ReplayVRS
+          path: tests/vrs
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r tests/vrs/requirements.txt
+
+      - name: Compare VRS replay with expected recordings
+        run: |
+          cd tests/vrs/
+
+          # Fix bugs in vrs_pixmatch.py script.
+          patch -p0 < ../../thirdparty/meta_xr_simulator/vrs_pixmatch.patch
+
+          mkdir diffs
+          for replay in *-replay.vrs; do
+            expected=$(echo $replay | sed -e 's/-replay.vrs$/.vrs/')
+            python vrs_pixmatch.py "$replay" "$expected" --threshold 0.4 --best_match_pixels_diff_threshold 40000 --diffs_output_path diffs
+          done
+
+      - name: Upload VRS diff artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: ReplayVRSDiff
+          path: tests/vrs/diffs/**/*
diff --git a/demo/main.gd b/demo/main.gd
@@ -40,8 +40,9 @@ const COLORS = [
 # Called when the node enters the scene tree for the first time.
 func _ready():
 	xr_interface = XRServer.find_interface("OpenXR")
-	if xr_interface and xr_interface.is_initialized():
+	if xr_interface and xr_interface.initialize():
 		xr_interface.session_begun.connect(self.load_spatial_anchors_from_file)
+		xr_interface.session_stopping.connect(self._on_session_stopping)
 		var vp: Viewport = get_viewport()
 		vp.use_xr = true
 
@@ -52,6 +53,13 @@ func _ready():
 	randomize()
 
 
+func _on_session_stopping() -> void:
+	if "--quit-with-openxr" in OS.get_cmdline_user_args():
+		# When we're running tests via the XR Simulator, it will end the OpenXR
+		# session automatically, and in that case, we want to quit.
+		get_tree().quit()
+
+
 func load_spatial_anchors_from_file() -> void:
 	var file := FileAccess.open(SPATIAL_ANCHORS_FILE, FileAccess.READ)
 	if not file:
diff --git a/tests/vrs/simulation1.vrs b/tests/vrs/simulation1.vrs
diff --git a/thirdparty/meta_xr_simulator/vrs_pixmatch.patch b/thirdparty/meta_xr_simulator/vrs_pixmatch.patch
@@ -0,0 +1,39 @@
+--- vrs_pixmatch.py-orig	2024-05-14 17:41:44.639317754 -0500
++++ vrs_pixmatch.py	2024-05-14 17:42:42.316556319 -0500
+@@ -125,11 +125,12 @@
+ 
+     return numPixelsDiff
+ 
+-def findTheMatchingFrame(frame_stream, frame_name, target_frame_stream, target_frame_name, sampleIdx, max_test_frames, args, minPixelsDiff):
++def findTheMatchingFrame(frame_stream, frame_name, target_frame_stream, target_frame_name, sampleIdx, max_test_frames, args, minPixelsDiff, bestMatch):
+     sampleTS = getTimestamp(frame_stream, sampleIdx)
+     sampleFrame = getFrame(frame_stream, sampleIdx)
+     matchStartIdx = getFrameIdx(target_frame_stream, sampleTS)
+-    # search around the frame of interest until we find the closest match
++
++    # search around the frame of interest until the closest match is found
+ 
+     # find the matching frame in the target_frame_stream
+     for matchIdx in range(matchStartIdx, matchStartIdx+max_test_frames):
+@@ -203,12 +204,12 @@
+         bestMatch = None
+ 
+         # first check the replay against the record
+-        (minPixelsDiff, bestMatch) = findTheMatchingFrame(record_frame_stream, "record", replay_frame_stream, "replay", sampleIdx, max_test_frames, args, minPixelsDiff)
++        (minPixelsDiff, bestMatch) = findTheMatchingFrame(record_frame_stream, "record", replay_frame_stream, "replay", sampleIdx, max_test_frames, args, minPixelsDiff, bestMatch)
+ 
+-        if minPixelsDiff > 0:
+-            print("best match diff is >0 so matching record against replay")
++        if minPixelsDiff > args.best_match_pixels_diff_threshold:
++            print("best match diff is > ", args.best_match_pixels_diff_threshold, " so matching record against replay")
+             # then check the record against the replay
+-            (minPixelsDiff, bestMatch) = findTheMatchingFrame(replay_frame_stream, "replay", record_frame_stream, "record", sampleIdx, max_test_frames, args, minPixelsDiff)
++            (minPixelsDiff, bestMatch) = findTheMatchingFrame(replay_frame_stream, "replay", record_frame_stream, "record", sampleIdx, max_test_frames, args, minPixelsDiff, bestMatch)
+ 
+         # report the best match
+         if bestMatch:
+@@ -230,3 +231,4 @@
+ 
+ if __name__ == "__main__":
+     main()
++
