test: Add end-to-end integration tests

.github/actions/prepare-testing/action.yml

@@ -15,6 +15,7 @@ runs:
     - name: Checkout required submodules
       shell: bash
       run: |
+        git submodule update --init --depth 1 modules/app-runner
         git submodule update --init --depth 1 modules/gdUnit4
         git submodule update --init --depth 1 modules/godot-cpp
         git submodule update --init --depth 1 project/test/util/json_assert
@@ -39,6 +40,19 @@ runs:
         distribution: temurin
         java-version: 17
 
+    - name: Install Linux dependencies (32-bit)
+      if: runner.os == 'Linux' && inputs.arch == 'x86_32'
+      shell: bash
+      run: |
+        sudo dpkg --add-architecture i386
+        sudo apt-get update
+        sudo apt-get install \
+            libatomic1:i386 \
+            libudev1:i386 \
+            libfontconfig1:i386 \
+            libcurl4-openssl-dev:i386 \
+            zlib1g:i386
+
     - name: Set up Godot
       shell: bash
       env:

.github/workflows/ci.yml

@@ -40,6 +40,13 @@ jobs:
     needs: build-extension
     uses: ./.github/workflows/test_android.yml
 
+  test-integration:
+    name: 🧪 Integration tests
+    needs: build-extension
+    uses: ./.github/workflows/test_integration.yml
+    secrets:
+      SENTRY_API_TOKEN: ${{ secrets.SENTRY_API_TOKEN }}
+
   package:
     name: 📦 Package
     needs: build-extension
@@ -54,7 +61,7 @@ jobs:
   # See issue: https://github.com/getsentry/sentry-godot/issues/41
   cleanup:
     name: 🗑️ Cleanup
-    needs: [package, unit-tests, test-android, static-checks]
+    needs: [package, unit-tests, test-android, test-integration, static-checks]
     runs-on: ubuntu-latest
     steps:
       - name: Delete sentry-godot-gdextension artifact

.github/workflows/test_integration.yml

@@ -0,0 +1,57 @@
+name: 🧪 Integration tests
+
+on:
+  workflow_call:
+    secrets:
+      SENTRY_API_TOKEN:
+        required: true
+
+permissions:
+  contents: read
+
+jobs:
+  integration-tests:
+    name: Test ${{matrix.platform}} ${{matrix.arch}}
+    runs-on: ${{matrix.runner}}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - platform: Linux
+            runner: ubuntu-latest
+            arch: x86_64
+          - platform: Linux
+            runner: ubuntu-latest
+            arch: x86_32
+          - platform: Windows
+            runner: windows-latest
+            arch: x86_64
+          - platform: Windows
+            runner: windows-latest
+            arch: x86_32
+          - platform: macOS
+            runner: macos-latest
+            arch: universal
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v4
+        with:
+          submodules: false # don't initialize submodules automatically
+
+      - name: Prepare testing
+        uses: ./.github/actions/prepare-testing
+        timeout-minutes: 15
+        with:
+          arch: ${{ matrix.arch }}
+          android: ${{ matrix.platform == 'Android' && 'true' || 'false' }}
+
+      - name: Install Pester module for pwsh
+        shell: pwsh
+        run: Install-Module -Name Pester -Force -SkipPublisherCheck
+
+      - name: Run integration tests locally
+        shell: pwsh
+        env:
+          SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_API_TOKEN }}
+          SENTRY_TEST_PLATFORM: ${{ matrix.platform }}
+        run: Invoke-Pester -Script integration_tests/Integration.Tests.ps1

.github/workflows/unit_tests.yml

@@ -30,18 +30,6 @@ jobs:
         with:
           submodules: false # don't initialize submodules automatically
 
-      - name: Install Linux dependencies (32-bit)
-        if: matrix.runner == 'ubuntu-latest' && matrix.arch == 'x86_32'
-        run: |
-          sudo dpkg --add-architecture i386
-          sudo apt-get update
-          sudo apt-get install \
-              libatomic1:i386 \
-              libudev1:i386 \
-              libfontconfig1:i386 \
-              libcurl4-openssl-dev:i386 \
-              zlib1g:i386
-
       - name: Prepare testing
         uses: ./.github/actions/prepare-testing
         with:
@@ -50,12 +38,9 @@ jobs:
       - name: Run tests
         shell: bash
         timeout-minutes: 5
-        env:
-          SENTRY_TEST: 1
-          SENTRY_TEST_INCLUDE: "res://test/suites/"
         run: |
           # Exit status codes: 0 - success, 100 - failures, 101 - warnings, 104 - tests not found, 105 - didn't run.
-          ${GODOT} --headless --path project/
+          ${GODOT} --headless --path project/ -- run-tests "res://test/suites/"
 
       - name: Run isolated tests
         if: success() || failure()

.gitignore

@@ -12,6 +12,8 @@ project/.godot/
 exports/*
 !exports/export_presets.cfg
 
+integration_tests/results/
+
 # Source tree
 src/gen/
 

.gitmodules

@@ -10,3 +10,6 @@
 [submodule "project/test/util/json_assert"]
 	path = project/test/util/json_assert
 	url = https://github.com/getsentry/gdunit-json-assert
+[submodule "modules/app-runner"]
+	path = modules/app-runner
+	url = https://github.com/getsentry/app-runner.git

integration_tests/CommonTestCases.ps1

@@ -0,0 +1,134 @@
+# Defines a collection of reusable test cases that validate common Sentry event properties
+# and behaviors across different test suites for consistent integration testing.
+#
+# Available parameters:
+# - $SentryEvent: The Sentry event object retrieved from the REST API containing error/message details
+# - $TestType: String indicating the type of test being run (e.g., 'crash-capture', 'message-capture')
+# - $RunResult: Object containing the results of running the test application, including Output and ExitCode
+
+$CommonTestCases = @(
+    @{ Name = 'Outputs event ID'; TestBlock = {
+            param($RunResult)
+            $eventId = Get-EventIds -appOutput $RunResult.Output -expectedCount 1
+            $eventId | Should -Not -BeNullOrEmpty
+        }
+    }
+    @{ Name = 'Captures event in sentry.io'; TestBlock = {
+            param($SentryEvent)
+            $SentryEvent | Should -Not -BeNullOrEmpty
+        }
+    }
+    @{ Name = 'Has title'; TestBlock = {
+            param($SentryEvent)
+            $SentryEvent.title | Should -Not -BeNullOrEmpty
+        }
+    }
+    @{ Name = 'Has correct release version'; TestBlock = {
+            param($SentryEvent)
+            $SentryEvent.release.version | Should -Be '[email protected]'
+        }
+    }
+    @{ Name = 'Has correct platform'; TestBlock = {
+            param($SentryEvent)
+            $SentryEvent.platform | Should -Not -BeNullOrEmpty
+            if ($IsLinux -or $IsWindows) {
+                $SentryEvent.platform | Should -Be "native"
+            } elseif ($IsMacOS) {
+                $SentryEvent.platform | Should -Be "cocoa"
+            }
+        }
+    }
+    @{ Name = 'Has correct dist attribute'; TestBlock = {
+            param($SentryEvent)
+            $SentryEvent.dist | Should -Be "test-dist"
+        }
+    }
+    @{ Name = 'Has tags'; TestBlock = {
+            param($SentryEvent)
+            $SentryEvent.tags | Should -Not -BeNullOrEmpty
+        }
+    }
+    @{ Name = 'Has correct integration test tags'; TestBlock = {
+            param($SentryEvent, $TestType)
+            ($SentryEvent.tags | Where-Object { $_.key -eq 'test.suite' }).value | Should -Be 'integration'
+            ($SentryEvent.tags | Where-Object { $_.key -eq 'test.type' }).value | Should -Be $TestType
+        }
+    }
+    @{ Name = 'Has correct environment tag'; TestBlock = {
+            param($SentryEvent)
+            ($SentryEvent.tags | Where-Object { $_.key -eq 'environment' }).value | Should -Be 'integration-test'
+        }
+    }
+    @{ Name = 'Has correct OS tag'; TestBlock = {
+            param($SentryEvent)
+            if ($IsLinux) {
+                $expectedOS = "Linux"
+            } elseif ($IsMacOS) {
+                $expectedOS = "macOS"
+            } else {
+                $expectedOS = "Windows"
+            }
+            ($SentryEvent.tags | Where-Object { $_.key -eq 'os' }).value | Should -Match $expectedOS
+        }
+    }
+    @{ Name = 'Contains user information'; TestBlock = {
+            param($SentryEvent)
+            $SentryEvent.user | Should -Not -BeNullOrEmpty
+            $SentryEvent.user.username | Should -Be 'TestUser'
+            $SentryEvent.user.email | Should -Be '[email protected]'
+            $SentryEvent.user.id | Should -Be '12345'
+        }
+    }
+    @{ Name = 'Contains breadcrumbs'; TestBlock = {
+            param($SentryEvent)
+            $SentryEvent.breadcrumbs | Should -Not -BeNullOrEmpty
+            $SentryEvent.breadcrumbs.values | Should -Not -BeNullOrEmpty
+        }
+    }
+    @{ Name = 'Contains SDK information'; TestBlock = {
+            param($SentryEvent)
+            $SentryEvent.sdk | Should -Not -BeNullOrEmpty
+            $SentryEvent.sdk.name | Should -Not -BeNullOrEmpty
+            $SentryEvent.sdk.version | Should -Not -BeNullOrEmpty
+        }
+    }
+    @{ Name = 'Contains app context'; TestBlock = {
+            param($SentryEvent)
+            $SentryEvent.contexts.app | Should -Not -BeNullOrEmpty
+            $SentryEvent.contexts.app.app_name | Should -Not -BeNullOrEmpty
+            $SentryEvent.contexts.app.app_version | Should -Not -BeNullOrEmpty
+        }
+    }
+    @{ Name = 'Contains device context'; TestBlock = {
+            param($SentryEvent)
+            $SentryEvent.contexts.device | Should -Not -BeNullOrEmpty
+            $SentryEvent.contexts.device.arch | Should -Not -BeNullOrEmpty
+            $SentryEvent.contexts.device.free_memory | Should -Not -BeNullOrEmpty
+            $SentryEvent.contexts.device.usable_memory | Should -Not -BeNullOrEmpty
+        }
+    }
+    @{ Name = 'Contains OS context'; TestBlock = {
+            param($SentryEvent)
+            $SentryEvent.contexts.os | Should -Not -BeNullOrEmpty
+            $SentryEvent.contexts.os.os | Should -Not -BeNullOrEmpty
+            $SentryEvent.contexts.os.name | Should -Not -BeNullOrEmpty
+            $SentryEvent.contexts.os.version | Should -Not -BeNullOrEmpty
+        }
+    }
+    @{ Name = 'Contains Godot contexts'; TestBlock = {
+            param($SentryEvent)
+            $SentryEvent.contexts.godot_engine | Should -Not -BeNullOrEmpty
+            $SentryEvent.contexts.godot_engine.version | Should -Not -BeNullOrEmpty
+            $SentryEvent.contexts.godot_engine.version_commit | Should -Not -BeNullOrEmpty
+            $SentryEvent.contexts.godot_engine.godot_sdk_version | Should -Not -BeNullOrEmpty
+            $SentryEvent.contexts.godot_performance | Should -Not -BeNullOrEmpty
+        }
+    }
+    @{ Name = 'Contains expected breadcrumbs'; TestBlock = {
+            param($SentryEvent)
+            $SentryEvent.breadcrumbs.values | Should -Not -BeNullOrEmpty
+            $SentryEvent.breadcrumbs.values | Where-Object { $_.message -eq "Integration test started" } | Should -Not -BeNullOrEmpty
+            $SentryEvent.breadcrumbs.values | Where-Object { $_.message -eq "Context configuration finished" } | Should -Not -BeNullOrEmpty
+        }
+    }
+)