diff --git a/.github/workflows/android-app.yml b/.github/workflows/android-app.yml index a016fad97e16..d164622beec5 100644 --- a/.github/workflows/android-app.yml +++ b/.github/workflows/android-app.yml @@ -37,15 +37,22 @@ on: type: boolean required: false mockapi_test_repeat: - description: Mockapi test repeat(self hosted) + description: Mockapi test repeat (self hosted) default: '1' required: true type: string e2e_test_repeat: - description: e2e test repeat(self hosted) + description: e2e test repeat (self hosted) default: '0' required: true type: string + e2e_tests_infra_flavor: + description: > + Infra environment to run e2e tests on (prod/stagemole). + If set to 'stagemole' test-related artefacts will be uploaded. + default: 'stagemole' + required: true + type: string # Build if main is updated to ensure up-to-date caches are available push: branches: [main] @@ -341,7 +348,9 @@ jobs: - name: Build stagemole app uses: burrunan/gradle-cache-action@v1 - if: github.event.inputs.run_firebase_tests == 'true' + if: > + (github.event.inputs.e2e_test_repeat != '0' && github.event.inputs.e2e_tests_infra_flavor == 'stagemole') || + github.event.inputs.run_firebase_tests == 'true' with: job-id: jdk17 arguments: assemblePlayStagemoleDebug @@ -472,9 +481,18 @@ jobs: TEST_TYPE: ${{ matrix.test-type }} BILLING_FLAVOR: oss INFRA_FLAVOR: prod + TEST_DEVICE_OUTPUTS_DIR: '/sdcard/Download/test-outputs' REPORT_DIR: ${{ steps.prepare-report-dir.outputs.report_dir }} run: ./android/scripts/run-instrumented-tests-repeat.sh ${{ matrix.test-repeat }} + - name: Pull test report + if: always() && matrix.test-repeat != 0 && github.event.inputs.e2e_tests_infra_flavor == 'stagemole' + shell: bash -ieo pipefail {0} + env: + TEST_DEVICE_OUTPUTS_DIR: '/sdcard/Download/test-outputs' + REPORT_DIR: ${{ steps.prepare-report-dir.outputs.report_dir }} + run: ./android/scripts/pull-test-output.sh --test-type ${{ matrix.test-type }} + - name: Upload instrumentation report (${{ matrix.test-type }}) uses: actions/upload-artifact@v4 if: always() && matrix.test-repeat != 0 @@ -526,7 +544,7 @@ jobs: - name: Calculate timeout id: calculate-timeout - run: echo "timeout=$(( ${{ matrix.test-repeat }} * 10 ))" >> $GITHUB_OUTPUT + run: echo "timeout=$(( ${{ matrix.test-repeat }} * 15 ))" >> $GITHUB_OUTPUT shell: bash - name: Run instrumented test script @@ -536,15 +554,36 @@ jobs: env: AUTO_FETCH_TEST_HELPER_APKS: true TEST_TYPE: e2e - BILLING_FLAVOR: oss - INFRA_FLAVOR: prod - VALID_TEST_ACCOUNT_NUMBER: ${{ secrets.ANDROID_PROD_TEST_ACCOUNT }} + BILLING_FLAVOR: ${{ github.event.inputs.e2e_tests_infra_flavor == 'prod' && 'oss' || 'play' }} + INFRA_FLAVOR: ${{ github.event.inputs.e2e_tests_infra_flavor }} + PARTNER_AUTH: ${{ github.event.inputs.e2e_tests_infra_flavor == 'stagemole' && secrets.STAGEMOLE_PARTNER_AUTH || '' }} + VALID_TEST_ACCOUNT_NUMBER: ${{ github.event.inputs.e2e_tests_infra_flavor == 'prod' && secrets.ANDROID_PROD_TEST_ACCOUNT || '' }} INVALID_TEST_ACCOUNT_NUMBER: '0000000000000000' ENABLE_HIGHLY_RATE_LIMITED_TESTS: ${{ github.event_name == 'schedule' && 'true' || 'false' }} ENABLE_ACCESS_TO_LOCAL_API_TESTS: true + TEST_DEVICE_OUTPUTS_DIR: '/sdcard/Download/test-outputs' REPORT_DIR: ${{ steps.prepare-report-dir.outputs.report_dir }} run: ./android/scripts/run-instrumented-tests-repeat.sh ${{ matrix.test-repeat }} + - name: Pull test report + if: > + always() && matrix.test-repeat != 0 && + github.event.inputs.e2e_tests_infra_flavor == 'stagemole' + shell: bash -ieo pipefail {0} + env: + TEST_DEVICE_OUTPUTS_DIR: '/sdcard/Download/test-outputs' + REPORT_DIR: ${{ steps.prepare-report-dir.outputs.report_dir }} + run: ./android/scripts/pull-test-output.sh --test-type e2e + + - name: Upload e2e instrumentation report + uses: actions/upload-artifact@v4 + if: > + always() && matrix.test-repeat != 0 && + github.event.inputs.e2e_tests_infra_flavor == 'stagemole' + with: + name: e2e-instrumentation-report + path: ${{ steps.prepare-report-dir.outputs.report_dir }} + firebase-tests: name: Run firebase tests if: github.event.inputs.run_firebase_tests == 'true' diff --git a/android/scripts/pull-test-output.sh b/android/scripts/pull-test-output.sh new file mode 100755 index 000000000000..3b89f70effa5 --- /dev/null +++ b/android/scripts/pull-test-output.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +set -eu + +TEST_DEVICE_OUTPUTS_DIR="${TEST_DEVICE_OUTPUTS_DIR:-/sdcard/Download/test-outputs/attachments}" # Must match the path where e2e tests output their attachments +REPORT_DIR="${REPORT_DIR:-}" + +while [[ "$#" -gt 0 ]]; do + case $1 in + --test-type) + if [[ -n "${2-}" && "$2" =~ ^(app|mockapi|e2e)$ ]]; then + TEST_TYPE="$2" + else + echo "Error: Bad or missing test type. Must be one of: app, mockapi, e2e" + exit 1 + fi + shift 2 + ;; + *) + echo "Unknown argument: $1" + exit 1 + ;; + esac +done + +if [[ -z $TEST_DEVICE_OUTPUTS_DIR ]]; then + echo "" + echo "Error: The variable TEST_DEVICE_OUTPUTS_DIR must be set." + exit 1 +fi + +if [[ -z $REPORT_DIR || ! -d $REPORT_DIR ]]; then + echo "" + echo "Error: The variable REPORT_DIR must be set and the directory must exist." + exit 1 +fi + +echo "Collecting produced test attachments and logs..." +adb pull "$TEST_DEVICE_OUTPUTS_DIR" "$REPORT_DIR" diff --git a/android/scripts/run-instrumented-tests.sh b/android/scripts/run-instrumented-tests.sh index 8d835ddb5ab7..fa39b53962f2 100755 --- a/android/scripts/run-instrumented-tests.sh +++ b/android/scripts/run-instrumented-tests.sh @@ -2,6 +2,20 @@ set -eu + +cleanup() { + echo "### Trapped termination signal, clean up ###" + echo "### Store logcat and instrumentation log ###" + adb shell "mkdir -p $TEST_DEVICE_OUTPUTS_DIR" + adb shell "logcat -d > $TEST_DEVICE_OUTPUTS_DIR/logcat.txt" + + if [[ -n ${TEMP_DOWNLOAD_DIR-} ]]; then + rm -rf "$TEMP_DOWNLOAD_DIR" +fi +} + +trap cleanup SIGHUP EXIT + SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$SCRIPT_DIR" @@ -18,6 +32,7 @@ VALID_TEST_ACCOUNT_NUMBER="${VALID_TEST_ACCOUNT_NUMBER:-}" INVALID_TEST_ACCOUNT_NUMBER="${INVALID_TEST_ACCOUNT_NUMBER:-}" ENABLE_HIGHLY_RATE_LIMITED_TESTS="${ENABLE_HIGHLY_RATE_LIMITED_TESTS:-false}" ENABLE_ACCESS_TO_LOCAL_API_TESTS="${ENABLE_ACCESS_TO_LOCAL_API_TESTS:-false}" +TEST_DEVICE_OUTPUTS_DIR="${TEST_DEVICE_OUTPUTS_DIR:-/sdcard/Download/test-outputs/attachments}" # Must match the path where e2e tests output their attachments REPORT_DIR="${REPORT_DIR:-}" while [[ "$#" -gt 0 ]]; do @@ -72,6 +87,7 @@ if [[ -z ${BILLING_FLAVOR-} ]]; then fi echo "### Configuration ###" +echo "Test device outputs dir: $TEST_DEVICE_OUTPUTS_DIR" echo "Report dir: $REPORT_DIR" echo "Test type: $TEST_TYPE" echo "Infra flavor: $INFRA_FLAVOR" @@ -144,6 +160,12 @@ case "$TEST_TYPE" in ;; esac +if [[ -z $TEST_DEVICE_OUTPUTS_DIR ]]; then + echo "" + echo "Error: The variable TEST_DEVICE_OUTPUTS_DIR must be set." + exit 1 +fi + if [[ -z $REPORT_DIR || ! -d $REPORT_DIR ]]; then echo "" echo "Error: The variable REPORT_DIR must be set and the directory must exist." @@ -153,17 +175,14 @@ fi GRADLE_ENVIRONMENT_VARIABLES="TEST_E2E_ENABLEACCESSTOLOCALAPITESTS=$ENABLE_ACCESS_TO_LOCAL_API_TESTS" INSTRUMENTATION_LOG_FILE_PATH="$REPORT_DIR/instrumentation-log.txt" -LOGCAT_FILE_PATH="$REPORT_DIR/logcat.txt" -LOCAL_SCREENSHOT_PATH="$REPORT_DIR/screenshots" DEVICE_SCREENSHOT_PATH="/sdcard/Pictures/mullvad-$TEST_TYPE" -DEVICE_TEST_ATTACHMENTS_PATH="/sdcard/Download/test-attachments" echo "" echo "### Ensure clean report structure ###" rm -rf "${REPORT_DIR:?}/*" adb logcat --clear adb shell rm -rf "$DEVICE_SCREENSHOT_PATH" -adb shell rm -rf "$DEVICE_TEST_ATTACHMENTS_PATH" +adb shell rm -rf "$TEST_DEVICE_OUTPUTS_DIR" echo "" if [[ "${USE_ORCHESTRATOR-}" == "true" ]]; then @@ -238,13 +257,5 @@ echo "### Checking logs for success message ###" if grep -q -E "$LOG_SUCCESS_REGEX" "$INSTRUMENTATION_LOG_FILE_PATH"; then echo "Success, no failures!" else - echo "One or more tests failed, see logs for more details." - echo "Collecting report..." - adb pull "$DEVICE_SCREENSHOT_PATH" "$LOCAL_SCREENSHOT_PATH" || echo "No screenshots" - adb logcat -d > "$LOGCAT_FILE_PATH" exit 1 fi - -if [[ -n ${TEMP_DOWNLOAD_DIR-} ]]; then - rm -rf "$TEMP_DOWNLOAD_DIR" -fi diff --git a/android/test/common/src/main/kotlin/net/mullvad/mullvadvpn/test/common/misc/Attachment.kt b/android/test/common/src/main/kotlin/net/mullvad/mullvadvpn/test/common/misc/Attachment.kt index 57e6ab542b89..78246348c55a 100644 --- a/android/test/common/src/main/kotlin/net/mullvad/mullvadvpn/test/common/misc/Attachment.kt +++ b/android/test/common/src/main/kotlin/net/mullvad/mullvadvpn/test/common/misc/Attachment.kt @@ -7,11 +7,12 @@ import java.io.IOException import org.junit.jupiter.api.fail object Attachment { - private const val DIRECTORY_NAME = "test-attachments" + val DIRECTORY_PATH = "${Environment.DIRECTORY_DOWNLOADS}/test-outputs/attachments" + private val testAttachmentsDirectory = File( Environment.getExternalStorageDirectory(), - "${Environment.DIRECTORY_DOWNLOADS}/$DIRECTORY_NAME", + DIRECTORY_PATH, ) fun saveAttachment(fileName: String, data: ByteArray) { diff --git a/android/test/common/src/main/kotlin/net/mullvad/mullvadvpn/test/common/misc/CaptureScreenRecordingsExtension.kt b/android/test/common/src/main/kotlin/net/mullvad/mullvadvpn/test/common/misc/CaptureScreenRecordingsExtension.kt index 21cc915482e6..5c82b8ed6f5f 100644 --- a/android/test/common/src/main/kotlin/net/mullvad/mullvadvpn/test/common/misc/CaptureScreenRecordingsExtension.kt +++ b/android/test/common/src/main/kotlin/net/mullvad/mullvadvpn/test/common/misc/CaptureScreenRecordingsExtension.kt @@ -55,6 +55,6 @@ class CaptureScreenRecordingsExtension : BeforeEachCallback, AfterEachCallback { companion object { val OUTPUT_DIRECTORY = - "${Environment.getExternalStorageDirectory().path}/Download/test-attachments/video" + "${Environment.getExternalStorageDirectory().path}/Download/test-outputs/attachments/video" } } diff --git a/android/test/common/src/main/kotlin/net/mullvad/mullvadvpn/test/common/rule/CaptureScreenshotOnFailedTestRule.kt b/android/test/common/src/main/kotlin/net/mullvad/mullvadvpn/test/common/rule/CaptureScreenshotOnFailedTestRule.kt index ecabb3c60a15..dac98d1b4d19 100644 --- a/android/test/common/src/main/kotlin/net/mullvad/mullvadvpn/test/common/rule/CaptureScreenshotOnFailedTestRule.kt +++ b/android/test/common/src/main/kotlin/net/mullvad/mullvadvpn/test/common/rule/CaptureScreenshotOnFailedTestRule.kt @@ -5,6 +5,7 @@ import android.content.ContentValues import android.graphics.Bitmap import android.os.Build import android.os.Environment +import android.os.Environment.DIRECTORY_DOWNLOADS import android.os.Environment.DIRECTORY_PICTURES import android.provider.MediaStore import androidx.annotation.RequiresApi @@ -16,6 +17,7 @@ import java.io.IOException import java.nio.file.Paths import java.time.OffsetDateTime import java.time.temporal.ChronoUnit +import net.mullvad.mullvadvpn.test.common.misc.Attachment import org.junit.jupiter.api.extension.ExtensionContext import org.junit.jupiter.api.extension.TestWatcher @@ -55,7 +57,7 @@ class CaptureScreenshotOnFailedTestRule(private val testTag: String) : TestWatch ) { contentValues.apply { put(MediaStore.MediaColumns.DISPLAY_NAME, filename) - put(MediaStore.Images.Media.RELATIVE_PATH, "$DIRECTORY_PICTURES/$baseDir") + put(MediaStore.Images.Media.RELATIVE_PATH, "${Attachment.DIRECTORY_PATH}/$baseDir") } val uri = @@ -78,7 +80,7 @@ class CaptureScreenshotOnFailedTestRule(private val testTag: String) : TestWatch private fun Bitmap.writeToExternalStorage(baseDir: String, filename: String) { val screenshotBaseDirectory = Paths.get( - Environment.getExternalStoragePublicDirectory(DIRECTORY_PICTURES).path, + Environment.getExternalStoragePublicDirectory(Attachment.DIRECTORY_PATH).path, baseDir, ) .toFile() diff --git a/android/test/e2e/README.md b/android/test/e2e/README.md index adbcc042cde7..86d0733bf081 100644 --- a/android/test/e2e/README.md +++ b/android/test/e2e/README.md @@ -58,4 +58,4 @@ docker run --rm --volumes-from gcloud-config -v ${PWD}:/android gcr.io/google.co ``` ## Test artefacts -Test artefacts are stored on the test device in `/sdcard/Download/test-attachments`. In CI this directory is cleared in between each test run, but note that when running tests locally the directory isn't cleared but already existing files are overwritten. +Test artefacts are stored on the test device in `/sdcard/Download/test-outputs`. In CI this directory is cleared in between each test run, but note that when running tests locally the directory isn't cleared but already existing files are overwritten. diff --git a/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/misc/SimpleMullvadHttpClient.kt b/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/misc/SimpleMullvadHttpClient.kt index b5dcc1d64790..c8ccc60ddb4d 100644 --- a/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/misc/SimpleMullvadHttpClient.kt +++ b/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/misc/SimpleMullvadHttpClient.kt @@ -17,6 +17,7 @@ import net.mullvad.mullvadvpn.test.e2e.constant.DEVICE_LIST_URL import net.mullvad.mullvadvpn.test.e2e.constant.PARTNER_ACCOUNT_URL import org.json.JSONArray import org.json.JSONObject +import org.junit.jupiter.api.fail class SimpleMullvadHttpClient(context: Context) { @@ -201,6 +202,10 @@ class SimpleMullvadHttpClient(context: Context) { private val onErrorResponse = { error: VolleyError -> if (error.networkResponse != null) { + if (error.networkResponse.statusCode == 429) { + fail("Request failed with response status code 429: Too many requests") + } + Logger.e( "Response returned error message: ${error.message} " + "status code: ${error.networkResponse.statusCode}"