Re-enable nightly firefox tests (#7954)

* Re-enable nightly firefox tests * Use ios 15 webgl2 for layers tests --------- Co-authored-by: Ping Yu <[email protected]>
tensorflow · Sep 13, 2023 · 0bd03f7 · 0bd03f7
1 parent 8879e72
commit 0bd03f7
Show file tree

Hide file tree

Showing 13 changed files with 26 additions and 34 deletions.
diff --git a/e2e/scripts/run-browserstack-tests.sh b/e2e/scripts/run-browserstack-tests.sh
@@ -39,8 +39,7 @@ if [[ "$NIGHTLY" = true || "$RELEASE" = true ]]; then
   COMMANDS+=(
     "yarn run-browserstack --browsers=bs_ios_12 --tags '$TAGS' --testEnv webgl --flags '{\"\\"\"WEBGL_VERSION\"\\"\": 1, \"\\"\"WEBGL_CPU_FORWARD\"\\"\": false, \"\\"\"WEBGL_SIZE_UPLOAD_UNIFORM\"\\"\": 0}'"
     "yarn run-browserstack --browsers=bs_safari_mac --tags '$TAGS' --testEnv webgl --flags '{\"\\"\"WEBGL_VERSION\"\\"\": 1, \"\\"\"WEBGL_CPU_FORWARD\"\\"\": false, \"\\"\"WEBGL_SIZE_UPLOAD_UNIFORM\"\\"\": 0}'"
-    # TODO(mattSoulanille): Re-enable firefox once it works on browserstack.
-    # "yarn run-browserstack --browsers=bs_firefox_mac --tags '$TAGS'"
+    "yarn run-browserstack --browsers=bs_firefox_mac --tags '$TAGS'"
     "yarn run-browserstack --browsers=bs_android_10 --tags '$TAGS'"
     # Test script tag bundles
     "karma start ./script_tag_tests/tfjs-core-cpu/karma.conf.js --browserstack --browsers=bs_chrome_mac"

diff --git a/tfjs-automl/scripts/test-ci.sh b/tfjs-automl/scripts/test-ci.sh
@@ -27,7 +27,5 @@ yarn run-flaky "yarn run-browserstack --browsers=bs_chrome_mac"
 # Run the rest of the karma tests in parallel. These runs will reuse the
 # already downloaded binary.
 npm-run-all -p -c --aggregate-output \
+  "run-flaky \"yarn run-browserstack --browsers=bs_firefox_mac\"" \
   "run-flaky \"yarn run-browserstack --browsers=bs_safari_mac --testEnv webgl1\""
-# TODO(mattSoulanille): Re-enable firefox once it works on browserstack.
-# "run-flaky \"yarn run-browserstack --browsers=bs_firefox_mac\"" \
-
diff --git a/tfjs-backend-wasm/BUILD.bazel b/tfjs-backend-wasm/BUILD.bazel
@@ -151,8 +151,7 @@ tfjs_web_test(
     ],
     browsers = [
         "bs_chrome_mac",
-        # TODO(mattSoulanille): Re-enable firefox once it works on browserstack.
-        # "bs_firefox_mac",
+        "bs_firefox_mac",
         "bs_safari_mac",
         "bs_ios_12",
         # TODO(mattsoulanille): Fix clipByValue on Android.

diff --git a/tfjs-backend-webgl/BUILD.bazel b/tfjs-backend-webgl/BUILD.bazel
@@ -106,10 +106,9 @@ tfjs_web_test(
         "webgl2",
     ],
     browsers = [
-        "bs_chrome_mac",
-        # TODO(mattSoulanille): Re-enable firefox once it works on browserstack.
-        # "bs_firefox_mac",
         "bs_android_10",
+        "bs_chrome_mac",
+        "bs_firefox_mac",
         "win_10_chrome",
     ],
     headless = False,
@@ -130,8 +129,8 @@ tfjs_web_test(
         "webgl1",
     ],
     browsers = [
-        "bs_safari_mac",
         "bs_ios_12",
+        "bs_safari_mac",
     ],
     headless = False,
     presubmit_browsers = [

diff --git a/tfjs-core/BUILD.bazel b/tfjs-core/BUILD.bazel
@@ -110,12 +110,11 @@ tfjs_web_test(
         "//tfjs-core/src:tfjs-core_test_bundle",
     ],
     browsers = [
+        "bs_android_10",
         "bs_chrome_mac",
-        # TODO(mattSoulanille): Re-enable firefox once it works on browserstack.
-        # "bs_firefox_mac",
-        "bs_safari_mac",
+        "bs_firefox_mac",
         "bs_ios_12",
-        "bs_android_10",
+        "bs_safari_mac",
         "win_10_chrome",
     ],
     static_files = [

diff --git a/tfjs-core/src/BUILD.bazel b/tfjs-core/src/BUILD.bazel
@@ -222,8 +222,7 @@ tfjs_web_test(
     name = "worker_test",
     browsers = [
         "bs_chrome_mac",
-        # TODO(mattSoulanille): Re-enable firefox once it works on browserstack.
-        # "bs_firefox_mac",
+        "bs_firefox_mac",
         "bs_safari_mac",
         # Temporarily disabled because BrowserStack does not support loading
         # absolute paths in iOS, which is required for loading the worker.

diff --git a/tfjs-layers/BUILD.bazel b/tfjs-layers/BUILD.bazel
@@ -51,11 +51,11 @@ tfjs_web_test(
     ],
     browsers = [
         "bs_chrome_mac",
-        # TODO(mattSoulanille): Re-enable firefox once it works on browserstack.
-        # "bs_firefox_mac",
+        "bs_firefox_mac",
         # disabled android test due to training flakiness
         # "bs_android_10",
         "win_10_chrome",
+        "bs_ios_15",
     ],
     headless = False,
     seed = "12345",
@@ -76,7 +76,6 @@ tfjs_web_test(
     ],
     browsers = [
         "bs_safari_mac",
-        "bs_ios_12",
     ],
     headless = False,
     seed = "12345",

diff --git a/tfjs-tfdf/BUILD.bazel b/tfjs-tfdf/BUILD.bazel
@@ -107,8 +107,7 @@ tfjs_web_test(
     browsers = [
         # TODO: Support Safari.
         "bs_chrome_mac",
-        # TODO(mattSoulanille): Re-enable firefox once it works on browserstack.
-        # "bs_firefox_mac",
+        "bs_firefox_mac",
         "bs_android_10",
         "win_10_chrome",
     ],

diff --git a/tfjs-tflite/src/BUILD.bazel b/tfjs-tflite/src/BUILD.bazel
@@ -104,8 +104,7 @@ tfjs_web_test(
     name = "worker_test",
     browsers = [
         "bs_chrome_mac",
-        # TODO(mattSoulanille): Re-enable firefox once it works on browserstack.
-        # "bs_firefox_mac",
+        "bs_firefox_mac",
         "bs_safari_mac",
         # Temporarily disabled because BrowserStack does not support loading
         # absolute paths in iOS, which is required for loading the worker.

diff --git a/tfjs-vis/scripts/test-ci.sh b/tfjs-vis/scripts/test-ci.sh
@@ -26,9 +26,6 @@ yarn run-flaky "yarn run-browserstack --browsers=bs_chrome_mac"
 
 # Run the rest of the karma tests in parallel. These runs will reuse the
 # already downloaded binary.
-yarn run-flaky "yarn run-browserstack --browsers=bs_safari_mac"
-#npm-run-all -p -c --aggregate-output \
-  #"run-flaky \"yarn run-browserstack --browsers=bs_safari_mac\""
-  # TODO(mattSoulanille): Re-enable firefox once it works on browserstack.
-  # "run-flaky \"yarn run-browserstack --browsers=bs_firefox_mac\"" \
-
+npm-run-all -p -c --aggregate-output \
+  "run-flaky \"yarn run-browserstack --browsers=bs_firefox_mac\"" \
+  "run-flaky \"yarn run-browserstack --browsers=bs_safari_mac\""
diff --git a/tfjs/scripts/test-ci.sh b/tfjs/scripts/test-ci.sh
@@ -17,8 +17,7 @@
 # Exit immediately if a command exits with a non-zero status.
 set -e
 
-# TODO(mattSoulanille): Re-enable firefox once it works on browserstack.
-# node ../scripts/run_flaky.js "yarn karma start --browsers='bs_firefox_mac' --singleRun"
+node ../scripts/run_flaky.js "yarn karma start --browsers='bs_firefox_mac' --singleRun"
 node ../scripts/run_flaky.js "yarn karma start --browsers='bs_chrome_mac' --singleRun"
 yarn test-tools
 

diff --git a/tools/karma_template.conf.js b/tools/karma_template.conf.js
@@ -62,6 +62,13 @@ const CUSTOM_LAUNCHERS = {
     os_version: '12.3',
     real_mobile: true
   },
+  bs_ios_15: {
+    base: 'BrowserStack',
+    device: 'iPhone 11 Pro',
+    os: 'ios',
+    os_version: '15',
+    real_mobile: true
+  },
   bs_android_10: {
     base: 'BrowserStack',
     device: 'Google Pixel 4 XL',

diff --git a/tools/tfjs_web_test.bzl b/tools/tfjs_web_test.bzl
@@ -122,8 +122,7 @@ def tfjs_web_test(name, ci = True, args = [], **kwargs):
 
     browsers = kwargs.pop("browsers", [
         "bs_chrome_mac",
-        # TODO(mattSoulanille): Re-enable firefox once it works on browserstack.
-        # "bs_firefox_mac",
+        "bs_firefox_mac",
         "bs_safari_mac",
         "bs_ios_12",
         "bs_android_10",