[java] tag os-sensitive browser tests per file and run them on Windows and macOS

.github/workflows/ci-java.yml

@@ -34,14 +34,17 @@ jobs:
       needs-display: true
       rerun-with-debug: true
       run: >
-        bazel test --flaky_test_attempts 2
+        bazel test
+        --build_tests_only
+        --keep_going
+        --flaky_test_attempts 2
+        --local_test_jobs 1
+        --pin_browsers=false
         ${{ inputs.compile-manager && '--manager=host' || '' }}
-        //java/test/org/openqa/selenium/chrome:ChromeDriverFunctionalTest
-        //java/test/org/openqa/selenium/federatedcredentialmanagement:FederatedCredentialManagementTest
-        //java/test/org/openqa/selenium/firefox:FirefoxDriverBuilderTest
-        //java/test/org/openqa/selenium/grid/router:RemoteWebDriverDownloadTest
-        //java/test/org/openqa/selenium/remote:RemoteWebDriverBuilderTest
-        //java/test/org/openqa/selenium/devtools:NetworkInterceptorRestTest
+        --test_env=SE_FORCE_BROWSER_DOWNLOAD=true
+        --test_env=SE_SKIP_DRIVER_IN_PATH=true
+        --test_tag_filters=os-sensitive
+        //java/...
 
   browser-tests-macos:
     name: Browser Tests
@@ -57,9 +60,14 @@ jobs:
       needs-display: true
       rerun-with-debug: true
       run: >
-        bazel test --flaky_test_attempts 2
-        //java/test/org/openqa/selenium/chrome:ChromeDriverFunctionalTest-remote
-        //java/test/org/openqa/selenium/federatedcredentialmanagement:FederatedCredentialManagementTest
-        //java/test/org/openqa/selenium/firefox:FirefoxDriverBuilderTest
-        //java/test/org/openqa/selenium/grid/router:RemoteWebDriverDownloadTest
-        //java/test/org/openqa/selenium/remote:RemoteWebDriverBuilderTest
+        bazel test
+        --build_tests_only
+        --keep_going
+        --flaky_test_attempts 2
+        --local_test_jobs 1
+        --pin_browsers=false
+        ${{ inputs.compile-manager && '--manager=host' || '' }}
+        --test_env=SE_FORCE_BROWSER_DOWNLOAD=true
+        --test_env=SE_SKIP_DRIVER_IN_PATH=true
+        --test_tag_filters=os-sensitive
+        //java/...

java/private/selenium_test.bzl

@@ -85,10 +85,11 @@ def selenium_test(name, test_class, size = "medium", browsers = DEFAULT_BROWSERS
     jvm_flags = kwargs["jvm_flags"] if "jvm_flags" in kwargs else []
     tags = kwargs["tags"] if "tags" in kwargs else []
 
-    remote = False
-    if "selenium-remote" in tags:
-        tags.remove("selenium-remote")
-        remote = True
+    remote = "selenium-remote" in tags
+
+    # These belong only to the stable local target of each browser family.
+    stable_local_tags = [t for t in tags if t in ["os-sensitive"]]
+    tags = [t for t in tags if t != "selenium-remote" and t not in stable_local_tags]
 
     stripped_args = dict(**kwargs)
     stripped_args.pop("data", None)
@@ -103,14 +104,15 @@ def selenium_test(name, test_class, size = "medium", browsers = DEFAULT_BROWSERS
             fail("Unrecognized browser: " + browser)
 
         test = name if browser == default_browser else "%s-%s" % (name, browser)
+        local_tags = stable_local_tags if browser == browser.split("-")[0] else []
 
         junit5_test(
             name = test,
             test_class = test_class,
             size = size,
             jvm_flags = BROWSERS[browser]["jvm_flags"] + jvm_flags,
             # Only allow linting on the default test
-            tags = BROWSERS[browser]["tags"] + tags + ([] if test == name else ["no-lint"]),
+            tags = BROWSERS[browser]["tags"] + tags + local_tags + ([] if test == name else ["no-lint"]),
             data = BROWSERS[browser]["data"] + data,
             env_inherit = inherited_env,
             target_compatible_with = BROWSERS[browser]["target_compatible_with"],

java/private/suite.bzl

@@ -37,7 +37,12 @@ def _generate_test_suite(
         deps,
         tags,
         test_identifiers,
+        extra_tags = {},
         **kwargs):
+    unknown = [src for src in extra_tags if src not in srcs]
+    if unknown:
+        fail("extra_tags keys not in srcs of %s: %s" % (name, unknown))
+
     lib_srcs = [src for src in srcs if not _matches(test_identifiers, src)]
     test_srcs = [src for src in srcs if _matches(test_identifiers, src)]
 
@@ -71,7 +76,7 @@ def _generate_test_suite(
             size = size,
             srcs = [test],
             deps = deps,
-            tags = tags + additional_tags,
+            tags = tags + additional_tags + extra_tags.get(test, []),
             **kwargs
         )
 
@@ -135,6 +140,7 @@ def java_selenium_test_suite(
         deps = None,
         jvm_flags = [],
         tags = [],
+        extra_tags = {},
         **kwargs):
     args = {}
     args.update(kwargs)
@@ -148,5 +154,6 @@ def java_selenium_test_suite(
         deps = deps,
         tags = tags,
         test_identifiers = test_identifiers,
+        extra_tags = extra_tags,
         **args
     )

java/test/org/openqa/selenium/BUILD.bazel

@@ -68,6 +68,11 @@ java_selenium_test_suite(
     env_inherit = [
         "DISPLAY",
     ],
+    extra_tags = {
+        "PositionAndSizeTest.java": ["os-sensitive"],
+        "TakesScreenshotTest.java": ["os-sensitive"],
+        "WindowTest.java": ["os-sensitive"],
+    },
     javacopts = [
         "--release",
         TOOLS_JAVA_VERSION,

java/test/org/openqa/selenium/PositionAndSizeTest.java

@@ -82,6 +82,7 @@ public void testShouldGetCoordinatesOfAnInvisibleElement() {
   }
 
   @Test
+  @NotYetImplemented(SAFARI)
   void testShouldScrollPageAndGetCoordinatesOfAnElementThatIsOutOfViewPort() {
     driver.get(appServer.whereIs("coordinates_tests/page_with_element_out_of_view.html"));
     int windowHeight = driver.manage().window().getSize().getHeight();
@@ -94,7 +95,6 @@ void testShouldScrollPageAndGetCoordinatesOfAnElementThatIsOutOfViewPort() {
 
   @SwitchToTopAfterTest
   @Test
-  @NotYetImplemented(SAFARI)
   public void testShouldGetCoordinatesOfAnElementInAFrame() {
     driver.get(appServer.whereIs("coordinates_tests/element_in_frame.html"));
     driver.switchTo().frame("ifr");
@@ -133,6 +133,7 @@ public void testShouldGetCoordinatesInViewPortOfAnElementInANestedFrame() {
   }
 
   @Test
+  @NotYetImplemented(SAFARI)
   void testShouldGetCoordinatesOfAnElementWithFixedPosition() {
     driver.get(appServer.whereIs("coordinates_tests/page_with_fixed_element.html"));
     assertThat(getLocationInViewPort(By.id("fixed")).getY()).isZero();

java/test/org/openqa/selenium/TakesScreenshotTest.java

@@ -24,6 +24,7 @@
 import static org.openqa.selenium.support.ui.ExpectedConditions.visibilityOfAllElementsLocatedBy;
 import static org.openqa.selenium.testing.drivers.Browser.CHROME;
 import static org.openqa.selenium.testing.drivers.Browser.EDGE;
+import static org.openqa.selenium.testing.drivers.Browser.SAFARI;
 
 import java.awt.image.BufferedImage;
 import java.awt.image.Raster;
@@ -105,6 +106,9 @@ void testGetScreenshotAsBinary() {
 
   @Test
   @Ignore(value = CHROME, gitHubActions = true)
+  @Ignore(
+      value = SAFARI,
+      reason = "Safari applies the display color profile, so colors do not match exactly")
   public void testShouldCaptureScreenshotOfCurrentViewport() throws IOException {
     driver.get(appServer.whereIs("screen/screen.html"));
 
@@ -144,6 +148,9 @@ void testShouldCaptureScreenshotOfAnElement() throws Exception {
 
   @Test
   @Ignore(value = CHROME, gitHubActions = true)
+  @Ignore(
+      value = SAFARI,
+      reason = "Safari applies the display color profile, so colors do not match exactly")
   public void testShouldCaptureScreenshotAtFramePage() throws IOException {
     driver.get(appServer.whereIs("screen/screen_frames.html"));
     wait.until(frameToBeAvailableAndSwitchToIt(By.id("frame1")));
@@ -181,6 +188,9 @@ public void testShouldCaptureScreenshotAtFramePage() throws IOException {
   @Test
   @Ignore(CHROME)
   @Ignore(EDGE)
+  @Ignore(
+      value = SAFARI,
+      reason = "Safari applies the display color profile, so colors do not match exactly")
   public void testShouldCaptureScreenshotAtIFramePage() throws IOException {
     driver.get(appServer.whereIs("screen/screen_iframes.html"));
 

java/test/org/openqa/selenium/WindowTest.java

@@ -25,15 +25,26 @@
 import static org.openqa.selenium.testing.drivers.Browser.SAFARI;
 
 import java.util.function.Consumer;
+import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.openqa.selenium.support.ui.ExpectedCondition;
 import org.openqa.selenium.testing.Ignore;
 import org.openqa.selenium.testing.JupiterTestBase;
+import org.openqa.selenium.testing.NoDriverAfterTest;
 import org.openqa.selenium.testing.SwitchToTopAfterTest;
 import org.openqa.selenium.testing.TestUtilities;
 
 class WindowTest extends JupiterTestBase {
 
+  // The driver is shared, so undo any fullscreen or screen-sized window an earlier test left behind
+  @BeforeEach
+  void resetWindow() {
+    if (!TestUtilities.getEffectivePlatform(driver).is(ANDROID)) {
+      driver.manage().window().setPosition(new Point(50, 50));
+      changeSizeTo(new Dimension(600, 500));
+    }
+  }
+
   @Test
   void testGetsTheSizeOfTheCurrentWindow() {
     Dimension size = driver.manage().window().getSize();
@@ -116,27 +127,26 @@ void testSetsThePositionOfTheCurrentWindow() {
 
   @Test
   @Ignore(value = FIREFOX, gitHubActions = true)
+  @NoDriverAfterTest
   public void testCanMaximizeTheWindow() {
     // Browser window cannot be resized or moved on ANDROID (and most mobile platforms
     // though others aren't defined in org.openqa.selenium.Platform).
     assumeFalse(TestUtilities.getEffectivePlatform(driver).is(ANDROID));
 
-    changeSizeTo(new Dimension(640, 323));
     enlargeBy(WebDriver.Window::maximize);
   }
 
   @SwitchToTopAfterTest
   @Test
   @Ignore(value = CHROME, gitHubActions = true)
   @Ignore(value = FIREFOX, gitHubActions = true)
+  @NoDriverAfterTest
   public void testCanMaximizeTheWindowFromFrame() {
     // Browser window cannot be resized or moved on ANDROID (and most mobile platforms
     // though others aren't defined in org.openqa.selenium.Platform).
     assumeFalse(TestUtilities.getEffectivePlatform(driver).is(ANDROID));
 
     driver.get(pages.framesetPage);
-    changeSizeTo(new Dimension(640, 324));
-
     driver.switchTo().frame("fourth");
     enlargeBy(WebDriver.Window::maximize);
   }
@@ -145,20 +155,20 @@ public void testCanMaximizeTheWindowFromFrame() {
   @Test
   @Ignore(value = CHROME, gitHubActions = true)
   @Ignore(value = FIREFOX, gitHubActions = true)
+  @NoDriverAfterTest
   public void testCanMaximizeTheWindowFromIframe() {
     // Browser window cannot be resized or moved on ANDROID (and most mobile platforms
     // though others aren't defined in org.openqa.selenium.Platform).
     assumeFalse(TestUtilities.getEffectivePlatform(driver).is(ANDROID));
 
     driver.get(pages.iframePage);
-    changeSizeTo(new Dimension(640, 400));
-
     driver.switchTo().frame("iframe1-name");
     enlargeBy(WebDriver.Window::maximize);
   }
 
   @Test
   @Ignore(gitHubActions = true)
+  @NoDriverAfterTest
   public void canMinimizeTheWindow() {
     // Browser window cannot be resized or moved on ANDROID (and most mobile platforms
     // though others aren't defined in org.openqa.selenium.Platform).
@@ -174,6 +184,7 @@ public void canMinimizeTheWindow() {
   @Test
   @Ignore(value = FIREFOX, gitHubActions = true)
   @Ignore(SAFARI)
+  @NoDriverAfterTest
   public void canFullscreenTheWindow() {
     // Browser window cannot be resized or moved on ANDROID (and most mobile platforms
     // though others aren't defined in org.openqa.selenium.Platform).
@@ -191,6 +202,7 @@ public void canFullscreenTheWindow() {
   @Test
   @Ignore(value = FIREFOX, gitHubActions = true)
   @Ignore(SAFARI)
+  @NoDriverAfterTest
   public void canFullscreenTheWindowFromFrame() {
     // Browser window cannot be resized or moved on ANDROID (and most mobile platforms
     // though others aren't defined in org.openqa.selenium.Platform).
@@ -207,6 +219,7 @@ public void canFullscreenTheWindowFromFrame() {
   @Test
   @Ignore(value = FIREFOX, gitHubActions = true)
   @Ignore(SAFARI)
+  @NoDriverAfterTest
   public void canFullscreenTheWindowFromIframe() {
     // Browser window cannot be resized or moved on ANDROID (and most mobile platforms
     // though others aren't defined in org.openqa.selenium.Platform).

java/test/org/openqa/selenium/firefox/BUILD.bazel

@@ -56,6 +56,9 @@ java_selenium_test_suite(
     data = [
         "//common/extensions",
     ],
+    extra_tags = {
+        "TakesFullPageScreenshotTest.java": ["os-sensitive"],
+    },
     resources = [
         "//common/extensions",
     ],

java/test/org/openqa/selenium/interactions/BUILD.bazel

@@ -30,6 +30,13 @@ java_selenium_test_suite(
         ["*Test.java"],
         exclude = SMALL_TESTS,
     ),
+    extra_tags = {
+        "CombinedInputActionsTest.java": ["os-sensitive"],
+        "DefaultKeyboardTest.java": ["os-sensitive"],
+        "DefaultMouseTest.java": ["os-sensitive"],
+        "DefaultWheelTest.java": ["os-sensitive"],
+        "DragAndDropTest.java": ["os-sensitive"],
+    },
     javacopts = [
         "--release",
         TOOLS_JAVA_VERSION,

java/test/org/openqa/selenium/interactions/CombinedInputActionsTest.java

@@ -167,6 +167,7 @@ public void testControlClickingOnMultiSelectionList() {
 
   @Test
   @Ignore(IE)
+  @NotYetImplemented(SAFARI)
   public void testControlClickingOnCustomMultiSelectionList() {
     driver.get(pages.selectableItemsPage);
     Keys key = getEffectivePlatform(driver).is(Platform.MAC) ? Keys.COMMAND : Keys.CONTROL;
@@ -200,6 +201,7 @@ public void testControlClickingOnCustomMultiSelectionList() {
   @Test
   @Ignore(IE)
   @NotYetImplemented(FIREFOX)
+  @NotYetImplemented(SAFARI)
   public void testControlClickingWithMultiplePointers() {
     driver.get(pages.selectableItemsPage);
 
@@ -368,7 +370,6 @@ public void testCombiningShiftAndClickResultsInANewWindow() {
 
   @Test
   @Ignore(IE)
-  @NotYetImplemented(SAFARI)
   public void testHoldingDownShiftKeyWhileClicking() {
     driver.get(pages.clickEventPage);
 

java/test/org/openqa/selenium/interactions/DefaultMouseTest.java

@@ -88,7 +88,6 @@ private void performDragAndDropWithMouse() {
   }
 
   @Test
-  @NotYetImplemented(SAFARI)
   public void testDraggingElementWithMouseMovesItToAnotherList() {
     performDragAndDropWithMouse();
     WebElement dragInto = wait.until(visibilityOfElementLocated(By.id("sortable1")));
@@ -98,7 +97,6 @@ public void testDraggingElementWithMouseMovesItToAnotherList() {
   // This test is very similar to testDraggingElementWithMouse. The only
   // difference is that this test also verifies the correct events were fired.
   @Test
-  @NotYetImplemented(SAFARI)
   public void testDraggingElementWithMouseFiresEvents() {
     performDragAndDropWithMouse();
     WebElement dragReporter = driver.findElement(By.id("dragging_reports"));
@@ -120,7 +118,6 @@ void testDoubleClickThenGet() {
   }
 
   @Test
-  @NotYetImplemented(SAFARI)
   public void testDragAndDrop() {
     driver.get(pages.droppableItems);
 
@@ -139,7 +136,6 @@ public void testDragAndDrop() {
   }
 
   @Test
-  @NotYetImplemented(SAFARI)
   public void testDoubleClick() {
     driver.get(pages.javascriptPage);
 
@@ -203,7 +199,6 @@ void testCannotMoveToANullLocator() {
   }
 
   @Test
-  @NotYetImplemented(SAFARI)
   public void testMovingPastViewPortThrowsException() {
     assertThatExceptionOfType(MoveTargetOutOfBoundsException.class)
         .isThrownBy(() -> getBuilder(driver).moveByOffset(-1000, -1000).perform());
@@ -326,7 +321,6 @@ public void testMoveRelativeToBody() {
   }
 
   @Test
-  @NotYetImplemented(SAFARI)
   public void testMoveMouseByOffsetOverAndOutOfAnElement() {
     driver.get(pages.mouseOverPage);
 
@@ -361,7 +355,6 @@ public void testMoveMouseByOffsetOverAndOutOfAnElement() {
   }
 
   @Test
-  @NotYetImplemented(SAFARI)
   public void testCanMoveOverAndOutOfAnElement() {
     driver.get(pages.mouseOverPage);