[SPARK-32121][SHUFFLE] Support Windows OS in ExecutorDiskUtils

common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExecutorDiskUtils.java

@@ -27,7 +27,14 @@
 
 public class ExecutorDiskUtils {
 
-  private static final Pattern MULTIPLE_SEPARATORS = Pattern.compile(File.separator + "{2,}");
+  private static final Pattern MULTIPLE_SEPARATORS;
+  static {
+    if (isWindows()) {
+      MULTIPLE_SEPARATORS = Pattern.compile("[/\\\\]+");
+    } else {
+      MULTIPLE_SEPARATORS = Pattern.compile("/{2,}");
+    }
+  }
 
   /**
    * Hashes a filename into the corresponding local directory, in a manner consistent with
@@ -41,6 +48,13 @@ public static File getFile(String[] localDirs, int subDirsPerLocalDir, String fi
         localDir, String.format("%02x", subDirId), filename));
   }
 
+  /** Returns whether the OS is Windows. */
+  @VisibleForTesting
+  static boolean isWindows() {
+    String os = System.getProperty("os.name");
+    return os.startsWith("Windows");
+  }
+
   /**
    * This method is needed to avoid the situation when multiple File instances for the
    * same pathname "foo/bar" are created, each with a separate copy of the "foo/bar" String.
@@ -50,14 +64,18 @@ public static File getFile(String[] localDirs, int subDirsPerLocalDir, String fi
    * the internal code in java.io.File would normalize it later, creating a new "foo/bar"
    * String copy. Unfortunately, we cannot just reuse the normalization code that java.io.File
    * uses, since it is in the package-private class java.io.FileSystem.
+   *
+   * On Windows, separator "/" should be "\".
+   *
+   * "\\" is legal character in path name on Unix like OS, but illegal on Windows.
    */
   @VisibleForTesting
   static String createNormalizedInternedPathname(String dir1, String dir2, String fname) {
     String pathname = dir1 + File.separator + dir2 + File.separator + fname;
     Matcher m = MULTIPLE_SEPARATORS.matcher(pathname);
-    pathname = m.replaceAll("/");
+    pathname = m.replaceAll(Matcher.quoteReplacement(File.separator));
     // A single trailing slash needs to be taken care of separately
-    if (pathname.length() > 1 && pathname.endsWith("/")) {
+    if (pathname.length() > 1 && pathname.endsWith(File.separator)) {
       pathname = pathname.substring(0, pathname.length() - 1);
     }
     return pathname.intern();

common/network-shuffle/src/test/java/org/apache/spark/network/shuffle/ExternalShuffleBlockResolverSuite.java

@@ -146,12 +146,23 @@ public void jsonSerializationOfExecutorRegistration() throws IOException {
 
   @Test
   public void testNormalizeAndInternPathname() {
-    assertPathsMatch("/foo", "bar", "baz", "/foo/bar/baz");
-    assertPathsMatch("//foo/", "bar/", "//baz", "/foo/bar/baz");
-    assertPathsMatch("foo", "bar", "baz///", "foo/bar/baz");
-    assertPathsMatch("/foo/", "/bar//", "/baz", "/foo/bar/baz");
-    assertPathsMatch("/", "", "", "/");
-    assertPathsMatch("/", "/", "/", "/");
+    assertPathsMatch("/foo", "bar", "baz",
+      File.separator + "foo" + File.separator + "bar" + File.separator + "baz");
+    assertPathsMatch("//foo/", "bar/", "//baz",
+      File.separator + "foo" + File.separator + "bar" + File.separator + "baz");
+    assertPathsMatch("foo", "bar", "baz///",
+      "foo" + File.separator + "bar" + File.separator + "baz");
+    assertPathsMatch("/foo/", "/bar//", "/baz",
+      File.separator + "foo" + File.separator + "bar" + File.separator + "baz");
+    assertPathsMatch("/", "", "", File.separator);
+    assertPathsMatch("/", "/", "/", File.separator);
+    if (ExecutorDiskUtils.isWindows()) {
+      assertPathsMatch("/foo\\/", "bar", "baz",
+        File.separator + "foo" + File.separator + "bar" + File.separator + "baz");
+    } else {
+      assertPathsMatch("/foo\\/", "bar", "baz",
+        File.separator + "foo\\" + File.separator + "bar" + File.separator + "baz");
+    }
   }
 
   private void assertPathsMatch(String p1, String p2, String p3, String expectedPathname) {
@@ -160,6 +171,6 @@ private void assertPathsMatch(String p1, String p2, String p3, String expectedPa
     assertEquals(expectedPathname, normPathname);
     File file = new File(normPathname);
     String returnedPath = file.getPath();
-    assertTrue(normPathname == returnedPath);
+    assertEquals(normPathname, returnedPath);
   }
 }