From 3764e55c8deb7aa46e74c5e2124d4243549d5e1d Mon Sep 17 00:00:00 2001 From: zuotingbing Date: Wed, 31 May 2017 16:19:41 +0800 Subject: [PATCH 1/3] Lack of the most important case about the test of resolveURI in UtilsSuite, and add it as needed. --- core/src/test/scala/org/apache/spark/util/UtilsSuite.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala b/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala index 3339d5b35d3b..1b34eaea9bed 100644 --- a/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala +++ b/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala @@ -463,6 +463,7 @@ class UtilsSuite extends SparkFunSuite with ResetSystemProperties with Logging { assume(before.split(",").length === 1) // Repeated invocations of resolveURI should yield the same result def resolve(uri: String): String = Utils.resolveURI(uri).toString + assert(resolve(before) === after) assert(resolve(after) === after) assert(resolve(resolve(after)) === after) assert(resolve(resolve(resolve(after))) === after) @@ -473,7 +474,7 @@ class UtilsSuite extends SparkFunSuite with ResetSystemProperties with Logging { val rawCwd = System.getProperty("user.dir") val cwd = if (Utils.isWindows) s"/$rawCwd".replace("\\", "/") else rawCwd assertResolves("hdfs:/root/spark.jar", "hdfs:/root/spark.jar") - assertResolves("hdfs:///root/spark.jar#app.jar", "hdfs:/root/spark.jar#app.jar") + assertResolves("hdfs:///root/spark.jar#app.jar", "hdfs:///root/spark.jar#app.jar") assertResolves("spark.jar", s"file:$cwd/spark.jar") assertResolves("spark.jar#app.jar", s"file:$cwd/spark.jar#app.jar") assertResolves("path to/file.txt", s"file:$cwd/path%20to/file.txt") @@ -482,7 +483,7 @@ class UtilsSuite extends SparkFunSuite with ResetSystemProperties with Logging { assertResolves("C:\\path to\\file.txt", "file:/C:/path%20to/file.txt") } assertResolves("file:/C:/path/to/file.txt", "file:/C:/path/to/file.txt") - assertResolves("file:///C:/path/to/file.txt", "file:/C:/path/to/file.txt") + assertResolves("file:///C:/path/to/file.txt", "file:///C:/path/to/file.txt") assertResolves("file:/C:/file.txt#alias.txt", "file:/C:/file.txt#alias.txt") assertResolves("file:foo", s"file:foo") assertResolves("file:foo:baby", s"file:foo:baby") From b9bc87e831906d0b25dc8e15e432655ce0f2c517 Mon Sep 17 00:00:00 2001 From: zuotingbing Date: Thu, 1 Jun 2017 14:44:23 +0800 Subject: [PATCH 2/3] identify the cases between resolveURI and resolveURIs --- core/src/test/scala/org/apache/spark/util/UtilsSuite.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala b/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala index 1b34eaea9bed..27d2464efbcc 100644 --- a/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala +++ b/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala @@ -467,9 +467,7 @@ class UtilsSuite extends SparkFunSuite with ResetSystemProperties with Logging { assert(resolve(after) === after) assert(resolve(resolve(after)) === after) assert(resolve(resolve(resolve(after))) === after) - // Also test resolveURIs with single paths - assert(new URI(Utils.resolveURIs(before)) === new URI(after)) - assert(new URI(Utils.resolveURIs(after)) === new URI(after)) + } val rawCwd = System.getProperty("user.dir") val cwd = if (Utils.isWindows) s"/$rawCwd".replace("\\", "/") else rawCwd @@ -512,6 +510,8 @@ class UtilsSuite extends SparkFunSuite with ResetSystemProperties with Logging { s"hdfs:/jar1,file:/jar2,file:$cwd/jar3,file:/C:/pi.py%23py.pi,file:/C:/path%20to/jar4") } assertResolves(",jar1,jar2", s"file:$cwd/jar1,file:$cwd/jar2") + // Also test resolveURIs with single paths + assertResolves("hdfs:/root/spark.jar", "hdfs:/root/spark.jar") } test("nonLocalPaths") { From 9455fb25cad229e329169a44c4591472237223d1 Mon Sep 17 00:00:00 2001 From: zuotingbing Date: Thu, 1 Jun 2017 15:07:03 +0800 Subject: [PATCH 3/3] 1. In resolveURIs test,These two cases [assert(Utils.resolveURIs(after) === after)] and [assert(resolve(after) === after)] duplicate each other,delete one. 2. little fixs make the suit clear. --- .../scala/org/apache/spark/util/UtilsSuite.scala | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala b/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala index 27d2464efbcc..f7bc8f888b0d 100644 --- a/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala +++ b/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala @@ -461,13 +461,12 @@ class UtilsSuite extends SparkFunSuite with ResetSystemProperties with Logging { def assertResolves(before: String, after: String): Unit = { // This should test only single paths assume(before.split(",").length === 1) - // Repeated invocations of resolveURI should yield the same result def resolve(uri: String): String = Utils.resolveURI(uri).toString assert(resolve(before) === after) assert(resolve(after) === after) + // Repeated invocations of resolveURI should yield the same result assert(resolve(resolve(after)) === after) assert(resolve(resolve(resolve(after))) === after) - } val rawCwd = System.getProperty("user.dir") val cwd = if (Utils.isWindows) s"/$rawCwd".replace("\\", "/") else rawCwd @@ -483,18 +482,17 @@ class UtilsSuite extends SparkFunSuite with ResetSystemProperties with Logging { assertResolves("file:/C:/path/to/file.txt", "file:/C:/path/to/file.txt") assertResolves("file:///C:/path/to/file.txt", "file:///C:/path/to/file.txt") assertResolves("file:/C:/file.txt#alias.txt", "file:/C:/file.txt#alias.txt") - assertResolves("file:foo", s"file:foo") - assertResolves("file:foo:baby", s"file:foo:baby") + assertResolves("file:foo", "file:foo") + assertResolves("file:foo:baby", "file:foo:baby") } test("resolveURIs with multiple paths") { def assertResolves(before: String, after: String): Unit = { assume(before.split(",").length > 1) - assert(Utils.resolveURIs(before) === after) - assert(Utils.resolveURIs(after) === after) - // Repeated invocations of resolveURIs should yield the same result def resolve(uri: String): String = Utils.resolveURIs(uri) + assert(resolve(before) === after) assert(resolve(after) === after) + // Repeated invocations of resolveURIs should yield the same result assert(resolve(resolve(after)) === after) assert(resolve(resolve(resolve(after))) === after) }