Skip to content

[SPARK-34339][CORE][SQL] Expose the number of total paths in Utils.buildLocationMetadata() #31464

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
HeartSaVioR wants to merge 3 commits into from
Closed

[SPARK-34339][CORE][SQL] Expose the number of total paths in Utils.buildLocationMetadata() #31464

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
f0a02fd
WIP buildLocationMetadata-show-remaining-number-of-paths
HeartSaVioR Feb 3, 2021
56abfe2
Changed the approach to print out the number of paths on header
HeartSaVioR Feb 4, 2021
e766209
Fix UTs
HeartSaVioR Feb 4, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions core/src/main/scala/org/apache/spark/util/Utils.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2989,6 +2989,9 @@ private[spark] object Utils extends Logging {
metadata.append(paths(index).toString)
index += 1
}
if (paths.length > index) {
metadata.append(s", ... ${paths.length - index} more")
Comment thread
HyukjinKwon marked this conversation as resolved.
Outdated
}
metadata.append("]")
metadata.toString
}
Expand Down
12 changes: 6 additions & 6 deletions core/src/test/scala/org/apache/spark/util/UtilsSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1304,16 +1304,16 @@ class UtilsSuite extends SparkFunSuite with ResetSystemProperties with Logging {

test("pathsToMetadata") {
val paths = (0 to 4).map(i => new Path(s"path$i"))
assert(Utils.buildLocationMetadata(paths, 5) == "[path0]")
assert(Utils.buildLocationMetadata(paths, 10) == "[path0, path1]")
assert(Utils.buildLocationMetadata(paths, 15) == "[path0, path1, path2]")
assert(Utils.buildLocationMetadata(paths, 25) == "[path0, path1, path2, path3]")
assert(Utils.buildLocationMetadata(paths, 5) == "[path0, ... 4 more]")
assert(Utils.buildLocationMetadata(paths, 10) == "[path0, path1, ... 3 more]")
assert(Utils.buildLocationMetadata(paths, 15) == "[path0, path1, path2, ... 2 more]")
assert(Utils.buildLocationMetadata(paths, 25) == "[path0, path1, path2, path3, ... 1 more]")

// edge-case: we should consider the fact non-path chars including '[' and ", " are accounted
// 1. second path is not added due to the addition of '['
assert(Utils.buildLocationMetadata(paths, 6) == "[path0]")
assert(Utils.buildLocationMetadata(paths, 6) == "[path0, ... 4 more]")
// 2. third path is not added due to the addition of ", "
assert(Utils.buildLocationMetadata(paths, 13) == "[path0, path1]")
assert(Utils.buildLocationMetadata(paths, 13) == "[path0, path1, ... 3 more]")
}

test("checkHost supports both IPV4 and IPV6") {
Expand Down
6 changes: 3 additions & 3 deletions ...core/src/test/scala/org/apache/spark/sql/execution/DataSourceScanExecRedactionSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,8 +122,6 @@ class DataSourceScanExecRedactionSuite extends DataSourceScanRedactionTest {

test("SPARK-31793: FileSourceScanExec metadata should contain limited file paths") {
withTempPath { path =>
val dir = path.getCanonicalPath

// create a sub-directory with long name so that each root path will always exceed the limit
// this is to ensure we always test the case for the path truncation
val dataDirName = Random.alphanumeric.take(100).toList.mkString
Expand Down Expand Up @@ -155,7 +153,9 @@ class DataSourceScanExecRedactionSuite extends DataSourceScanRedactionTest {
location.get.indexOf('[') + 1, location.get.indexOf(']')).split(", ").toSeq

// the only one path should be available
assert(pathsInLocation.size == 1)
assert(pathsInLocation.size == 2)
// indicator ("... N more") should be available
assert(pathsInLocation.exists(_.contains("... ")))
}
}
}
Expand Down