From 6f0f5aba39d7986b12f6413bd13296c3ca3538d9 Mon Sep 17 00:00:00 2001 From: Prakash PC Date: Thu, 25 Jun 2015 11:29:38 -0700 Subject: [PATCH 01/14] [SPARK-8603] [sparkR] In windows, Incorrect file seperator passed to Java and Scripts from R --- R/pkg/R/client.R | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/R/pkg/R/client.R b/R/pkg/R/client.R index 78c7a3037ffa..1fbbf419c90e 100644 --- a/R/pkg/R/client.R +++ b/R/pkg/R/client.R @@ -42,6 +42,19 @@ determineSparkSubmitBin <- function() { } sparkSubmitBinName } +# R supports both file separator in the file path (Unix : / and Windows: \) irrespective of the operating system +# but when passing file path to Java or script program, it has to be converted according to operating system + +determinefileSeperator <- function() { + if (.Platform$OS.type == "unix") { + fileSeperator <- .Platform$file.sep + } else { +# .Platform$file.sep contains "/" for windows too +# http://www.inside-r.org/r-doc/base/file.path + fileSeperator <- "\\" + } + fileSeperator +} generateSparkSubmitArgs <- function(args, sparkHome, jars, sparkSubmitOpts, packages) { if (jars != "") { @@ -59,7 +72,8 @@ generateSparkSubmitArgs <- function(args, sparkHome, jars, sparkSubmitOpts, pack launchBackend <- function(args, sparkHome, jars, sparkSubmitOpts, packages) { sparkSubmitBinName <- determineSparkSubmitBin() if (sparkHome != "") { - sparkSubmitBin <- file.path(sparkHome, "bin", sparkSubmitBinName) + fileSeperator <- determinefileSeperator() + sparkSubmitBin <- file.path(sparkHome, "bin", sparkSubmitBinName, fsep = fileSeperator) } else { sparkSubmitBin <- sparkSubmitBinName } From 454b7f53c1bf587325154535204695247c1b3d6a Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Wed, 18 May 2016 16:52:51 +0900 Subject: [PATCH 02/14] Correct condition, fix style and comments --- R/pkg/R/client.R | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/R/pkg/R/client.R b/R/pkg/R/client.R index 04fce9602d3d..87f0129d10bf 100644 --- a/R/pkg/R/client.R +++ b/R/pkg/R/client.R @@ -42,18 +42,16 @@ determineSparkSubmitBin <- function() { } sparkSubmitBinName } -# R supports both file separator in the file path (Unix : / and Windows: \) irrespective of the operating system -# but when passing file path to Java or script program, it has to be converted according to operating system +# R supports both file separator in the file path (Unix : / and Windows: \) irrespective +# of the operating system. So, this should be manaully changed. determinefileSeperator <- function() { - if (.Platform$OS.type == "unix") { - fileSeperator <- .Platform$file.sep - } else { -# .Platform$file.sep contains "/" for windows too -# http://www.inside-r.org/r-doc/base/file.path - fileSeperator <- "\\" - } - fileSeperator + if (.Platform$OS.type == "windows") { + fileSeperator <- "\\" + } else { + fileSeperator <- .Platform$file.sep + } + fileSeperator } generateSparkSubmitArgs <- function(args, sparkHome, jars, sparkSubmitOpts, packages) { From 3057844fbf406cf492638b8078e642139a199d51 Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Wed, 18 May 2016 17:22:02 +0900 Subject: [PATCH 03/14] Update indentation --- R/pkg/R/client.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/pkg/R/client.R b/R/pkg/R/client.R index 87f0129d10bf..240caf4215bb 100644 --- a/R/pkg/R/client.R +++ b/R/pkg/R/client.R @@ -74,7 +74,7 @@ generateSparkSubmitArgs <- function(args, sparkHome, jars, sparkSubmitOpts, pack launchBackend <- function(args, sparkHome, jars, sparkSubmitOpts, packages) { sparkSubmitBinName <- determineSparkSubmitBin() if (sparkHome != "") { - fileSeperator <- determinefileSeperator() + fileSeperator <- determinefileSeperator() sparkSubmitBin <- file.path(sparkHome, "bin", sparkSubmitBinName, fsep = fileSeperator) } else { sparkSubmitBin <- sparkSubmitBinName From 935d796039a74e068b954b4c331a2b754bbdbede Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Wed, 18 May 2016 17:54:01 +0900 Subject: [PATCH 04/14] Fix typo --- R/pkg/R/client.R | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/R/pkg/R/client.R b/R/pkg/R/client.R index 240caf4215bb..a3a40f3fda04 100644 --- a/R/pkg/R/client.R +++ b/R/pkg/R/client.R @@ -45,13 +45,13 @@ determineSparkSubmitBin <- function() { # R supports both file separator in the file path (Unix : / and Windows: \) irrespective # of the operating system. So, this should be manaully changed. -determinefileSeperator <- function() { +determinefileSeparator <- function() { if (.Platform$OS.type == "windows") { - fileSeperator <- "\\" + fileSeparator <- "\\" } else { - fileSeperator <- .Platform$file.sep + fileSeparator <- .Platform$file.sep } - fileSeperator + fileSeparator } generateSparkSubmitArgs <- function(args, sparkHome, jars, sparkSubmitOpts, packages) { @@ -74,8 +74,8 @@ generateSparkSubmitArgs <- function(args, sparkHome, jars, sparkSubmitOpts, pack launchBackend <- function(args, sparkHome, jars, sparkSubmitOpts, packages) { sparkSubmitBinName <- determineSparkSubmitBin() if (sparkHome != "") { - fileSeperator <- determinefileSeperator() - sparkSubmitBin <- file.path(sparkHome, "bin", sparkSubmitBinName, fsep = fileSeperator) + fileSeparator <- determinefileSeparator() + sparkSubmitBin <- file.path(sparkHome, "bin", sparkSubmitBinName, fsep = fileSeparator) } else { sparkSubmitBin <- sparkSubmitBinName } From 6f0933d26f135258e72a64323b754a1cd6fe2868 Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Wed, 25 May 2016 21:10:08 +0900 Subject: [PATCH 05/14] Address comments, fix failing tests on windos, add a test. --- R/pkg/R/client.R | 25 +++++++++------------ R/pkg/inst/tests/testthat/test_Windows.R | 23 +++++++++++++++++++ R/pkg/inst/tests/testthat/test_context.R | 6 ++--- R/pkg/inst/tests/testthat/test_includeJAR.R | 10 +++++++-- 4 files changed, 45 insertions(+), 19 deletions(-) create mode 100644 R/pkg/inst/tests/testthat/test_Windows.R diff --git a/R/pkg/R/client.R b/R/pkg/R/client.R index a3a40f3fda04..3cc4e06dd497 100644 --- a/R/pkg/R/client.R +++ b/R/pkg/R/client.R @@ -43,17 +43,6 @@ determineSparkSubmitBin <- function() { sparkSubmitBinName } -# R supports both file separator in the file path (Unix : / and Windows: \) irrespective -# of the operating system. So, this should be manaully changed. -determinefileSeparator <- function() { - if (.Platform$OS.type == "windows") { - fileSeparator <- "\\" - } else { - fileSeparator <- .Platform$file.sep - } - fileSeparator -} - generateSparkSubmitArgs <- function(args, sparkHome, jars, sparkSubmitOpts, packages) { jars <- paste0(jars, collapse = ",") if (jars != "") { @@ -71,15 +60,23 @@ generateSparkSubmitArgs <- function(args, sparkHome, jars, sparkSubmitOpts, pack combinedArgs } +determineLauncher <- function(sparkSubmitBin, combinedArgs, capture = FALSE) { + if (.Platform$OS.type == "windows") { + sparkSubmitWithArgs <- paste(sparkSubmitBin, combinedArgs, sep = " ") + shell(sparkSubmitWithArgs, translate = TRUE, wait = ! capture, intern = capture) + } else { + system2(sparkSubmitBin, combinedArgs, wait = F, stdout = capture) + } +} + launchBackend <- function(args, sparkHome, jars, sparkSubmitOpts, packages) { sparkSubmitBinName <- determineSparkSubmitBin() if (sparkHome != "") { - fileSeparator <- determinefileSeparator() - sparkSubmitBin <- file.path(sparkHome, "bin", sparkSubmitBinName, fsep = fileSeparator) + sparkSubmitBin <- file.path(sparkHome, "bin", sparkSubmitBinName) } else { sparkSubmitBin <- sparkSubmitBinName } combinedArgs <- generateSparkSubmitArgs(args, sparkHome, jars, sparkSubmitOpts, packages) cat("Launching java with spark-submit command", sparkSubmitBin, combinedArgs, "\n") - invisible(system2(sparkSubmitBin, combinedArgs, wait = F)) + invisible(determineLauncher(sparkSubmitBin, combinedArgs)) } diff --git a/R/pkg/inst/tests/testthat/test_Windows.R b/R/pkg/inst/tests/testthat/test_Windows.R new file mode 100644 index 000000000000..a6a24345552f --- /dev/null +++ b/R/pkg/inst/tests/testthat/test_Windows.R @@ -0,0 +1,23 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +context("test the support SparkR on Windows") + +test_that("sparkJars tag in SparkContext", { + testOutput <- determineLauncher("ECHO" ,"/a/b/c" , capture = TRUE) + abcPath <- testOutput[1] + expect_equal(abcPath, "\\a\\b\\c") +}) diff --git a/R/pkg/inst/tests/testthat/test_context.R b/R/pkg/inst/tests/testthat/test_context.R index 0e5e15c0a96c..80ae9f678bbd 100644 --- a/R/pkg/inst/tests/testthat/test_context.R +++ b/R/pkg/inst/tests/testthat/test_context.R @@ -125,13 +125,13 @@ test_that("getClientModeSparkSubmitOpts() returns spark-submit args from whiteli test_that("sparkJars sparkPackages as comma-separated strings", { expect_warning(processSparkJars(" a, b ")) jars <- suppressWarnings(processSparkJars(" a, b ")) - expect_equal(jars, c("a", "b")) + expect_equal(sapply(jars, basename), c("a", "b")) jars <- suppressWarnings(processSparkJars(" abc ,, def ")) - expect_equal(jars, c("abc", "def")) + expect_equal(sapply(jars, basename), c("abc", "def")) jars <- suppressWarnings(processSparkJars(c(" abc ,, def ", "", "xyz", " ", "a,b"))) - expect_equal(jars, c("abc", "def", "xyz", "a", "b")) + expect_equal(sapply(jars, basename), c("abc", "def", "xyz", "a", "b")) p <- processSparkPackages(c("ghi", "lmn")) expect_equal(p, c("ghi", "lmn")) diff --git a/R/pkg/inst/tests/testthat/test_includeJAR.R b/R/pkg/inst/tests/testthat/test_includeJAR.R index f89aa8e507fd..fae6c7d7d854 100644 --- a/R/pkg/inst/tests/testthat/test_includeJAR.R +++ b/R/pkg/inst/tests/testthat/test_includeJAR.R @@ -21,10 +21,16 @@ runScript <- function() { sparkTestJarPath <- "R/lib/SparkR/test_support/sparktestjar_2.10-1.0.jar" jarPath <- paste("--jars", shQuote(file.path(sparkHome, sparkTestJarPath))) scriptPath <- file.path(sparkHome, "R/lib/SparkR/tests/testthat/jarTest.R") - submitPath <- file.path(sparkHome, "bin/spark-submit") - res <- system2(command = submitPath, + if (.Platform$OS.type == "windows") { + submitPath <- file.path(sparkHome, "bin/spark-submit2.cmd") + command <- paste(submitPath, jarPath, scriptPath, sep = " ") + res <- shell(command, translate = TRUE, intern = TRUE) + } else { + submitPath <- file.path(sparkHome, "bin/spark-submit") + res <- system2(command = submitPath, args = c(jarPath, scriptPath), stdout = TRUE) + } tail(res, 2) } From 64141d20049098c51ac7dd99c38b840b25ca073a Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Wed, 25 May 2016 22:28:54 +0900 Subject: [PATCH 06/14] Skip Windows test on Linux and repleace jars to list --- R/pkg/inst/tests/testthat/test_Windows.R | 3 +++ R/pkg/inst/tests/testthat/test_context.R | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/R/pkg/inst/tests/testthat/test_Windows.R b/R/pkg/inst/tests/testthat/test_Windows.R index a6a24345552f..fe4f7bd9030a 100644 --- a/R/pkg/inst/tests/testthat/test_Windows.R +++ b/R/pkg/inst/tests/testthat/test_Windows.R @@ -17,6 +17,9 @@ context("test the support SparkR on Windows") test_that("sparkJars tag in SparkContext", { + if (.Platform$OS.type != "windows") { + skip("This test is only for Windows, skipped") + } testOutput <- determineLauncher("ECHO" ,"/a/b/c" , capture = TRUE) abcPath <- testOutput[1] expect_equal(abcPath, "\\a\\b\\c") diff --git a/R/pkg/inst/tests/testthat/test_context.R b/R/pkg/inst/tests/testthat/test_context.R index 80ae9f678bbd..810b365f48de 100644 --- a/R/pkg/inst/tests/testthat/test_context.R +++ b/R/pkg/inst/tests/testthat/test_context.R @@ -125,13 +125,13 @@ test_that("getClientModeSparkSubmitOpts() returns spark-submit args from whiteli test_that("sparkJars sparkPackages as comma-separated strings", { expect_warning(processSparkJars(" a, b ")) jars <- suppressWarnings(processSparkJars(" a, b ")) - expect_equal(sapply(jars, basename), c("a", "b")) + expect_equal(lapply(jars, basename), list("a", "b")) jars <- suppressWarnings(processSparkJars(" abc ,, def ")) - expect_equal(sapply(jars, basename), c("abc", "def")) + expect_equal(lapply(jars, basename), list("abc", "def")) jars <- suppressWarnings(processSparkJars(c(" abc ,, def ", "", "xyz", " ", "a,b"))) - expect_equal(sapply(jars, basename), c("abc", "def", "xyz", "a", "b")) + expect_equal(lapply(jars, basename), list("abc", "def", "xyz", "a", "b")) p <- processSparkPackages(c("ghi", "lmn")) expect_equal(p, c("ghi", "lmn")) From 4a53360b6917a4851f8a2dff61f6e706ea6470f9 Mon Sep 17 00:00:00 2001 From: Hyukjin Kwon Date: Wed, 25 May 2016 22:44:04 +0900 Subject: [PATCH 07/14] Fix R style --- R/pkg/inst/tests/testthat/test_Windows.R | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/R/pkg/inst/tests/testthat/test_Windows.R b/R/pkg/inst/tests/testthat/test_Windows.R index fe4f7bd9030a..7b09576d039c 100644 --- a/R/pkg/inst/tests/testthat/test_Windows.R +++ b/R/pkg/inst/tests/testthat/test_Windows.R @@ -20,7 +20,7 @@ test_that("sparkJars tag in SparkContext", { if (.Platform$OS.type != "windows") { skip("This test is only for Windows, skipped") } - testOutput <- determineLauncher("ECHO" ,"/a/b/c" , capture = TRUE) + testOutput <- determineLauncher("ECHO", "a/b/c", capture = TRUE) abcPath <- testOutput[1] - expect_equal(abcPath, "\\a\\b\\c") + expect_equal(abcPath, "a\\b\\c") }) From 397fd1429e52a6f856c32851fc4c8696028867c1 Mon Sep 17 00:00:00 2001 From: Hyukjin Kwon Date: Wed, 25 May 2016 23:01:45 +0900 Subject: [PATCH 08/14] Fix logical bool for capture (to make sure it waits) --- R/pkg/R/client.R | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/R/pkg/R/client.R b/R/pkg/R/client.R index 3cc4e06dd497..30261b259d57 100644 --- a/R/pkg/R/client.R +++ b/R/pkg/R/client.R @@ -63,9 +63,9 @@ generateSparkSubmitArgs <- function(args, sparkHome, jars, sparkSubmitOpts, pack determineLauncher <- function(sparkSubmitBin, combinedArgs, capture = FALSE) { if (.Platform$OS.type == "windows") { sparkSubmitWithArgs <- paste(sparkSubmitBin, combinedArgs, sep = " ") - shell(sparkSubmitWithArgs, translate = TRUE, wait = ! capture, intern = capture) + shell(sparkSubmitWithArgs, translate = TRUE, wait = capture, intern = capture) } else { - system2(sparkSubmitBin, combinedArgs, wait = F, stdout = capture) + system2(sparkSubmitBin, combinedArgs, wait = capture, stdout = capture) } } From 0a288b1333860fd758f180bc0bd8d653e5106c65 Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Thu, 26 May 2016 11:50:07 +0900 Subject: [PATCH 09/14] Address comments --- R/pkg/R/client.R | 11 +---------- R/pkg/R/utils.R | 9 +++++++++ R/pkg/inst/tests/testthat/test_Windows.R | 2 +- R/pkg/inst/tests/testthat/test_includeJAR.R | 7 ++----- 4 files changed, 13 insertions(+), 16 deletions(-) diff --git a/R/pkg/R/client.R b/R/pkg/R/client.R index 30261b259d57..96eb43b215ed 100644 --- a/R/pkg/R/client.R +++ b/R/pkg/R/client.R @@ -60,15 +60,6 @@ generateSparkSubmitArgs <- function(args, sparkHome, jars, sparkSubmitOpts, pack combinedArgs } -determineLauncher <- function(sparkSubmitBin, combinedArgs, capture = FALSE) { - if (.Platform$OS.type == "windows") { - sparkSubmitWithArgs <- paste(sparkSubmitBin, combinedArgs, sep = " ") - shell(sparkSubmitWithArgs, translate = TRUE, wait = capture, intern = capture) - } else { - system2(sparkSubmitBin, combinedArgs, wait = capture, stdout = capture) - } -} - launchBackend <- function(args, sparkHome, jars, sparkSubmitOpts, packages) { sparkSubmitBinName <- determineSparkSubmitBin() if (sparkHome != "") { @@ -78,5 +69,5 @@ launchBackend <- function(args, sparkHome, jars, sparkSubmitOpts, packages) { } combinedArgs <- generateSparkSubmitArgs(args, sparkHome, jars, sparkSubmitOpts, packages) cat("Launching java with spark-submit command", sparkSubmitBin, combinedArgs, "\n") - invisible(determineLauncher(sparkSubmitBin, combinedArgs)) + invisible(launchScript(sparkSubmitBin, combinedArgs)) } diff --git a/R/pkg/R/utils.R b/R/pkg/R/utils.R index 784f7371807e..d5567fdaf551 100644 --- a/R/pkg/R/utils.R +++ b/R/pkg/R/utils.R @@ -664,3 +664,12 @@ varargsToJProperties <- function(...) { } props } + +launchScript <- function(script, combinedArgs, capture = FALSE) { + if (.Platform$OS.type == "windows") { + scriptWithArgs <- paste(script, combinedArgs, sep = " ") + shell(scriptWithArgs, translate = TRUE, wait = capture, intern = capture) + } else { + system2(script, combinedArgs, wait = capture, stdout = capture) + } +} diff --git a/R/pkg/inst/tests/testthat/test_Windows.R b/R/pkg/inst/tests/testthat/test_Windows.R index 7b09576d039c..4ed0f4359061 100644 --- a/R/pkg/inst/tests/testthat/test_Windows.R +++ b/R/pkg/inst/tests/testthat/test_Windows.R @@ -20,7 +20,7 @@ test_that("sparkJars tag in SparkContext", { if (.Platform$OS.type != "windows") { skip("This test is only for Windows, skipped") } - testOutput <- determineLauncher("ECHO", "a/b/c", capture = TRUE) + testOutput <- launchScript("ECHO", "/a/b/c" , capture = TRUE) abcPath <- testOutput[1] expect_equal(abcPath, "a\\b\\c") }) diff --git a/R/pkg/inst/tests/testthat/test_includeJAR.R b/R/pkg/inst/tests/testthat/test_includeJAR.R index fae6c7d7d854..2dd25e8900bf 100644 --- a/R/pkg/inst/tests/testthat/test_includeJAR.R +++ b/R/pkg/inst/tests/testthat/test_includeJAR.R @@ -23,14 +23,11 @@ runScript <- function() { scriptPath <- file.path(sparkHome, "R/lib/SparkR/tests/testthat/jarTest.R") if (.Platform$OS.type == "windows") { submitPath <- file.path(sparkHome, "bin/spark-submit2.cmd") - command <- paste(submitPath, jarPath, scriptPath, sep = " ") - res <- shell(command, translate = TRUE, intern = TRUE) } else { submitPath <- file.path(sparkHome, "bin/spark-submit") - res <- system2(command = submitPath, - args = c(jarPath, scriptPath), - stdout = TRUE) } + combinedArgs <- paste(jarPath, scriptPath, sep = " ") + res <- launchScript(submitPath, combinedArgs, capture = TRUE) tail(res, 2) } From 0482ebbc43ff1bef8e7a6a16376c6ec36840a366 Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Thu, 26 May 2016 12:32:04 +0900 Subject: [PATCH 10/14] R style improvement --- R/pkg/R/utils.R | 2 +- R/pkg/inst/tests/testthat/test_Windows.R | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/R/pkg/R/utils.R b/R/pkg/R/utils.R index d5567fdaf551..e73436613194 100644 --- a/R/pkg/R/utils.R +++ b/R/pkg/R/utils.R @@ -668,7 +668,7 @@ varargsToJProperties <- function(...) { launchScript <- function(script, combinedArgs, capture = FALSE) { if (.Platform$OS.type == "windows") { scriptWithArgs <- paste(script, combinedArgs, sep = " ") - shell(scriptWithArgs, translate = TRUE, wait = capture, intern = capture) + shell(scriptWithArgs, translate = TRUE, wait = capture, intern = capture) # nolint } else { system2(script, combinedArgs, wait = capture, stdout = capture) } diff --git a/R/pkg/inst/tests/testthat/test_Windows.R b/R/pkg/inst/tests/testthat/test_Windows.R index 4ed0f4359061..d492f92b48fd 100644 --- a/R/pkg/inst/tests/testthat/test_Windows.R +++ b/R/pkg/inst/tests/testthat/test_Windows.R @@ -20,7 +20,7 @@ test_that("sparkJars tag in SparkContext", { if (.Platform$OS.type != "windows") { skip("This test is only for Windows, skipped") } - testOutput <- launchScript("ECHO", "/a/b/c" , capture = TRUE) + testOutput <- launchScript("ECHO", "a/b/c", capture = TRUE) abcPath <- testOutput[1] expect_equal(abcPath, "a\\b\\c") }) From a3ccac62b450a10440e7838046bd6e41e221369b Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Thu, 26 May 2016 15:55:02 +0900 Subject: [PATCH 11/14] Reuse determineSparkSubmitBin() --- R/pkg/R/client.R | 2 +- R/pkg/inst/tests/testthat/test_includeJAR.R | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/R/pkg/R/client.R b/R/pkg/R/client.R index 96eb43b215ed..2d341d836c13 100644 --- a/R/pkg/R/client.R +++ b/R/pkg/R/client.R @@ -38,7 +38,7 @@ determineSparkSubmitBin <- function() { if (.Platform$OS.type == "unix") { sparkSubmitBinName <- "spark-submit" } else { - sparkSubmitBinName <- "spark-submit.cmd" + sparkSubmitBinName <- "spark-submit2.cmd" } sparkSubmitBinName } diff --git a/R/pkg/inst/tests/testthat/test_includeJAR.R b/R/pkg/inst/tests/testthat/test_includeJAR.R index 2dd25e8900bf..0ef2c918b2ce 100644 --- a/R/pkg/inst/tests/testthat/test_includeJAR.R +++ b/R/pkg/inst/tests/testthat/test_includeJAR.R @@ -21,11 +21,7 @@ runScript <- function() { sparkTestJarPath <- "R/lib/SparkR/test_support/sparktestjar_2.10-1.0.jar" jarPath <- paste("--jars", shQuote(file.path(sparkHome, sparkTestJarPath))) scriptPath <- file.path(sparkHome, "R/lib/SparkR/tests/testthat/jarTest.R") - if (.Platform$OS.type == "windows") { - submitPath <- file.path(sparkHome, "bin/spark-submit2.cmd") - } else { - submitPath <- file.path(sparkHome, "bin/spark-submit") - } + submitPath <- file.path(sparkHome, determineSparkSubmitBin()) combinedArgs <- paste(jarPath, scriptPath, sep = " ") res <- launchScript(submitPath, combinedArgs, capture = TRUE) tail(res, 2) From a1ccd5f383f13a7368516469989527557516ac92 Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Thu, 26 May 2016 16:46:31 +0900 Subject: [PATCH 12/14] Fix style and correct arguments --- R/pkg/inst/tests/testthat/test_includeJAR.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/pkg/inst/tests/testthat/test_includeJAR.R b/R/pkg/inst/tests/testthat/test_includeJAR.R index 0ef2c918b2ce..512dd39cb29f 100644 --- a/R/pkg/inst/tests/testthat/test_includeJAR.R +++ b/R/pkg/inst/tests/testthat/test_includeJAR.R @@ -21,7 +21,7 @@ runScript <- function() { sparkTestJarPath <- "R/lib/SparkR/test_support/sparktestjar_2.10-1.0.jar" jarPath <- paste("--jars", shQuote(file.path(sparkHome, sparkTestJarPath))) scriptPath <- file.path(sparkHome, "R/lib/SparkR/tests/testthat/jarTest.R") - submitPath <- file.path(sparkHome, determineSparkSubmitBin()) + submitPath <- file.path(sparkHome, paste("bin/", determineSparkSubmitBin(), sep = "")) combinedArgs <- paste(jarPath, scriptPath, sep = " ") res <- launchScript(submitPath, combinedArgs, capture = TRUE) tail(res, 2) From 521f07b704e38675fa0475a0d23c3181a678684b Mon Sep 17 00:00:00 2001 From: Hyukjin Kwon Date: Thu, 26 May 2016 22:34:46 +0900 Subject: [PATCH 13/14] Change test context name --- R/pkg/inst/tests/testthat/test_Windows.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/pkg/inst/tests/testthat/test_Windows.R b/R/pkg/inst/tests/testthat/test_Windows.R index d492f92b48fd..8813e18a1fa4 100644 --- a/R/pkg/inst/tests/testthat/test_Windows.R +++ b/R/pkg/inst/tests/testthat/test_Windows.R @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -context("test the support SparkR on Windows") +context("Windows-specific tests") test_that("sparkJars tag in SparkContext", { if (.Platform$OS.type != "windows") { From 752c40a549ce78105db7ea51d06249661fbb5429 Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Fri, 27 May 2016 09:27:00 +0900 Subject: [PATCH 14/14] Fix a typo --- R/WINDOWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/WINDOWS.md b/R/WINDOWS.md index f948ed397479..f67a1c51d178 100644 --- a/R/WINDOWS.md +++ b/R/WINDOWS.md @@ -28,6 +28,6 @@ To run the SparkR unit tests on Windows, the following steps are required —ass ``` R -e "install.packages('testthat', repos='http://cran.us.r-project.org')" - .\bin\spark-submit2.cmd --conf spark.hadoop.fs.defualt.name="file:///" R\pkg\tests\run-all.R + .\bin\spark-submit2.cmd --conf spark.hadoop.fs.default.name="file:///" R\pkg\tests\run-all.R ```