update from origin #2

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged

lxsmnv merged 58 commits into lxsmnv:master from apache:master

Mar 4, 2017

.travis.yml

-Original file line number
+Diff line change
@@ Expand Up / @@ -28,7 +28,6 @@ dist: trusty @@
     # 2. Choose language and target JDKs for parallel builds.
     language: java
     jdk:
-      - oraclejdk7
       - oraclejdk8
     # 3. Setup cache directory for SBT and Maven.
@@ Expand Down @@

R/WINDOWS.md

-Original file line number
+Diff line change
@@ Expand Up @@
         ```
         R -e "install.packages('testthat', repos='http://cran.us.r-project.org')"
-        .\bin\spark-submit2.cmd --conf spark.hadoop.fs.default.name="file:///" R\pkg\tests\run-all.R
+        .\bin\spark-submit2.cmd --conf spark.hadoop.fs.defaultFS="file:///" R\pkg\tests\run-all.R
         ```

R/pkg/R/DataFrame.R

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -280,7 +280,7 @@ setMethod("dtypes",
  
    #' Column Names of SparkDataFrame

    #'

    #' Return all column names as a list.

    #' Return a vector of column names.

    #'

    #' @param x a SparkDataFrame.

    #'

    @@ -338,7 +338,7 @@ setMethod("colnames",
  
              })

    #' @param value a character vector. Must have the same length as the number

    #'              of columns in the SparkDataFrame.

    #'              of columns to be renamed.

    #' @rdname columns

    #' @aliases colnames<-,SparkDataFrame-method

    #' @name colnames<-

R/pkg/R/SQLContext.R

-Original file line number
+Diff line change
@@ Expand Up / @@ -332,8 +332,10 @@ setMethod("toDF", signature(x = "RDD"), @@
     #' Create a SparkDataFrame from a JSON file.
     #'
-    #' Loads a JSON file (\href{http://jsonlines.org/}{JSON Lines text format or newline-delimited JSON}
-    #' ), returning the result as a SparkDataFrame
+    #' Loads a JSON file, returning the result as a SparkDataFrame
+    #' By default, (\href{http://jsonlines.org/}{JSON Lines text format or newline-delimited JSON}
+    #' ) is supported. For JSON (one record per file), set a named property \code{wholeFile} to
+    #' \code{TRUE}.
     #' It goes through the entire dataset once to determine the schema.
     #'
     #' @param path Path of file to read. A vector of multiple paths is allowed.
@@ Expand All / @@ -346,6 +348,7 @@ setMethod("toDF", signature(x = "RDD"), @@
     #' sparkR.session()
     #' path <- "path/to/file.json"
     #' df <- read.json(path)
+    #' df <- read.json(path, wholeFile = TRUE)
     #' df <- jsonFile(path)
     #' }
     #' @name read.json
@@ Expand Down Expand Up / @@ -778,14 +781,15 @@ dropTempView <- function(viewName) { @@
     #' @return SparkDataFrame
     #' @rdname read.df
     #' @name read.df
+    #' @seealso \link{read.json}
     #' @export
     #' @examples
     #'\dontrun{
     #' sparkR.session()
     #' df1 <- read.df("path/to/file.json", source = "json")
     #' schema <- structType(structField("name", "string"),
     #'                      structField("info", "map<string,double>"))
-    #' df2 <- read.df(mapTypeJsonPath, "json", schema)
+    #' df2 <- read.df(mapTypeJsonPath, "json", schema, wholeFile = TRUE)
     #' df3 <- loadDF("data/test_table", "parquet", mergeSchema = "true")
     #' }
     #' @name read.df
@@ Expand Down @@

R/pkg/R/mllib_classification.R

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -75,9 +75,9 @@ setClass("NaiveBayesModel", representation(jobj = "jobj"))
  
    #' @examples

    #' \dontrun{

    #' sparkR.session()

    #' df <- createDataFrame(iris)

    #' training <- df[df$Species %in% c("versicolor", "virginica"), ]

    #' model <- spark.svmLinear(training, Species ~ ., regParam = 0.5)

    #' t <- as.data.frame(Titanic)

    #' training <- createDataFrame(t)

    #' model <- spark.svmLinear(training, Survived ~ ., regParam = 0.5)

    #' summary <- summary(model)

    #'

    #' # fitted values on training data

    @@ -220,9 +220,9 @@ function(object, path, overwrite = FALSE) {
  
    #' \dontrun{

    #' sparkR.session()

    #' # binary logistic regression

    #' df <- createDataFrame(iris)

    #' training <- df[df$Species %in% c("versicolor", "virginica"), ]

    #' model <- spark.logit(training, Species ~ ., regParam = 0.5)

    #' t <- as.data.frame(Titanic)

    #' training <- createDataFrame(t)

    #' model <- spark.logit(training, Survived ~ ., regParam = 0.5)

    #' summary <- summary(model)

    #'

    #' # fitted values on training data

    @@ -239,8 +239,7 @@ function(object, path, overwrite = FALSE) {
  
    #'

    #' # multinomial logistic regression

    #'

    #' df <- createDataFrame(iris)

    #' model <- spark.logit(df, Species ~ ., regParam = 0.5)

    #' model <- spark.logit(training, Class ~ ., regParam = 0.5)

    #' summary <- summary(model)

    #'

    #' }

R/pkg/R/mllib_clustering.R

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -72,8 +72,9 @@ setClass("LDAModel", representation(jobj = "jobj"))
  
    #' @examples

    #' \dontrun{

    #' sparkR.session()

    #' df <- createDataFrame(iris)

    #' model <- spark.bisectingKmeans(df, Sepal_Length ~ Sepal_Width, k = 4)

    #' t <- as.data.frame(Titanic)

    #' df <- createDataFrame(t)

    #' model <- spark.bisectingKmeans(df, Class ~ Survived, k = 4)

    #' summary(model)

    #'

    #' # get fitted result from a bisecting k-means model

    @@ -82,7 +83,7 @@ setClass("LDAModel", representation(jobj = "jobj"))
  
    #'

    #' # fitted values on training data

    #' fitted <- predict(model, df)

    #' head(select(fitted, "Sepal_Length", "prediction"))

    #' head(select(fitted, "Class", "prediction"))

    #'

    #' # save fitted model to input path

    #' path <- "path/to/model"

    @@ -338,14 +339,14 @@ setMethod("write.ml", signature(object = "GaussianMixtureModel", path = "charact
  
    #' @examples

    #' \dontrun{

    #' sparkR.session()

    #' data(iris)

    #' df <- createDataFrame(iris)

    #' model <- spark.kmeans(df, Sepal_Length ~ Sepal_Width, k = 4, initMode = "random")

    #' t <- as.data.frame(Titanic)

    #' df <- createDataFrame(t)

    #' model <- spark.kmeans(df, Class ~ Survived, k = 4, initMode = "random")

    #' summary(model)

    #'

    #' # fitted values on training data

    #' fitted <- predict(model, df)

    #' head(select(fitted, "Sepal_Length", "prediction"))

    #' head(select(fitted, "Class", "prediction"))

    #'

    #' # save fitted model to input path

    #' path <- "path/to/model"

R/pkg/R/mllib_regression.R

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -68,14 +68,14 @@ setClass("IsotonicRegressionModel", representation(jobj = "jobj"))
  
    #' @examples

    #' \dontrun{

    #' sparkR.session()

    #' data(iris)

    #' df <- createDataFrame(iris)

    #' model <- spark.glm(df, Sepal_Length ~ Sepal_Width, family = "gaussian")

    #' t <- as.data.frame(Titanic)

    #' df <- createDataFrame(t)

    #' model <- spark.glm(df, Freq ~ Sex + Age, family = "gaussian")

    #' summary(model)

    #'

    #' # fitted values on training data

    #' fitted <- predict(model, df)

    #' head(select(fitted, "Sepal_Length", "prediction"))

    #' head(select(fitted, "Freq", "prediction"))

    #'

    #' # save fitted model to input path

    #' path <- "path/to/model"

    @@ -137,9 +137,9 @@ setMethod("spark.glm", signature(data = "SparkDataFrame", formula = "formula"),
  
    #' @examples

    #' \dontrun{

    #' sparkR.session()

    #' data(iris)

    #' df <- createDataFrame(iris)

    #' model <- glm(Sepal_Length ~ Sepal_Width, df, family = "gaussian")

    #' t <- as.data.frame(Titanic)

    #' df <- createDataFrame(t)

    #' model <- glm(Freq ~ Sex + Age, df, family = "gaussian")

    #' summary(model)

    #' }

    #' @note glm since 1.5.0

R/pkg/R/mllib_tree.R

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -143,14 +143,15 @@ print.summary.treeEnsemble <- function(x) {
  
    #'

    #' # fit a Gradient Boosted Tree Classification Model

    #' # label must be binary - Only binary classification is supported for GBT.

    #' df <- createDataFrame(iris[iris$Species != "virginica", ])

    #' model <- spark.gbt(df, Species ~ Petal_Length + Petal_Width, "classification")

    #' t <- as.data.frame(Titanic)

    #' df <- createDataFrame(t)

    #' model <- spark.gbt(df, Survived ~ Age + Freq, "classification")

    #'

    #' # numeric label is also supported

    #' iris2 <- iris[iris$Species != "virginica", ]

    #' iris2$NumericSpecies <- ifelse(iris2$Species == "setosa", 0, 1)

    #' df <- createDataFrame(iris2)

    #' model <- spark.gbt(df, NumericSpecies ~ ., type = "classification")

    #' t2 <- as.data.frame(Titanic)

    #' t2$NumericGender <- ifelse(t2$Sex == "Male", 0, 1)

    #' df <- createDataFrame(t2)

    #' model <- spark.gbt(df, NumericGender ~ ., type = "classification")

    #' }

    #' @note spark.gbt since 2.1.0

    setMethod("spark.gbt", signature(data = "SparkDataFrame", formula = "formula"),

    @@ -351,8 +352,9 @@ setMethod("write.ml", signature(object = "GBTClassificationModel", path = "chara
  
    #' summary(savedModel)

    #'

    #' # fit a Random Forest Classification Model

    #' df <- createDataFrame(iris)

    #' model <- spark.randomForest(df, Species ~ Petal_Length + Petal_Width, "classification")

    #' t <- as.data.frame(Titanic)

    #' df <- createDataFrame(t)

    #' model <- spark.randomForest(df, Survived ~ Freq + Age, "classification")

    #' }

    #' @note spark.randomForest since 2.1.0

    setMethod("spark.randomForest", signature(data = "SparkDataFrame", formula = "formula"),

R/pkg/inst/tests/testthat/test_sparkSQL.R

-Original file line number
+Diff line change
@@ Expand Up / @@ -898,6 +898,12 @@ test_that("names() colnames() set the column names", { @@
       expect_equal(names(z)[3], "c")
       names(z)[3] <- "c2"
       expect_equal(names(z)[3], "c2")
+      # Test subset assignment
+      colnames(df)[1] <- "col5"
+      expect_equal(colnames(df)[1], "col5")
+      names(df)[2] <- "col6"
+      expect_equal(names(df)[2], "col6")
     })
     test_that("head() and first() return the correct data", {
@@ Expand Down @@

R/pkg/vignettes/sparkr-vignettes.Rmd

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -565,11 +565,10 @@ We use a simple example to demonstrate `spark.logit` usage. In general, there ar
  
    and 3). Obtain the coefficient matrix of the fitted model using `summary` and use the model for prediction with `predict`.

    Binomial logistic regression

    ```{r, warning=FALSE}

    df <- createDataFrame(iris)

    # Create a DataFrame containing two classes

    training <- df[df$Species %in% c("versicolor", "virginica"), ]

    model <- spark.logit(training, Species ~ ., regParam = 0.00042)

    ```{r}

    t <- as.data.frame(Titanic)

    training <- createDataFrame(t)

    model <- spark.logit(training, Survived ~ ., regParam = 0.04741301)

    summary(model)

    ```

    @@ -579,10 +578,11 @@ fitted <- predict(model, training)
  
    ```

    Multinomial logistic regression against three classes

    ```{r, warning=FALSE}

    df <- createDataFrame(iris)

    ```{r}

    t <- as.data.frame(Titanic)

    training <- createDataFrame(t)

    # Note in this case, Spark infers it is multinomial logistic regression, so family = "multinomial" is optional.

    model <- spark.logit(df, Species ~ ., regParam = 0.056)

    model <- spark.logit(training, Class ~ ., regParam = 0.07815179)

    summary(model)

    ```

    @@ -609,11 +609,12 @@ MLPC employs backpropagation for learning the model. We use the logistic loss fu
  
    `spark.mlp` requires at least two columns in `data`: one named `"label"` and the other one `"features"`. The `"features"` column should be in libSVM-format.

    We use iris data set to show how to use `spark.mlp` in classification.

    ```{r, warning=FALSE}

    df <- createDataFrame(iris)

    We use Titanic data set to show how to use `spark.mlp` in classification.

    ```{r}

    t <- as.data.frame(Titanic)

    training <- createDataFrame(t)

    # fit a Multilayer Perceptron Classification Model

    model <- spark.mlp(df, Species ~ ., blockSize = 128, layers = c(4, 3), solver = "l-bfgs", maxIter = 100, tol = 0.5, stepSize = 1, seed = 1, initialWeights = c(0, 0, 0, 0, 0, 5, 5, 5, 5, 5, 9, 9, 9, 9, 9))

    model <- spark.mlp(training, Survived ~ Age + Sex, blockSize = 128, layers = c(2, 3), solver = "l-bfgs", maxIter = 100, tol = 0.5, stepSize = 1, seed = 1, initialWeights = c( 0, 0, 0, 5, 5, 5, 9, 9, 9))

    ```

    To avoid lengthy display, we only present partial results of the model summary. You can check the full result from your sparkR shell.

    @@ -630,7 +631,7 @@ options(ops)
  
    ```

    ```{r}

    # make predictions use the fitted model

    predictions <- predict(model, df)

    predictions <- predict(model, training)

    head(select(predictions, predictions$prediction))

    ```

    @@ -769,12 +770,13 @@ predictions <- predict(rfModel, df)
  
    `spark.bisectingKmeans` is a kind of [hierarchical clustering](https://en.wikipedia.org/wiki/Hierarchical_clustering) using a divisive (or "top-down") approach: all observations start in one cluster, and splits are performed recursively as one moves down the hierarchy.

    ```{r, warning=FALSE}

    df <- createDataFrame(iris)

    model <- spark.bisectingKmeans(df, Sepal_Length ~ Sepal_Width, k = 4)

    ```{r}

    t <- as.data.frame(Titanic)

    training <- createDataFrame(t)

    model <- spark.bisectingKmeans(training, Class ~ Survived, k = 4)

    summary(model)

    fitted <- predict(model, df)

    head(select(fitted, "Sepal_Length", "prediction"))

    fitted <- predict(model, training)

    head(select(fitted, "Class", "prediction"))

    ```

    #### Gaussian Mixture Model

    @@ -912,9 +914,10 @@ testSummary
  
    ### Model Persistence

    The following example shows how to save/load an ML model by SparkR.

    ```{r, warning=FALSE}

    irisDF <- createDataFrame(iris)

    gaussianGLM <- spark.glm(irisDF, Sepal_Length ~ Sepal_Width + Species, family = "gaussian")

    ```{r}

    t <- as.data.frame(Titanic)

    training <- createDataFrame(t)

    gaussianGLM <- spark.glm(training, Freq ~ Sex + Age, family = "gaussian")

    # Save and then load a fitted MLlib model

    modelPath <- tempfile(pattern = "ml", fileext = ".tmp")

    @@ -925,7 +928,7 @@ gaussianGLM2 <- read.ml(modelPath)
  
    summary(gaussianGLM2)

    # Check model prediction

    gaussianPredictions <- predict(gaussianGLM2, irisDF)

    gaussianPredictions <- predict(gaussianGLM2, training)

    head(gaussianPredictions)

    unlink(modelPath)

R/run-tests.sh

-Original file line number
+Diff line change
@@ Expand Up / @@ -23,7 +23,7 @@ FAILED=0 @@
     LOGFILE=$FWDIR/unit-tests.out
     rm -f $LOGFILE
-    SPARK_TESTING=1 $FWDIR/../bin/spark-submit --driver-java-options "-Dlog4j.configuration=file:$FWDIR/log4j.properties" --conf spark.hadoop.fs.default.name="file:///" $FWDIR/pkg/tests/run-all.R 2>&1 | tee -a $LOGFILE
+    SPARK_TESTING=1 $FWDIR/../bin/spark-submit --driver-java-options "-Dlog4j.configuration=file:$FWDIR/log4j.properties" --conf spark.hadoop.fs.defaultFS="file:///" $FWDIR/pkg/tests/run-all.R 2>&1 | tee -a $LOGFILE
     FAILED=$((PIPESTATUS[0]||$FAILED))
     NUM_TEST_WARNING="$(grep -c -e 'Warnings ----------------' $LOGFILE)"
@@ Expand Down @@

appveyor.yml

-Original file line number
+Diff line change
@@ Expand Up / @@ -46,7 +46,7 @@ build_script: @@
       - cmd: mvn -DskipTests -Psparkr -Phive -Phive-thriftserver package
     test_script:
-      - cmd: .\bin\spark-submit2.cmd --conf spark.hadoop.fs.default.name="file:///" R\pkg\tests\run-all.R
+      - cmd: .\bin\spark-submit2.cmd --conf spark.hadoop.fs.defaultFS="file:///" R\pkg\tests\run-all.R
     notifications:
       - provider: Email
@@ Expand Down @@

.../network-common/src/test/java/org/apache/spark/network/TransportResponseHandlerSuite.java

-Original file line number
+Diff line change
@@ Expand Up / @@ -26,7 +26,6 @@ @@
     import static org.junit.Assert.assertEquals;
     import static org.mockito.Mockito.*;
-    import org.apache.spark.network.buffer.ManagedBuffer;
     import org.apache.spark.network.buffer.NioManagedBuffer;
     import org.apache.spark.network.client.ChunkReceivedCallback;
     import org.apache.spark.network.client.RpcResponseCallback;
@@ Expand Down @@

common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java

-Original file line number
+Diff line change
@@ Expand Up / @@ -153,7 +153,8 @@ public void writeTo(ByteBuffer buffer) { @@
        *
        * Unlike getBytes this will not create a copy the array if this is a slice.
        */
-      public @Nonnull ByteBuffer getByteBuffer() {
+      @Nonnull
+      public ByteBuffer getByteBuffer() {
         if (base instanceof byte[] && offset >= BYTE_ARRAY_OFFSET) {
           final byte[] bytes = (byte[]) base;
@@ Expand Down @@

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

update from origin #2

Uh oh!

Diff view

Diff view

Uh oh!

There are no files selected for viewing

Uh oh!