Skip to content

[SPARK-16519][SPARKR] Handle SparkR RDD generics that create warnings in R CMD check #14626

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
felixcheung wants to merge 3 commits into from
Closed

[SPARK-16519][SPARKR] Handle SparkR RDD generics that create warnings in R CMD check #14626

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
fd53bcb
rename RDD functions
felixcheung Aug 12, 2016
88bd34d
more fix, reordered generics so they match the right section group
felixcheung Aug 13, 2016
2723eca
move partitionBy setGeneric
felixcheung Aug 16, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
100 changes: 50 additions & 50 deletions R/pkg/R/RDD.R
View file
Open in desktop

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion R/pkg/R/SQLContext.R
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -218,7 +218,7 @@ createDataFrame.default <- function(data, schema = NULL, samplingRatio = 1.0) {
}

if (is.null(schema) || (!inherits(schema, "structType") && is.null(names(schema)))) {
row <- first(rdd)
row <- firstRDD(rdd)
names <- if (is.null(schema)) {
names(row)
} else {
Expand Down
2 changes: 1 addition & 1 deletion R/pkg/R/context.R
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -267,7 +267,7 @@ spark.lapply <- function(list, func) {
sc <- getSparkContext()
rdd <- parallelize(sc, list, length(list))
results <- map(rdd, func)
local <- collect(results)
local <- collectRDD(results)
local
}

Expand Down
91 changes: 58 additions & 33 deletions R/pkg/R/generics.R
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,7 @@
setGeneric("aggregateRDD",
function(x, zeroValue, seqOp, combOp) { standardGeneric("aggregateRDD") })

# @rdname cache-methods
# @export
setGeneric("cache", function(x) { standardGeneric("cache") })
setGeneric("cacheRDD", function(x) { standardGeneric("cacheRDD") })

# @rdname coalesce
# @seealso repartition
Expand All @@ -36,9 +34,7 @@ setGeneric("coalesce", function(x, numPartitions, ...) { standardGeneric("coales
# @export
setGeneric("checkpoint", function(x) { standardGeneric("checkpoint") })

# @rdname collect-methods
# @export
setGeneric("collect", function(x, ...) { standardGeneric("collect") })
setGeneric("collectRDD", function(x, ...) { standardGeneric("collectRDD") })

# @rdname collect-methods
# @export
Expand All @@ -51,9 +47,9 @@ setGeneric("collectPartition",
standardGeneric("collectPartition")
})

# @rdname nrow
# @export
setGeneric("count", function(x) { standardGeneric("count") })
setGeneric("countRDD", function(x) { standardGeneric("countRDD") })

setGeneric("lengthRDD", function(x) { standardGeneric("lengthRDD") })

# @rdname countByValue
# @export
Expand All @@ -74,17 +70,13 @@ setGeneric("approxQuantile",
standardGeneric("approxQuantile")
})

# @rdname distinct
# @export
setGeneric("distinct", function(x, numPartitions = 1) { standardGeneric("distinct") })
setGeneric("distinctRDD", function(x, numPartitions = 1) { standardGeneric("distinctRDD") })

# @rdname filterRDD
# @export
setGeneric("filterRDD", function(x, f) { standardGeneric("filterRDD") })

# @rdname first
# @export
setGeneric("first", function(x, ...) { standardGeneric("first") })
setGeneric("firstRDD", function(x, ...) { standardGeneric("firstRDD") })

# @rdname flatMap
# @export
Expand All @@ -110,6 +102,8 @@ setGeneric("glom", function(x) { standardGeneric("glom") })
# @export
setGeneric("histogram", function(df, col, nbins=10) { standardGeneric("histogram") })

setGeneric("joinRDD", function(x, y, ...) { standardGeneric("joinRDD") })

# @rdname keyBy
# @export
setGeneric("keyBy", function(x, func) { standardGeneric("keyBy") })
Expand Down Expand Up @@ -152,9 +146,7 @@ setGeneric("getNumPartitions", function(x) { standardGeneric("getNumPartitions")
# @export
setGeneric("numPartitions", function(x) { standardGeneric("numPartitions") })

# @rdname persist
# @export
setGeneric("persist", function(x, newLevel) { standardGeneric("persist") })
setGeneric("persistRDD", function(x, newLevel) { standardGeneric("persistRDD") })

# @rdname pipeRDD
# @export
Expand All @@ -168,10 +160,7 @@ setGeneric("pivot", function(x, colname, values = list()) { standardGeneric("piv
# @export
setGeneric("reduce", function(x, func) { standardGeneric("reduce") })

# @rdname repartition
# @seealso coalesce
# @export
setGeneric("repartition", function(x, ...) { standardGeneric("repartition") })
setGeneric("repartitionRDD", function(x, ...) { standardGeneric("repartitionRDD") })

# @rdname sampleRDD
# @export
Expand All @@ -193,16 +182,16 @@ setGeneric("saveAsTextFile", function(x, path) { standardGeneric("saveAsTextFile
# @export
setGeneric("setName", function(x, name) { standardGeneric("setName") })

setGeneric("showRDD", function(object, ...) { standardGeneric("showRDD") })

# @rdname sortBy
# @export
setGeneric("sortBy",
function(x, func, ascending = TRUE, numPartitions = 1) {
standardGeneric("sortBy")
})

# @rdname take
# @export
setGeneric("take", function(x, num) { standardGeneric("take") })
setGeneric("takeRDD", function(x, num) { standardGeneric("takeRDD") })

# @rdname takeOrdered
# @export
Expand All @@ -223,9 +212,7 @@ setGeneric("top", function(x, num) { standardGeneric("top") })
# @export
setGeneric("unionRDD", function(x, y) { standardGeneric("unionRDD") })

# @rdname unpersist-methods
# @export
setGeneric("unpersist", function(x, ...) { standardGeneric("unpersist") })
setGeneric("unpersistRDD", function(x, ...) { standardGeneric("unpersistRDD") })

# @rdname zipRDD
# @export
Expand Down Expand Up @@ -343,9 +330,7 @@ setGeneric("join", function(x, y, ...) { standardGeneric("join") })
# @export
setGeneric("leftOuterJoin", function(x, y, numPartitions) { standardGeneric("leftOuterJoin") })

#' @rdname partitionBy
#' @export
setGeneric("partitionBy", function(x, ...) { standardGeneric("partitionBy") })
setGeneric("partitionByRDD", function(x, ...) { standardGeneric("partitionByRDD") })

# @rdname reduceByKey
# @seealso groupByKey
Expand Down Expand Up @@ -414,6 +399,14 @@ setGeneric("as.data.frame",
#' @export
setGeneric("attach")

#' @rdname cache
#' @export
setGeneric("cache", function(x) { standardGeneric("cache") })

#' @rdname collect
#' @export
setGeneric("collect", function(x, ...) { standardGeneric("collect") })

#' @rdname columns
#' @export
setGeneric("colnames", function(x, do.NULL = TRUE, prefix = "col") { standardGeneric("colnames") })
Expand All @@ -434,6 +427,10 @@ setGeneric("coltypes<-", function(x, value) { standardGeneric("coltypes<-") })
#' @export
setGeneric("columns", function(x) {standardGeneric("columns") })

#' @rdname nrow
#' @export
setGeneric("count", function(x) { standardGeneric("count") })

#' @rdname cov
#' @export
setGeneric("cov", function(x, ...) {standardGeneric("cov") })
Expand Down Expand Up @@ -477,6 +474,10 @@ setGeneric("gapplyCollect", function(x, ...) { standardGeneric("gapplyCollect")
#' @export
setGeneric("describe", function(x, col, ...) { standardGeneric("describe") })

#' @rdname distinct
#' @export
setGeneric("distinct", function(x) { standardGeneric("distinct") })

#' @rdname drop
#' @export
setGeneric("drop", function(x, ...) { standardGeneric("drop") })
Expand Down Expand Up @@ -519,6 +520,10 @@ setGeneric("fillna", function(x, value, cols = NULL) { standardGeneric("fillna")
#' @export
setGeneric("filter", function(x, condition) { standardGeneric("filter") })

#' @rdname first
#' @export
setGeneric("first", function(x, ...) { standardGeneric("first") })

#' @rdname groupBy
#' @export
setGeneric("group_by", function(x, ...) { standardGeneric("group_by") })
Expand Down Expand Up @@ -555,17 +560,25 @@ setGeneric("mutate", function(.data, ...) {standardGeneric("mutate") })
#' @export
setGeneric("orderBy", function(x, col, ...) { standardGeneric("orderBy") })

#' @rdname persist
#' @export
setGeneric("persist", function(x, newLevel) { standardGeneric("persist") })

#' @rdname printSchema
#' @export
setGeneric("printSchema", function(x) { standardGeneric("printSchema") })

#' @rdname registerTempTable-deprecated
#' @export
setGeneric("registerTempTable", function(x, tableName) { standardGeneric("registerTempTable") })

#' @rdname rename
#' @export
setGeneric("rename", function(x, ...) { standardGeneric("rename") })

#' @rdname registerTempTable-deprecated
#' @rdname repartition
#' @export
setGeneric("registerTempTable", function(x, tableName) { standardGeneric("registerTempTable") })
setGeneric("repartition", function(x, ...) { standardGeneric("repartition") })

#' @rdname sample
#' @export
Expand All @@ -592,6 +605,10 @@ setGeneric("saveAsTable", function(df, tableName, source = NULL, mode = "error",
#' @export
setGeneric("str")

#' @rdname take
#' @export
setGeneric("take", function(x, num) { standardGeneric("take") })

#' @rdname mutate
#' @export
setGeneric("transform", function(`_data`, ...) {standardGeneric("transform") })
Expand Down Expand Up @@ -674,6 +691,10 @@ setGeneric("union", function(x, y) { standardGeneric("union") })
#' @export
setGeneric("unionAll", function(x, y) { standardGeneric("unionAll") })

#' @rdname unpersist-methods
#' @export
setGeneric("unpersist", function(x, ...) { standardGeneric("unpersist") })

#' @rdname filter
#' @export
setGeneric("where", function(x, condition) { standardGeneric("where") })
Expand Down Expand Up @@ -771,6 +792,10 @@ setGeneric("over", function(x, window) { standardGeneric("over") })

###################### WindowSpec Methods ##########################

#' @rdname partitionBy
#' @export
setGeneric("partitionBy", function(x, ...) { standardGeneric("partitionBy") })

#' @rdname rowsBetween
#' @export
setGeneric("rowsBetween", function(x, start, end) { standardGeneric("rowsBetween") })
Expand Down
Loading