hide APIs as much as I can

Author: mengxr

examples/src/main/java/org/apache/spark/examples/ml/JavaSimpleTextClassificationPipeline.java

@@ -27,16 +27,16 @@
 import org.apache.spark.ml.PipelineStage;
 import org.apache.spark.ml.classification.LogisticRegression;
 import org.apache.spark.ml.feature.HashingTF;
+import org.apache.spark.ml.feature.Tokenizer;
 import org.apache.spark.sql.api.java.JavaSQLContext;
 import org.apache.spark.sql.api.java.JavaSchemaRDD;
 import org.apache.spark.sql.api.java.Row;
 import org.apache.spark.SparkConf;
 
 /**
  * A simple text classification pipeline that recognizes "spark" from input text. It uses the Java
- * bean classes {@link LabeledDocument} and {@link Document}, and the tokenizer {@link MyTokenizer}
- * defined in the Scala counterpart of this example {@link SimpleTextClassificationPipeline}.
- * Run with
+ * bean classes {@link LabeledDocument} and {@link Document} defined in the Scala counterpart of
+ * this example {@link SimpleTextClassificationPipeline}. Run with
  * <pre>
  * bin/run-example ml.JavaSimpleTextClassificationPipeline
  * </pre>
@@ -58,7 +58,7 @@ public static void main(String[] args) {
       jsql.applySchema(jsc.parallelize(localTraining), LabeledDocument.class);
 
     // Configure an ML pipeline, which consists of three stages: tokenizer, hashingTF, and lr.
-    MyTokenizer tokenizer = new MyTokenizer()
+    Tokenizer tokenizer = new Tokenizer()
       .setInputCol("text")
       .setOutputCol("words");
     HashingTF hashingTF = new HashingTF()

examples/src/main/scala/org/apache/spark/examples/ml/SimpleTextClassificationPipeline.scala

@@ -20,35 +20,20 @@ package org.apache.spark.examples.ml
 import scala.beans.BeanInfo
 
 import org.apache.spark.{SparkConf, SparkContext}
-import org.apache.spark.ml.{Pipeline, UnaryTransformer}
+import org.apache.spark.ml.Pipeline
 import org.apache.spark.ml.classification.LogisticRegression
-import org.apache.spark.ml.feature.HashingTF
-import org.apache.spark.ml.param.ParamMap
-import org.apache.spark.sql.{DataType, SQLContext, StringType}
+import org.apache.spark.ml.feature.{HashingTF, Tokenizer}
+import org.apache.spark.sql.SQLContext
 
 @BeanInfo
 case class LabeledDocument(id: Long, text: String, label: Double)
 
 @BeanInfo
 case class Document(id: Long, text: String)
 
-/**
- * A tokenizer that converts the input string to lowercase and then splits it by white spaces.
- */
-class MyTokenizer extends UnaryTransformer[String, Seq[String], MyTokenizer] {
-
-  override def createTransformFunc(paramMap: ParamMap): String => Seq[String] = {
-    _.toLowerCase.split("\\s")
-  }
-
-  override protected def validateInputType(inputType: DataType): Unit = {
-    require(inputType == StringType, s"Input type must be string type but got $inputType.")
-  }
-}
-
 /**
  * A simple text classification pipeline that recognizes "spark" from input text. This is to show
- * how to define a simple tokenizer and then use it as part of a ML pipeline. Run with
+ * how to create and configure an ML pipeline. Run with
  * {{{
  * bin/run-example ml.SimpleTextClassificationPipeline
  * }}}
@@ -69,7 +54,7 @@ object SimpleTextClassificationPipeline {
       LabeledDocument(3L, "hadoop mapreduce", 0.0)))
 
     // Configure an ML pipeline, which consists of three stages: tokenizer, hashingTF, and lr.
-    val tokenizer = new MyTokenizer()
+    val tokenizer = new Tokenizer()
       .setInputCol("text")
       .setOutputCol("words")
     val hashingTF = new HashingTF()

mllib/src/main/scala/org/apache/spark/ml/Identifiable.scala

@@ -19,18 +19,15 @@ package org.apache.spark.ml
 
 import java.util.UUID
 
-import org.apache.spark.annotation.AlphaComponent
-
 /**
- * :: AlphaComponent ::
  * Object with a unique id.
  */
-@AlphaComponent
-trait Identifiable extends Serializable {
+private[ml] trait Identifiable extends Serializable {
 
   /**
    * A unique id for the object. The default implementation concatenates the class name, "-", and 8
    * random hex chars.
    */
-  val uid: String = this.getClass.getSimpleName + "-" + UUID.randomUUID().toString.take(8)
+  private[ml] val uid: String =
+    this.getClass.getSimpleName + "-" + UUID.randomUUID().toString.take(8)
 }

mllib/src/main/scala/org/apache/spark/ml/Pipeline.scala

@@ -21,20 +21,20 @@ import scala.collection.mutable.ListBuffer
 
 import org.apache.spark.Logging
 import org.apache.spark.annotation.AlphaComponent
-import org.apache.spark.ml.param.{Param, ParamMap}
+import org.apache.spark.ml.param.{Params, Param, ParamMap}
 import org.apache.spark.sql.{SchemaRDD, StructType}
 
 /**
  * :: AlphaComponent ::
- * A stage in a pipeline, either an Estimator or an Transformer.
+ * A stage in a pipeline, either an [[Estimator]] or a [[Transformer]].
  */
 @AlphaComponent
 abstract class PipelineStage extends Serializable with Logging {
 
   /**
    * Derives the output schema from the input schema and parameters.
    */
-  def transformSchema(schema: StructType, paramMap: ParamMap): StructType
+  private[ml] def transformSchema(schema: StructType, paramMap: ParamMap): StructType
 
   /**
    * Derives the output schema from the input schema and parameters, optionally with logging.
@@ -123,7 +123,7 @@ class Pipeline extends Estimator[PipelineModel] {
     new PipelineModel(this, map, transformers.toArray)
   }
 
-  override def transformSchema(schema: StructType, paramMap: ParamMap): StructType = {
+  private[ml] override def transformSchema(schema: StructType, paramMap: ParamMap): StructType = {
     val map = this.paramMap ++ paramMap
     val theStages = map(stages)
     require(theStages.toSet.size == theStages.size,
@@ -137,23 +137,23 @@ class Pipeline extends Estimator[PipelineModel] {
  * Represents a compiled pipeline.
  */
 @AlphaComponent
-class PipelineModel(
+class PipelineModel private[ml] (
     override val parent: Pipeline,
     override val fittingParamMap: ParamMap,
-    val transformers: Array[Transformer])
+    private[ml] val stages: Array[Transformer])
   extends Model[PipelineModel] with Logging {
 
   /**
    * Gets the model produced by the input estimator. Throws an NoSuchElementException is the input
    * estimator does not exist in the pipeline.
    */
-  def getModel[M <: Model[M]](estimator: Estimator[M]): M = {
-    val matched = transformers.filter {
-      case m: Model[_] => m.parent.eq(estimator)
+  def getModel[M <: Model[M]](stage: Estimator[M]): M = {
+    val matched = stages.filter {
+      case m: Model[_] => m.parent.eq(stage)
       case _ => false
     }
     if (matched.isEmpty) {
-      throw new NoSuchElementException(s"Cannot find estimator $estimator from the pipeline.")
+      throw new NoSuchElementException(s"Cannot find stage $stage from the pipeline.")
     } else if (matched.size > 1) {
       throw new IllegalStateException(s"Cannot have duplicate estimators in the sample pipeline.")
     } else {
@@ -163,10 +163,10 @@ class PipelineModel(
 
   override def transform(dataset: SchemaRDD, paramMap: ParamMap): SchemaRDD = {
     transformSchema(dataset.schema, paramMap, logging = true)
-    transformers.foldLeft(dataset)((cur, transformer) => transformer.transform(cur, paramMap))
+    stages.foldLeft(dataset)((cur, transformer) => transformer.transform(cur, paramMap))
   }
 
-  override def transformSchema(schema: StructType, paramMap: ParamMap): StructType = {
-    transformers.foldLeft(schema)((cur, transformer) => transformer.transformSchema(cur, paramMap))
+  private[ml] override def transformSchema(schema: StructType, paramMap: ParamMap): StructType = {
+    stages.foldLeft(schema)((cur, transformer) => transformer.transformSchema(cur, paramMap))
   }
 }

mllib/src/main/scala/org/apache/spark/ml/Transformer.scala

@@ -31,7 +31,7 @@ import org.apache.spark.sql.catalyst.dsl._
 import org.apache.spark.sql.catalyst.types._
 
 /**
- * :: AlphaComponet ::
+ * :: AlphaComponent ::
  * Abstract class for transformers that transform one dataset into another.
  */
 @AlphaComponent
@@ -83,12 +83,10 @@ abstract class Transformer extends PipelineStage with Params {
 }
 
 /**
- * :: AlphaComponent ::
  * Abstract class for transformers that take one input column, apply transformation, and output the
  * result as a new column.
  */
-@AlphaComponent
-abstract class UnaryTransformer[IN, OUT: TypeTag, T <: UnaryTransformer[IN, OUT, T]]
+private[ml] abstract class UnaryTransformer[IN, OUT: TypeTag, T <: UnaryTransformer[IN, OUT, T]]
   extends Transformer with HasInputCol with HasOutputCol with Logging {
 
   def setInputCol(value: String): T = set(inputCol, value).asInstanceOf[T]

mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala

@@ -105,7 +105,7 @@ class LogisticRegression extends Estimator[LogisticRegressionModel] with Logisti
     lrm
   }
 
-  override def transformSchema(schema: StructType, paramMap: ParamMap): StructType = {
+  private[ml] override def transformSchema(schema: StructType, paramMap: ParamMap): StructType = {
     validateAndTransformSchema(schema, paramMap, fitting = true)
   }
 }
@@ -118,15 +118,15 @@ class LogisticRegression extends Estimator[LogisticRegressionModel] with Logisti
 class LogisticRegressionModel private[ml] (
     override val parent: LogisticRegression,
     override val fittingParamMap: ParamMap,
-    val weights: Vector)
+    weights: Vector)
   extends Model[LogisticRegressionModel] with LogisticRegressionParams {
 
   def setThreshold(value: Double): this.type = set(threshold, value)
   def setFeaturesCol(value: String): this.type = set(featuresCol, value)
   def setScoreCol(value: String): this.type = set(scoreCol, value)
   def setPredictionCol(value: String): this.type = set(predictionCol, value)
 
-  override def transformSchema(schema: StructType, paramMap: ParamMap): StructType = {
+  private[ml] override def transformSchema(schema: StructType, paramMap: ParamMap): StructType = {
     validateAndTransformSchema(schema, paramMap, fitting = false)
   }
 

mllib/src/main/scala/org/apache/spark/ml/feature/StandardScaler.scala

@@ -56,7 +56,7 @@ class StandardScaler extends Estimator[StandardScalerModel] with StandardScalerP
     model
   }
 
-  override def transformSchema(schema: StructType, paramMap: ParamMap): StructType = {
+  private[ml] override def transformSchema(schema: StructType, paramMap: ParamMap): StructType = {
     val map = this.paramMap ++ paramMap
     val inputType = schema(map(inputCol)).dataType
     require(inputType.isInstanceOf[VectorUDT],
@@ -92,7 +92,7 @@ class StandardScalerModel private[ml] (
     dataset.select(Star(None), scale.call(map(inputCol).attr) as map(outputCol))
   }
 
-  override def transformSchema(schema: StructType, paramMap: ParamMap): StructType = {
+  private[ml] override def transformSchema(schema: StructType, paramMap: ParamMap): StructType = {
     val map = this.paramMap ++ paramMap
     val inputType = schema(map(inputCol)).dataType
     require(inputType.isInstanceOf[VectorUDT],

mllib/src/main/scala/org/apache/spark/ml/feature/Tokenizer.scala

@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.feature
+
+import org.apache.spark.annotation.AlphaComponent
+import org.apache.spark.ml.UnaryTransformer
+import org.apache.spark.ml.param.ParamMap
+import org.apache.spark.sql.{DataType, StringType}
+
+/**
+ * :: AlphaComponent ::
+ * A tokenizer that converts the input string to lowercase and then splits it by white spaces.
+ */
+@AlphaComponent
+class Tokenizer extends UnaryTransformer[String, Seq[String], Tokenizer] {
+
+  protected override def createTransformFunc(paramMap: ParamMap): String => Seq[String] = {
+    _.toLowerCase.split("\\s")
+  }
+
+  protected override def validateInputType(inputType: DataType): Unit = {
+    require(inputType == StringType, s"Input type must be string type but got $inputType.")
+  }
+}

mllib/src/main/scala/org/apache/spark/ml/param/params.scala

@@ -187,7 +187,7 @@ private[ml] object Params {
    * @param parent the parent estimator
    * @param child the child model
    */
-  private[ml] def inheritValues[E <: Params, M <: E](
+  def inheritValues[E <: Params, M <: E](
       paramMap: ParamMap,
       parent: E,
       child: M): Unit = {

mllib/src/main/scala/org/apache/spark/ml/tuning/CrossValidator.scala

@@ -99,7 +99,7 @@ class CrossValidator extends Estimator[CrossValidatorModel] with CrossValidatorP
     cvModel
   }
 
-  override def transformSchema(schema: StructType, paramMap: ParamMap): StructType = {
+  private[ml] override def transformSchema(schema: StructType, paramMap: ParamMap): StructType = {
     val map = this.paramMap ++ paramMap
     map(estimator).transformSchema(schema, paramMap)
   }
@@ -120,7 +120,7 @@ class CrossValidatorModel private[ml] (
     bestModel.transform(dataset, paramMap)
   }
 
-  override def transformSchema(schema: StructType, paramMap: ParamMap): StructType = {
+  private[ml] override def transformSchema(schema: StructType, paramMap: ParamMap): StructType = {
     bestModel.transformSchema(schema, paramMap)
   }
 }