Add support for L2 without standardization.

Author: holdenk

mllib/src/main/scala/org/apache/spark/ml/regression/LinearRegression.scala

@@ -453,14 +453,35 @@ private class LeastSquaresCostFun(
           case (aggregator1, aggregator2) => aggregator1.merge(aggregator2)
         })
 
-    // regVal is the sum of weight squares for L2 regularization
-    val norm = brzNorm(weights, 2.0)
-    val regVal = 0.5 * effectiveL2regParam * norm * norm
+    // If we are not doing standardization go back to unscaled weights
+    if (standardization) {
+      // regVal is the sum of weight squares for L2 regularization
+      val norm = brzNorm(weights, 2.0)
+      val regVal = 0.5 * effectiveL2regParam * norm * norm
 
-    val loss = leastSquaresAggregator.loss + regVal
-    val gradient = leastSquaresAggregator.gradient
-    axpy(effectiveL2regParam, w, gradient)
+      val loss = leastSquaresAggregator.loss + regVal
+      val gradient = leastSquaresAggregator.gradient
+      axpy(effectiveL2regParam, w, gradient)
 
-    (loss, gradient.toBreeze.asInstanceOf[BDV[Double]])
+      (loss, gradient.toBreeze.asInstanceOf[BDV[Double]])
+    } else {
+      val unscaledWeights = weights.copy
+      val len = unscaledWeights.length
+      var i = 0
+      while (i < len) {
+        unscaledWeights(i) /= featuresStd(i)
+        i += 1
+      }
+      val norm = brzNorm(unscaledWeights, 2.0)
+
+      val regVal = 0.5 * effectiveL2regParam * norm * norm
+
+      val loss = leastSquaresAggregator.loss + regVal
+      val gradient = leastSquaresAggregator.gradient
+      val mw = Vectors.dense(unscaledWeights.toArray)
+      axpy(effectiveL2regParam, mw, gradient)
+
+      (loss, gradient.toBreeze.asInstanceOf[BDV[Double]])
+    }
   }
 }

mllib/src/test/scala/org/apache/spark/ml/regression/LinearRegressionSuite.scala

@@ -125,7 +125,7 @@ class LinearRegressionSuite extends SparkFunSuite with MLlibTestSparkContext {
     assert(modelWithoutIntercept.weights(1) ~== weightsWithoutInterceptR(1) relTol 1E-3)
   }
 
-  test("linear regression with intercept with L1 regularization") {
+  test("linear regression with intercept with L1 regularization with standardization") {
     val trainer = (new LinearRegression).setElasticNetParam(1.0).setRegParam(0.57)
     val model = trainer.fit(dataset)
 
@@ -153,7 +153,7 @@ class LinearRegressionSuite extends SparkFunSuite with MLlibTestSparkContext {
     }
   }
 
-  test("linear regression with intercept with L1 regularization with standardization turned off") {
+  test("linear regression with intercept with L1 regularization without standardization") {
     val trainer = (new LinearRegression).setElasticNetParam(1.0).setRegParam(0.57)
       .setStandardization(false)
     val model = trainer.fit(dataset)
@@ -215,9 +215,9 @@ class LinearRegressionSuite extends SparkFunSuite with MLlibTestSparkContext {
      * > weights
      *  3 x 1 sparse Matrix of class "dgCMatrix"
      *                           s0
-     * (Intercept)         6.328062
-     * as.numeric.data.V2. 3.222034
-     * as.numeric.data.V3. 4.926260
+     * (Intercept)         5.269376
+     * as.numeric.data.V2. 3.736216
+     * as.numeric.data.V3. 5.712356)
      */
     val interceptR = 5.269376
     val weightsR = Array(3.736216, 5.712356)
@@ -234,6 +234,36 @@ class LinearRegressionSuite extends SparkFunSuite with MLlibTestSparkContext {
     }
   }
 
+  test("linear regression with intercept with L2 regularization without standardization") {
+    val trainer = (new LinearRegression).setElasticNetParam(0.0).setRegParam(2.3)
+      .setStandardization(false)
+    val model = trainer.fit(dataset)
+
+    /**
+     * weights <- coef(glmnet(features, label, family="gaussian", alpha = 0.0, lambda = 2.3,
+     *   standardize=FALSE))
+     * > weights
+     *  3 x 1 sparse Matrix of class "dgCMatrix"
+     *                           s0
+     * (Intercept)         5.791109
+     * as.numeric.data.V2. 3.435466
+     * as.numeric.data.V3. 5.910406
+     */
+    val interceptR = 5.791109
+    val weightsR = Array(3.435466, 5.910406)
+
+    assert(model.intercept ~== interceptR relTol 1E-3)
+    assert(model.weights(0) ~== weightsR(0) relTol 1E-3)
+    assert(model.weights(1) ~== weightsR(1) relTol 1E-3)
+
+    model.transform(dataset).select("features", "prediction").collect().foreach {
+      case Row(features: DenseVector, prediction1: Double) =>
+        val prediction2 =
+          features(0) * model.weights(0) + features(1) * model.weights(1) + model.intercept
+        assert(prediction1 ~== prediction2 relTol 1E-5)
+    }
+  }
+
   test("linear regression without intercept with L2 regularization") {
     val trainer = (new LinearRegression).setElasticNetParam(0.0).setRegParam(2.3)
       .setFitIntercept(false)
@@ -292,6 +322,36 @@ class LinearRegressionSuite extends SparkFunSuite with MLlibTestSparkContext {
     }
   }
 
+  test("linear regression with intercept with ElasticNet regularization without standardization") {
+    val trainer = (new LinearRegression).setElasticNetParam(0.3).setRegParam(1.6)
+      .setStandardization(false)
+    val model = trainer.fit(dataset)
+
+    /**
+     * weights <- coef(glmnet(features, label, family="gaussian", alpha = 0.3, lambda = 1.6
+     *  standardize=FALSE))
+     * > weights
+     * 3 x 1 sparse Matrix of class "dgCMatrix"
+     * s0
+     * (Intercept)         6.114723
+     * as.numeric.data.V2. 3.409937
+     * as.numeric.data.V3. 6.146531
+     */
+    val interceptR = 6.114723
+    val weightsR = Array(3.409937, 6.146531)
+
+    assert(model.intercept ~== interceptR relTol 1E-3)
+    assert(model.weights(0) ~== weightsR(0) relTol 1E-3)
+    assert(model.weights(1) ~== weightsR(1) relTol 1E-3)
+
+    model.transform(dataset).select("features", "prediction").collect().foreach {
+      case Row(features: DenseVector, prediction1: Double) =>
+        val prediction2 =
+          features(0) * model.weights(0) + features(1) * model.weights(1) + model.intercept
+        assert(prediction1 ~== prediction2 relTol 1E-5)
+    }
+  }
+
   test("linear regression without intercept with ElasticNet regularization") {
     val trainer = (new LinearRegression).setElasticNetParam(0.3).setRegParam(1.6)
       .setFitIntercept(false)