Tuning spark regulation parameters

1. make sparkling parameters: rank, iteration, lambda can be loaded from
config.edn

2. change Java RRD to have only one partition.
 (defn trans-rating [item]
 -  (spark/tuple (rand-int 10)
 +  (spark/tuple 1

Author: humorless

Diff for: env/dev/resources/config.edn

@@ -1,4 +1,7 @@
 {:dev true
  :port 3000
+ :spark-rank      8
+ :spark-iteration 3
+ :spark-lambda    0.1
  ;; when :nrepl-port is set the application starts the nREPL server on load
  :nrepl-port 7000}

Diff for: src/clj/intowow/core.clj

@@ -42,6 +42,7 @@
     (log/info "model training starts at" (time/now))
     (try
       (spark/re-train)
+      (println "switch to the new matrix model!")
       (catch Exception e
         (log/error "Error in model training :" e)))
     (log/info "model training ends at" (time/now))))

Diff for: src/clj/intowow/sparkling.clj

@@ -1,6 +1,7 @@
 (ns intowow.sparkling
   (:gen-class)
   (:require [intowow.db.core :as db]
+            [intowow.config :refer [env]]
             [clojure.string :as str]
             [sparkling.conf :as conf]
             [sparkling.core :as spark]
@@ -14,7 +15,7 @@
            [org.apache.spark.mllib.recommendation ALS Rating]))
 
 (defn trans-rating [item]
-  (spark/tuple (rand-int 10)
+  (spark/tuple 1
                (Rating. (:user_id item) (:item_id item) (:rating item))))
 
 (defn get-db-ratings [sc]
@@ -56,9 +57,9 @@
   "create a new model.
    Class: org.apache.spark.mllib.recommendation.MatrixFactorizationModel"
   []
-  (let [options {:rank 8
-                 :num-iter 1
-                 :lambda 1.0}]
+  (let [options {:rank (env :spark-rank)
+                 :num-iter (env :spark-iteration)
+                 :lambda (env :spark-lambda)}]
     (-> (get-db-ratings sc)
         (spark/values)
         (alternating-least-squares options))))