WIP: added PythonTestInputStream

Author: giwa

examples/src/main/python/streaming/test_oprations.py

@@ -6,20 +6,14 @@
 from pyspark.streaming.duration import *
 
 if __name__ == "__main__":
-    if len(sys.argv) != 3:
-        print >> sys.stderr, "Usage: wordcount <hostname> <port>"
-        exit(-1)
     conf = SparkConf()
     conf.setAppName("PythonStreamingNetworkWordCount")
     ssc = StreamingContext(conf=conf, duration=Seconds(1))
 
-    lines = ssc.socketTextStream(sys.argv[1], int(sys.argv[2]))
-    words = lines.flatMap(lambda line: line.split(" "))
-#    ssc.checkpoint("checkpoint")
-    mapped_words = words.map(lambda word: (word, 1))
-    count = mapped_words.reduceByKey(add)
+    test_input = ssc._testInputStream([1,1,1,1])
+    mapped = test_input.map(lambda x: (x, 1))
+    mapped.pyprint()
 
-    count.pyprint()
     ssc.start()
-    ssc.awaitTermination()
+#    ssc.awaitTermination()
 #    ssc.stop()

python/pyspark/streaming/context.py

@@ -17,6 +17,7 @@
 
 import sys
 from signal import signal, SIGTERM, SIGINT
+from tempfile import NamedTemporaryFile
 
 
 from pyspark.conf import SparkConf

python/pyspark/streaming/dstream.py

@@ -156,6 +156,7 @@ def _mergeCombiners(iterator):
                     combiners[k] = v
                 else:
                     combiners[k] = mergeCombiners(combiners[k], v)
+            return combiners.iteritems()
 
         return shuffled.mapPartitions(_mergeCombiners)