add docs for use_unicode

Author: davies

python/pyspark/context.py

@@ -320,6 +320,10 @@ def textFile(self, name, minPartitions=None, use_unicode=True):
         nodes), or any Hadoop-supported file system URI, and return it as an
         RDD of Strings.
 
+        If use_unicode is False, the strings will be kept as `str` (encoding
+        as `utf-8`), which is faster and smaller than unicode. (Added in
+        Spark 1.1)
+
         >>> path = os.path.join(tempdir, "sample-text.txt")
         >>> with open(path, "w") as testFile:
         ...    testFile.write("Hello world!")
@@ -339,6 +343,10 @@ def wholeTextFiles(self, path, minPartitions=None, use_unicode=True):
         key-value pair, where the key is the path of each file, the
         value is the content of each file.
 
+        If use_unicode is False, the strings will be kept as `str` (encoding
+        as `utf-8`), which is faster and smaller than unicode. (Added in
+        Spark 1.1)
+
         For example, if you have the following files::
 
           hdfs://a-hdfs-path/part-00000

python/pyspark/serializers.py

@@ -412,18 +412,15 @@ class UTF8Deserializer(Serializer):
     def __init__(self, use_unicode=False):
         self.use_unicode = use_unicode
 
-    def loads(self, stream):
-        length = read_int(stream)
-        return stream.read(length)
-
     def load_stream(self, stream):
         try:
+            _read_int = read_int  # faster than global lookup
             if self.use_unicode:
                 while True:
-                    yield self.loads(stream).decode("utf-8")
+                    yield stream.read(_read_int(stream)).decode("utf-8")
             else:
                 while True:
-                    yield self.loads(stream)
+                    yield stream.read(_read_int(stream))
         except struct.error:
             return
         except EOFError: