try to accelerate

python/mxnet/gluon/data/text/base.py

@@ -29,6 +29,7 @@
 from ..datareader import DataReader
 from .utils import flatten_samples, collate
 
+
 class CorpusReader(DataReader):
     """Text reader that reads a whole corpus and produces a dataset based on provided
     sample splitter and word tokenizer.
@@ -63,13 +64,18 @@ def __init__(self, filename, encoding='utf8', flatten=False,
     def read(self):
         with io.open(self._filename, 'r', encoding=self._encoding) as fin:
             content = fin.read()
-        samples = (s.strip() for s in self._sample_splitter(content))
-        if self._tokenizer:
-            samples = [self._tokenizer(s) for s in samples if s]
-            if self._flatten:
-                samples = flatten_samples(samples)
-        else:
-            samples = [s for s in samples if s]
+        samples = []
+        for sample in self._sample_splitter(content):
+            if sample:
+                new_sample = sample.strip()
+                if self._tokenizer:
+                    new_sample = self._tokenizer(new_sample)
+                    if self._flatten:
+                        samples.extend(new_sample)
+                    else:
+                        samples.append(new_sample)
+                else:
+                    samples.append(new_sample)
         return SimpleDataset(samples)