moves function to bundle datasets into utils file under data

Author: SijanC147

docker/Dockerfile

@@ -1,4 +1,5 @@
 FROM tensorflow/serving
+COPY export/ /models/
 RUN echo '#!/bin/bash \n\n\
     tensorflow_model_server --port=8500 --rest_api_port=8501 \
     --model_config_file=/models/tfserve.conf \

main.py

@@ -15,6 +15,7 @@
     InteractiveAttentionNetwork
 )
 from tsaplay.models.Tang2016b.MemNet import MemNet
+from tsaplay.utils._data import bundle_datasets
 
 tf.logging.set_verbosity(tf.logging.INFO)
 
@@ -32,19 +33,28 @@
     "initializer": tf.initializers.random_uniform(minval=-0.1, maxval=0.1),
 }
 
-experiment = Experiment(
-    dataset=Dataset(
-        path=DATASETS.DEBUG_PATH,
-        parser=DATASETS.DEBUG_PARSER,
-        embedding=Embedding(path=EMBEDDINGS.DEBUG),
-    ),
-    model=LcrRot(),
-    contd_tag="debug",
-    # run_config=tf.estimator.RunConfig(tf_random_seed=1234),
+restaurants = Dataset(
+    path=DATASETS.XUE2018_RESTAURANTS_PATH, parser=DATASETS.XUE2018_PARSER
 )
-experiment.run(job="train+eval", steps=1)
-# experiment.export_model(overwrite=True)
-experiment.export_model(overwrite=True, restart_tfserve=True)
+laptops = Dataset(
+    path=DATASETS.XUE2018_LAPTOPS_PATH, parser=DATASETS.XUE2018_PARSER
+)
+
+rest_lapt = bundle_datasets(restaurants, laptops)
+
+# experiment = Experiment(
+#     dataset=Dataset(
+#         path=DATASETS.DEBUG_PATH,
+#         parser=DATASETS.DEBUG_PARSER,
+#         embedding=Embedding(path=EMBEDDINGS.DEBUG),
+#     ),
+#     model=LcrRot(),
+#     contd_tag="debug",
+#     # run_config=tf.estimator.RunConfig(tf_random_seed=1234),
+# )
+# experiment.run(job="train+eval", steps=1)
+# # experiment.export_model(overwrite=True)
+# experiment.export_model(overwrite=True, restart_tfserve=True)
 # experiment = Experiment(
 #     dataset=Dataset(
 #         path=DATASETS.XUE2018_LAPTOPS_PATH,

tsaplay/datasets/Dataset.py

@@ -20,7 +20,6 @@
     unpickle_file as _unpickle,
     pickle_file as _pickle,
 )
-from tsaplay.utils._data import concat_dicts_lists
 import tsaplay.datasets._constants as DATASETS
 
 
@@ -221,23 +220,6 @@ def get_features_and_labels(self, mode, distribution=None):
 
         return features, labels, stats
 
-    def __add__(self, other):
-        if isinstance(other, Dataset):
-            gen_name = "_".join([self.name, other.name])
-            gen_path = join(DATASETS.PARENT_DIR, "_generated", gen_name)
-
-            joined_train_dict = concat_dicts_lists(
-                self.train_dict, other.train_dict
-            )
-            joined_test_dict = concat_dicts_lists(
-                self.test_dict, other.test_dict
-            )
-
-            _pickle(joined_train_dict, join(gen_path, "train_dict.pkl"))
-            _pickle(joined_test_dict, join(gen_path, "test_dict.pkl"))
-
-            return Dataset(path=gen_path, parser=None)
-
     def _reset(self, path):
         self.__path = path
         self.__all_docs = None

tsaplay/utils/_data.py

@@ -1,9 +1,13 @@
 import tensorflow as tf
 from itertools import chain
+from os import makedirs
+from os.path import join, exists
 from collections import defaultdict
 from tensorflow.python.keras.preprocessing import (  # pylint: disable=E0611
     sequence
 )
+from tsaplay.datasets.Dataset import Dataset, DATASETS
+from tsaplay.utils._io import pickle_file
 
 
 def zip_str_join(first, second):
@@ -24,6 +28,33 @@ def concat_dicts_lists(first, second):
     return dict(new_dict)
 
 
+def bundle_datasets(*datasets, rebuild=False):
+    dataset_names = []
+    train_dict = {}
+    test_dict = {}
+    for dataset in datasets:
+        if isinstance(dataset, Dataset) and dataset.name not in dataset_names:
+            dataset_names.append(dataset.name)
+            train_dict = concat_dicts_lists(dataset.train_dict, train_dict)
+            test_dict = concat_dicts_lists(dataset.test_dict, test_dict)
+
+    dataset_name = "_".join(dataset_names)
+    gen_path = join(DATASETS.PARENT_DIR, "_generated", dataset_name)
+
+    makedirs(gen_path, exist_ok=True)
+
+    train_dict_path = join(gen_path, "train_dict.pkl")
+    test_dict_path = join(gen_path, "test_dict.pkl")
+
+    if not exists(train_dict_path) and not rebuild:
+        pickle_file(train_dict_path, train_dict)
+
+    if not exists(test_dict_path) and not rebuild:
+        pickle_file(test_dict_path, test_dict)
+
+    return Dataset(path=gen_path, parser=None)
+
+
 def make_labels_dataset_from_list(labels):
     low_bound = min(labels)
     if low_bound < 0: