implements minimum 1 bucket for oov

Author: SijanC147

.autoenv.zsh

@@ -0,0 +1,8 @@
+if [[ $autoenv_event == 'enter' ]]; then
+    typeset -ax PREVIOUS_LEFT_PROMPT_ELEMENTS 
+    PREVIOUS_LEFT_PROMPT_ELEMENTS=(${(v)POWERLEVEL9K_LEFT_PROMPT_ELEMENTS})
+    POWERLEVEL9K_LEFT_PROMPT_ELEMENTS=(time root_indicator virtualenv dir vcs vi_mode)
+else
+    typeset -ax POWERLEVEL9K_LEFT_PROMPT_ELEMENTS 
+    POWERLEVEL9K_LEFT_PROMPT_ELEMENTS=(${(v)PREVIOUS_LEFT_PROMPT_ELEMENTS})
+fi

.vscode/launch.json

@@ -64,19 +64,22 @@
             "console": "integratedTerminal"
         },
         {
-            "name": "Embedding-filter-fn-test",
+            "name": "min-1-oov-bucket-fix",
             "type": "python",
             "request": "launch",
             "module": "tsaplay.task",
             "args": [
                 "single",
-                "-em='twitter-50[corpus,only_adjectives]'",
+                "-em='wiki-50[corpus]'",
                 "-ds='dong'",
-                "-m=lstm",
-                "-b=25",
-                "-s=200",
+                "-m=lcrrot",
+                "-b=5",
+                "-s=100",
                 "-mp",
-                "num_oov_buckets=100"
+                "oov=true",
+                "hidden_units=5",
+                "-aux",
+                "attn_heatmaps=false"
             ],
             "console": "integratedTerminal"
         },

tsaplay/features.py

@@ -175,6 +175,7 @@ def _init_vocab(self):
             self._vocab = read_vocab_file(vocab_file_path)
         else:
             self._vocab = self._embedding.vocab
+            #! if 0 buckets and oov = true, the train vocab is added, each will be assigned a vector
             if self._oov_fn and not self._num_oov_buckets:
                 train_vocab = set(
                     corpora_vocab(
@@ -210,19 +211,13 @@ def _init_token_data(self):
                 data_dict = getattr(self, data_dict_attr)
                 to_tokenize[mode] = data_dict
         if to_tokenize:
-            include = set(self._vocab) | (
-                set(
-                    corpora_vocab(
-                        self._train_corpus,
-                        self._test_corpus,
-                        case_insensitive=self._embedding.case_insensitive,
-                    )
+            #! Regardless of buckets, all vocab must be tokenized, otherwise risk experiment failing with empty target
+            include = set(self._vocab) | set(
+                corpora_vocab(
+                    self._train_corpus,
+                    self._test_corpus,
+                    case_insensitive=self._embedding.case_insensitive,
                 )
-                if self._num_oov_buckets
-                #! an OOV target that appears only in the test dataset oov_buckets = 0
-                #! will break this system as it will not be included in the tokens,
-                #! resulting in an empty target
-                else set()
             )
             include_tokens_path = join(self._gen_dir, "_incl_tokens.pkl")
             pickle_file(path=include_tokens_path, data=include)
@@ -268,8 +263,9 @@ def _init_tfrecords(self):
                     write_vocab_file(
                         filtered_vocab_path, filtered_vocab, indices
                     )
+                #! There has to be at least 1 bucket for any test-time oov tokens (possibly targets)
                 lookup_table = ids_lookup_table(
-                    filtered_vocab_path, self._num_oov_buckets
+                    filtered_vocab_path, max(self._num_oov_buckets, 1)
                 )
                 fetch_dict = fetch_lookup_ops(lookup_table, **tokens_lists)
                 fetch_results = run_lookups(
@@ -289,18 +285,17 @@ def _init_tfrecords(self):
                 tfrecord_folder = "_{mode}".format(mode=mode)
                 tfrecord_path = join(self._gen_dir, tfrecord_folder)
                 write_tfrecords(tfrecord_path, tfexamples)
-                if self._num_oov_buckets:
-                    buckets = [
-                        BUCKET_TOKEN.format(num=n + 1)
-                        for n in range(self._num_oov_buckets)
-                    ]
-                    oov_buckets[mode] = tokens_by_assigned_id(
-                        string_features,
-                        int_features,
-                        start=len(self._vocab),
-                        keys=buckets,
-                    )
-            if oov_buckets:
+                #! There has to be at least 1 bucket for any test-time oov tokens (possibly targets)
+                buckets = [
+                    BUCKET_TOKEN.format(num=n + 1)
+                    for n in range(max(self._num_oov_buckets, 1))
+                ]
+                oov_buckets[mode] = tokens_by_assigned_id(
+                    string_features,
+                    int_features,
+                    start=len(self._vocab),
+                    keys=buckets,
+                )
                 accum_oov_buckets = accumulate_dicts(
                     **oov_buckets,
                     accum_fn=lambda prev, curr: list(set(prev) | set(curr)),
@@ -316,17 +311,19 @@ def _init_embedding_params(self):
         np.random.seed(RANDOM_SEED)
         dim_size = self._embedding.dim_size
         vectors = self._embedding.vectors
-        num_oov_vectors = len(self._vocab) - self._embedding.vocab_size
-        num_oov_vectors += self._num_oov_buckets
-        if num_oov_vectors:
-            oov_fn = self._oov_fn or DEFAULT_OOV_FN
-            oov_vectors = oov_fn(size=(num_oov_vectors, dim_size))
-            vectors = np.concatenate([vectors, oov_vectors], axis=0)
+        #! There has to be at least 1 bucket for any test-time oov tokens (possibly targets)
+        num_oov_vectors = (self._num_oov_buckets or 1) + (
+            len(self._vocab) - self._embedding.vocab_size
+        )
+        oov_fn = self._oov_fn or DEFAULT_OOV_FN
+        oov_vectors = oov_fn(size=(num_oov_vectors, dim_size))
+        vectors = np.concatenate([vectors, oov_vectors], axis=0)
         vocab_size = len(vectors)
         num_shards = partitioner_num_shards(vocab_size)
         init_fn = embedding_initializer_fn(vectors, num_shards)
         self._embedding_params = {
             "_vocab_size": vocab_size,
+            "_num_oov_buckets": max(self._num_oov_buckets, 1),
             "_vocab_file": self._vocab_file,
             "_embedding_dim": dim_size,
             "_embedding_init": init_fn,
@@ -345,6 +342,7 @@ def _write_info_file(self):
             "embedding": {
                 "uid": self.embedding.uid,
                 "name": self.embedding.name,
+                "params": {k:stringify(v) for k,v in self._embedding_params.items()}
             },
             "oov_policy": {
                 "oov": stringify(self._oov_fn),

tsaplay/models/tsa_model.py

@@ -184,7 +184,11 @@ def _serving_input_receiver_fn(self):
         }
         parsed_example = tf.parse_example(inputs_serialized, feature_spec)
 
-        ids_table = ids_lookup_table(self.params["_vocab_file"])
+        # TODO: Why does this function call not have a value for number of OOV buckets?
+        ids_table = ids_lookup_table(
+            self.params["_vocab_file"],
+            oov_buckets=self.params["_num_oov_buckets"],
+        )
         features = {
             "left": parsed_example["left"],
             "target": parsed_example["target"],

tsaplay/task.py

@@ -238,7 +238,7 @@ def run_next_experiment(batch_file_path, job_dir=None, defaults=None):
     try:
         task_args = task_parser.parse_args(tasks[task_index])
         cprnt("RUNNING TASK {0}: {1}".format(task_index, task_args))
-        # run_experiment(task_args, experiment_index=task_index)
+        run_experiment(task_args, experiment_index=task_index)
     except Exception:  # pylint: disable=W0703
         traceback.print_exc()
     environ["TSATASK"] = str(task_index + 1)

tsaplay/utils/tf.py

@@ -471,7 +471,7 @@ def image_to_summary(name, image):
     return summary
 
 
-def ids_lookup_table(vocab_file_path, oov_buckets=0):
+def ids_lookup_table(vocab_file_path, oov_buckets=1):
     return tf.contrib.lookup.index_table_from_file(
         vocabulary_file=vocab_file_path,
         key_column_index=0,