adding topics version to training

dod-advana · Feb 15, 2022 · a310817 · a310817
1 parent a70d704
commit a310817
Show file tree

Hide file tree

Showing 5 changed files with 17 additions and 30 deletions.
diff --git a/gamechangerml/api/fastapi/model_loader.py b/gamechangerml/api/fastapi/model_loader.py
@@ -108,7 +108,7 @@ def getTopicModel(self):
             )
             ModelLoader.initTopics()
         return ModelLoader.__topic_model
-    
+
     def getRecommender(self):
         if ModelLoader.__recommender is None:
             logger.warning(
@@ -211,7 +211,8 @@ def initSentenceSearcher(
         Args:
         Returns:
         """
-        logger.info(f"Loading Sentence Searcher with sent index path: {index_path}")
+        logger.info(
+            f"Loading Sentence Searcher with sent index path: {index_path}")
         try:
             if MODEL_LOAD_FLAG:
                 ModelLoader.__sentence_searcher = SentenceSearcher(
@@ -259,23 +260,24 @@ def initSentenceEncoder(transformer_path=LOCAL_TRANSFORMERS_DIR.value):
     def initSparse(model_name=latest_intel_model_trans.value):
         try:
             if MODEL_LOAD_FLAG:
-                ModelLoader.__sparse_reader = sparse.SparseReader(model_name=model_name)
+                ModelLoader.__sparse_reader = sparse.SparseReader(
+                    model_name=model_name)
                 logger.info(f"Sparse Reader: {model_name} loaded")
         except Exception as e:
             logger.warning("** Could not load Sparse Reader")
             logger.warning(e)
 
     @staticmethod
-    def initTopics() -> None:
+    def initTopics(model_path=TOPICS_MODEL.value) -> None:
         """initTopics - load topics model on start
         Args:
         Returns:
         """
         try:
             if MODEL_LOAD_FLAG:
-                logger.info("Starting Topic pipeline")
+                logger.info(f"Loading topic model {model_path}")
                 logger.info(TopicsConfig.DATA_ARGS)
-                ModelLoader.__topic_model = Topics(directory=TOPICS_MODEL.value)
+                ModelLoader.__topic_model = Topics(directory=model_path)
                 logger.info("Finished loading Topic Model")
         except Exception as e:
             logger.warning("** Could not load Topic model")
@@ -293,4 +295,4 @@ def initRecommender():
                 ModelLoader.__recommender = Recommender()
                 logger.info("Finished loading Recommender")
         except OSError:
-            logger.error(f"** Could not load Recommender") 
+            logger.error(f"** Could not load Recommender")
diff --git a/gamechangerml/api/fastapi/routers/controls.py b/gamechangerml/api/fastapi/routers/controls.py
@@ -518,21 +518,21 @@ def reload_thread(model_dict):
                         processmanager.reloading, progress, total
                     )
                 if "jbook_qexp" in model_dict:
-                    qexp_name = os.path.join(
+                    jbook_qexp_name = os.path.join(
                         Config.LOCAL_PACKAGED_MODELS_DIR, model_dict["jbook_qexp"]
                     )
                     # uses QEXP_MODEL_NAME by default
                     logger.info("Attempting to load Jbook QE")
-                    MODELS.initQE(qexp_name)
-                    QEXP_MODEL_NAME.value = qexp_name
+                    MODELS.initQEJBook(jbook_qexp_name)
+                    QEXP_JBOOK_MODEL.value = jbook_qexp_name
                     progress += 1
                     processmanager.update_status(
                         processmanager.reloading, progress, total
                     )
 
-                if "topics" in model_dict:
+                if "topic_models" in model_dict:
                     topics_name = os.path.join(
-                        Config.LOCAL_PACKAGED_MODELS_DIR, model_dict["topics"]
+                        Config.LOCAL_PACKAGED_MODELS_DIR, model_dict["topic_models"]
                     )
 
                     logger.info("Attempting to load Topics")
@@ -708,7 +708,8 @@ def train_topics(model_dict):
     logger.info("Attempting to train topic model")
     logger.info(model_dict)
     args = {"sample_rate": model_dict["sample_rate"],
-            "upload": model_dict["upload"]}
+            "upload": model_dict["upload"],
+            "version": model_dict["version"]}
     pipeline.run(
         build_type=model_dict["build_type"],
         run_name=datetime.now().strftime("%Y%m%d"),

diff --git a/gamechangerml/models/topic_models/__init__.py b/gamechangerml/models/topic_models/__init__.py
diff --git a/gamechangerml/models/topic_models/tfidf.py b/gamechangerml/models/topic_models/tfidf.py
diff --git a/gamechangerml/train/pipeline.py b/gamechangerml/train/pipeline.py
@@ -570,9 +570,8 @@ def create_ltr(self, daysBack: int = 180):
         except Exception as e:
             logger.error("Could not create LTR")
 
-    def create_topics(self, sample_rate=None, upload=False, corpus_dir=CORPUS_DIR):
+    def create_topics(self, sample_rate=None, upload=False, corpus_dir=CORPUS_DIR, version="v2"):
         try:
-            version = "v2"
             model_id = datetime.now().strftime("%Y%m%d%H%M%S")
             model_dir = DefaultConfig.LOCAL_MODEL_DIR