Try retrying to resolve intermittent failures (#89)

* Try retrying to resolve intermittent failures * Config adjustments
climatepolicyradar · Jan 31, 2024 · fbb08e7 · fbb08e7
1 parent 91cff61
commit fbb08e7
Show file tree

Hide file tree

Showing 5 changed files with 119 additions and 7 deletions.
diff --git a/cli/index_data.py b/cli/index_data.py
@@ -2,6 +2,7 @@
 
 import os
 import sys
+import time
 import logging
 import logging.config
 from pathlib import Path
@@ -119,7 +120,10 @@ def run_as_cli(
         tasks, embedding_dir_as_path = _get_index_tasks(
             indexer_input_dir, s3, files_to_index, limit
         )
+        start = time.time()
         populate_vespa(tasks=tasks, embedding_dir_as_path=embedding_dir_as_path)
+        duration = time.time() - start
+        _LOGGER.info(f"Vespa indexing completed after: {duration}s")
         sys.exit(0)
     _LOGGER.error(f"Unknown index type: {index_type}")
     sys.exit(1)