assert when package distributed cache missing

* raises assertion when package plugin cache is not found
* test for removed module file
* test missing package plugin cache

Signed-off-by: Jeffrey Martin <[email protected]>

garak/_plugins.py

@@ -60,8 +60,9 @@ def _extract_modules_klasses(base_klass):
         ]
 
     def _load_plugin_cache(self):
-        if not os.path.exists(self._plugin_cache_filename):
-            self._build_plugin_cache()
+        assert os.path.exists(
+            self._plugin_cache_filename
+        ), f"{self._plugin_cache_filename} is missing or corrupt."
 
         base_time = datetime.fromtimestamp(
             os.path.getmtime(self._plugin_cache_filename), tz=timezone.utc

tests/plugins/test_plugin_cache.py

@@ -1,8 +1,11 @@
+import json
 import pytest
 import os
+import sys
 import tempfile
 
-from garak._plugins import PluginCache
+from pathlib import Path
+from garak._plugins import PluginCache, PluginEncoder
 
 
 @pytest.fixture(autouse=True)
@@ -17,8 +20,7 @@ def reset_plugin_cache():
 def temp_cache_location(request) -> None:
     # override the cache file with a tmp location
     with tempfile.NamedTemporaryFile(buffering=0, delete=False) as tmp:
-        PluginCache._user_plugin_cache_filename = tmp.name
-        PluginCache._plugin_cache_filename = tmp.name
+        PluginCache._user_plugin_cache_filename = Path(tmp.name)
         tmp.close()
         os.remove(tmp.name)
     # reset the class level singleton
@@ -33,6 +35,25 @@ def remove_cache_file():
     return tmp.name
 
 
+@pytest.fixture
+def remove_package_cache(request) -> None:
+    # override the cache file with a tmp location
+    original_path = PluginCache._plugin_cache_filename
+    with tempfile.NamedTemporaryFile(buffering=0, delete=False) as tmp:
+        PluginCache._plugin_cache_filename = Path(tmp.name)
+        tmp.close()
+        os.remove(tmp.name)
+    # reset the class level singleton
+    PluginCache._plugin_cache_dict = None
+
+    def restore_package_path():
+        PluginCache._plugin_cache_filename = original_path
+
+    request.addfinalizer(restore_package_path)
+
+    return tmp.name
+
+
 def test_create(temp_cache_location):
     cache = PluginCache.instance()
     assert os.path.isfile(temp_cache_location)
@@ -74,3 +95,26 @@ def test_unknown_module():
     with pytest.raises(ValueError) as exc_info:
         info = PluginCache.plugin_info("probes.invalid.format.length")
     assert "plugin class" in str(exc_info.value)
+
+
+@pytest.skipif(
+    sys.platform == "win32", "Windows Github executor does not raise the ValueError"
+)
+def test_module_removed(temp_cache_location):
+    cache = PluginCache.instance()
+    cache["probes"]["probes.invalid.Removed"] = {
+        "description": "Testing value to be purged"
+    }
+    with open(temp_cache_location, "w", encoding="utf-8") as cache_file:
+        json.dump(cache, cache_file, cls=PluginEncoder, indent=2)
+    PluginCache._plugin_cache_dict = None
+    with pytest.raises(ValueError) as exc_info:
+        PluginCache.plugin_info("probes.invalid.Removed")
+    assert "plugin module" in str(exc_info.value)
+
+
+@pytest.mark.usefixtures("remove_package_cache")
+def test_report_missing_package_file():
+    with pytest.raises(AssertionError) as exc_info:
+        PluginCache.instance()
+    assert "is missing or corrupt" in str(exc_info.value)