diff --git a/src/autolabel/few_shot/__init__.py b/src/autolabel/few_shot/__init__.py
index 028a78d6..01ad1317 100644
--- a/src/autolabel/few_shot/__init__.py
+++ b/src/autolabel/few_shot/__init__.py
@@ -104,6 +104,10 @@ def initialize_selector(
 
         example_cls = ALGORITHM_TO_IMPLEMENTATION[algorithm]
 
-        params["cache"] = cache
+        if algorithm not in [
+            FewShotAlgorithm.FIXED,
+            FewShotAlgorithm.LABEL_DIVERSITY_RANDOM,
+        ]:
+            params["cache"] = cache
 
         return example_cls.from_examples(**params)