256 candidates (1024 with Grammar)

Author: Nexesenex

gpttype_adapter.cpp

@@ -1758,15 +1758,15 @@ const std::vector<samplers> & sampler_order, llama_grammar * grammar, float dyna
     bool use_grammar = grammar != nullptr;
     std::vector<llama_token_data> precache = (use_grammar ? std::vector<llama_token_data>(candidates) : std::vector<llama_token_data>(0));
 
-    sample_top_k(&candidates_p, 3000);
+    sample_top_k(&candidates_p, 256);
 
     if (use_grammar) {
         sample_grammar(file_format, n_vocab, &candidates_p, grammar);
-        // if top_k 3000 doesn't contain a valid candidate for this grammar, try again pre-cull
+        // if top_k 3000 (1024 for Croco) doesn't contain a valid candidate for this grammar, try again pre-cull
         if (candidates_p.size <= 0) {
             candidates_p = { precache.data(), precache.size(), false };
             sample_grammar(file_format, n_vocab, &candidates_p, grammar);
-            sample_top_k(&candidates_p, 3000);
+            sample_top_k(&candidates_p, 1024);
         }
     }