dont construct new locale every time

ggerganov · Feb 29, 2024 · 4a24bdf · 4a24bdf
1 parent 4bffc07
commit 4a24bdf
Showing 1 changed file with 2 additions and 1 deletion.
diff --git a/llama.cpp b/llama.cpp
@@ -9001,12 +9001,13 @@ struct llm_tokenizer_wpm {
     }
 
     uint32_t to_lower(uint32_t code) {
+        static const std::locale locale("en_US.UTF-8");
 #if defined(_WIN32)
         if (code > 0xFFFF) {
             return code;
         }
 #endif
-        return std::tolower(wchar_t(code), std::locale("en_US.UTF-8"));
+        return std::tolower(wchar_t(code), locale);
     }
 
     bool is_ascii_punct(uint32_t code) {