fix disable and enable llm setting in dialog

api/apps/dialog_app.py

@@ -35,13 +35,7 @@ def set_dialog():
     top_n = req.get("top_n", 6)
     similarity_threshold = req.get("similarity_threshold", 0.1)
     vector_similarity_weight = req.get("vector_similarity_weight", 0.3)
-    llm_setting = req.get("llm_setting", {
-        "temperature": 0.1,
-        "top_p": 0.3,
-        "frequency_penalty": 0.7,
-        "presence_penalty": 0.4,
-        "max_tokens": 215
-    })
+    llm_setting = req.get("llm_setting", {})
     default_prompt = {
         "system": """你是一个智能助手，请总结知识库的内容来回答问题，请列举知识库中的数据详细回答。当所有知识库内容都与问题无关时，你的回答必须包括“知识库中未找到您要的答案！”这句话。回答需要考虑聊天历史。
 以下是知识库：

api/utils/file_utils.py

@@ -67,7 +67,7 @@ def get_rag_python_directory(*args):
 
 
 def get_home_cache_dir():
-    dir = os.path.join(os.path.expanduser('~'), ".raglow")
+    dir = os.path.join(os.path.expanduser('~'), ".ragflow")
     try:
         os.mkdir(dir)
     except OSError as error:

rag/app/qa.py

@@ -116,18 +116,31 @@ def chunk(filename, binary=None, lang="Chinese", callback=None, **kwargs):
                         break
                     txt += l
         lines = txt.split("\n")
-        #is_english([rmPrefix(l) for l in lines[:100]])
+        comma, tab = 0, 0
+        for l in lines:
+            if len(l.split(",")) == 2: comma += 1
+            if len(l.split("\t")) == 2: tab += 1
+        delimiter = "\t" if tab >= comma else ","
+
         fails = []
-        for i, line in enumerate(lines):
-            arr = [l for l in line.split("\t") if len(l) > 1]
+        question, answer = "", ""
+        i = 0
+        while i < len(lines):
+            arr = lines[i].split(delimiter)
             if len(arr) != 2:
-                fails.append(str(i))
-                continue
-            res.append(beAdoc(deepcopy(doc), arr[0], arr[1], eng))
+                if question: answer += "\n" + lines[i]
+                else:
+                    fails.append(str(i+1))
+            elif len(arr) == 2:
+                if question and answer: res.append(beAdoc(deepcopy(doc), question, answer, eng))
+                question, answer = arr
+            i += 1
             if len(res) % 999 == 0:
                 callback(len(res) * 0.6 / len(lines), ("Extract Q&A: {}".format(len(res)) + (
                     f"{len(fails)} failure, line: %s..." % (",".join(fails[:3])) if fails else "")))
 
+        if question: res.append(beAdoc(deepcopy(doc), question, answer, eng))
+
         callback(0.6, ("Extract Q&A: {}".format(len(res)) + (
             f"{len(fails)} failure, line: %s..." % (",".join(fails[:3])) if fails else "")))
 

rag/llm/chat_model.py

@@ -141,12 +141,12 @@ def chat(self, system, history, gen_conf):
         if system:
             history.insert(0, {"role": "system", "content": system})
         try:
-            options = {"temperature": gen_conf.get("temperature", 0.1),
-                       "num_predict": gen_conf.get("max_tokens", 128),
-                       "top_k": gen_conf.get("top_p", 0.3),
-                       "presence_penalty": gen_conf.get("presence_penalty", 0.4),
-                       "frequency_penalty": gen_conf.get("frequency_penalty", 0.7),
-                       }
+            options = {}
+            if "temperature" in gen_conf: options["temperature"] = gen_conf["temperature"]
+            if "max_tokens" in gen_conf: options["num_predict"] = gen_conf["max_tokens"]
+            if "top_p" in gen_conf: options["top_k"] = gen_conf["top_p"]
+            if "presence_penalty" in gen_conf: options["presence_penalty"] = gen_conf["presence_penalty"]
+            if "frequency_penalty" in gen_conf: options["frequency_penalty"] = gen_conf["frequency_penalty"]
             response = self.client.chat(
                 model=self.model_name,
                 messages=history,

rag/llm/embedding_model.py

@@ -236,8 +236,8 @@ def __init__(self, key=None, model_name="maidalun1020/bce-embedding-base_v1", **
             try:
                 print("LOADING BCE...")
                 YoudaoEmbed._client = qanthing(model_name_or_path=os.path.join(
-                    get_project_base_directory(),
-                    "rag/res/bce-embedding-base_v1"))
+                    get_home_cache_dir(),
+                    "bce-embedding-base_v1"))
             except Exception as e:
                 YoudaoEmbed._client = qanthing(
                     model_name_or_path=model_name.replace(