Add required_temp support in jsonl format to support flexible tempera…

…ture setting for gen_api_answer (#2653)
lm-sys · Nov 8, 2023 · f2810e5 · f2810e5
1 parent 32c41de
commit f2810e5
Showing 1 changed file with 5 additions and 0 deletions.
diff --git a/fastchat/llm_judge/gen_api_answer.py b/fastchat/llm_judge/gen_api_answer.py
@@ -27,8 +27,13 @@
 def get_answer(
     question: dict, model: str, num_choices: int, max_tokens: int, answer_file: str
 ):
+    assert (
+        args.force_temperature is not None and "required_temperature" in question.keys()
+    ) == False
     if args.force_temperature is not None:
         temperature = args.force_temperature
+    elif "required_temperature" in question.keys():
+        temperature = question["required_temperature"]
     elif question["category"] in temperature_config:
         temperature = temperature_config[question["category"]]
     else: