Skip to content

Commit 38c03ad

Browse files
authored
Fix IFEval metric (#259)
* Fix IFEval metric * Remove dead code
1 parent 6041e36 commit 38c03ad

File tree

1 file changed

+16
-17
lines changed
  • src/lighteval/tasks/extended/ifeval

1 file changed

+16
-17
lines changed

src/lighteval/tasks/extended/ifeval/main.py

Lines changed: 16 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -46,23 +46,6 @@ def ifeval_prompt(line, task_name: str = None):
4646
)
4747

4848

49-
# We create the task config
50-
ifeval = LightevalTaskConfig(
51-
name="ifeval",
52-
prompt_function=ifeval_prompt,
53-
suite=["extended"],
54-
hf_repo="wis-k/instruction-following-eval",
55-
hf_subset="default",
56-
metric=["ifeval_metric"],
57-
hf_avail_splits=["train"],
58-
evaluation_splits=["train"],
59-
few_shots_split="train",
60-
few_shots_select="random_sampling",
61-
generation_size=1280,
62-
stop_sequence=[], # no stop sequence, will use eot token
63-
)
64-
65-
6649
submetric_names = [
6750
"prompt_level_strict_acc",
6851
"inst_level_strict_acc",
@@ -156,6 +139,22 @@ def agg_inst_level_acc(items):
156139
},
157140
)
158141

142+
# We create the task config
143+
ifeval = LightevalTaskConfig(
144+
name="ifeval",
145+
prompt_function=ifeval_prompt,
146+
suite=["extended"],
147+
hf_repo="wis-k/instruction-following-eval",
148+
hf_subset="default",
149+
metric=[ifeval_metrics],
150+
hf_avail_splits=["train"],
151+
evaluation_splits=["train"],
152+
few_shots_split="train",
153+
few_shots_select="random_sampling",
154+
generation_size=1280,
155+
stop_sequence=[], # no stop sequence, will use eot token
156+
)
157+
159158

160159
TASKS_TABLE = [ifeval]
161160

0 commit comments

Comments
 (0)