From 421fbc97e3f1407b368f08615804537257fa7324 Mon Sep 17 00:00:00 2001 From: Adam Stachowicz Date: Thu, 31 Oct 2024 17:01:01 +0200 Subject: [PATCH] [SW-206500] Fix trust_remote_code Change-Id: Id6deccec93fce6ffb6e1c588f126fdf4fea3dbf8 --- examples/text-generation/run_lm_eval.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/examples/text-generation/run_lm_eval.py b/examples/text-generation/run_lm_eval.py index 3299cadcbe..c827291416 100644 --- a/examples/text-generation/run_lm_eval.py +++ b/examples/text-generation/run_lm_eval.py @@ -195,6 +195,14 @@ def main(): args = setup_lm_eval_parser() model, _, tokenizer, generation_config = initialize_model(args, logger) + if args.trust_remote_code: + # trust_remote_code fix was introduced in lm_eval 0.4.3 + # https://github.com/EleutherAI/lm-evaluation-harness/pull/1998/files + # We need to cherry-pick the fix manually untill we upgrade (SW-190418) + import datasets + + datasets.config.HF_DATASETS_TRUST_REMOTE_CODE = True + lm_tasks = lm_eval.tasks.get_task_dict(args.tasks) with torch.no_grad(): lm = HabanaModelAdapter(tokenizer, model, args, generation_config)