diff --git a/examples/text-generation/run_lm_eval.py b/examples/text-generation/run_lm_eval.py index 3299cadcbe..c827291416 100644 --- a/examples/text-generation/run_lm_eval.py +++ b/examples/text-generation/run_lm_eval.py @@ -195,6 +195,14 @@ def main(): args = setup_lm_eval_parser() model, _, tokenizer, generation_config = initialize_model(args, logger) + if args.trust_remote_code: + # trust_remote_code fix was introduced in lm_eval 0.4.3 + # https://github.com/EleutherAI/lm-evaluation-harness/pull/1998/files + # We need to cherry-pick the fix manually untill we upgrade (SW-190418) + import datasets + + datasets.config.HF_DATASETS_TRUST_REMOTE_CODE = True + lm_tasks = lm_eval.tasks.get_task_dict(args.tasks) with torch.no_grad(): lm = HabanaModelAdapter(tokenizer, model, args, generation_config)