From e61e16cb156b957b85295c99775f731eb7c0cf39 Mon Sep 17 00:00:00 2001 From: Cheng-Ping Hsieh Date: Wed, 17 Sep 2025 16:21:36 -0700 Subject: [PATCH 01/88] add initial ruler2 Signed-off-by: Cheng-Ping Hsieh --- nemo_skills/dataset/ruler2/__init__.py | 15 + nemo_skills/dataset/ruler2/prepare.py | 379 +++++++++++++++++ nemo_skills/dataset/ruler2/prepare_mmlu.py | 416 +++++++++++++++++++ nemo_skills/dataset/ruler2/prepare_niah.py | 271 ++++++++++++ nemo_skills/dataset/ruler2/prepare_qa.py | 340 +++++++++++++++ nemo_skills/dataset/ruler2/ruler2_score.py | 42 ++ nemo_skills/dataset/ruler2/tokenizer.py | 127 ++++++ nemo_skills/evaluation/evaluator/__init__.py | 3 +- nemo_skills/evaluation/evaluator/ruler.py | 85 ++++ 9 files changed, 1677 insertions(+), 1 deletion(-) create mode 100644 nemo_skills/dataset/ruler2/__init__.py create mode 100644 nemo_skills/dataset/ruler2/prepare.py create mode 100644 nemo_skills/dataset/ruler2/prepare_mmlu.py create mode 100644 nemo_skills/dataset/ruler2/prepare_niah.py create mode 100644 nemo_skills/dataset/ruler2/prepare_qa.py create mode 100644 nemo_skills/dataset/ruler2/ruler2_score.py create mode 100644 nemo_skills/dataset/ruler2/tokenizer.py diff --git a/nemo_skills/dataset/ruler2/__init__.py b/nemo_skills/dataset/ruler2/__init__.py new file mode 100644 index 0000000000..8af131119c --- /dev/null +++ b/nemo_skills/dataset/ruler2/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +DATASET_GROUP = "long-context" \ No newline at end of file diff --git a/nemo_skills/dataset/ruler2/prepare.py b/nemo_skills/dataset/ruler2/prepare.py new file mode 100644 index 0000000000..5b8b955b72 --- /dev/null +++ b/nemo_skills/dataset/ruler2/prepare.py @@ -0,0 +1,379 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import argparse +import concurrent.futures +import json +import subprocess +from pathlib import Path + +DEFAULT_SETTINGS = """ +DATASET_GROUP = "long-context" +METRICS_TYPE = "ruler2" +EVAL_ARGS = "{eval_args}" +GENERATION_ARGS = ( + "++prompt_config=generic/default " +) +""" + +prepare_task = { + "mk_niah_basic": prepare_mk_niah_basic, + "mk_niah_easy": prepare_mk_niah_easy, + "mk_niah_medium": prepare_mk_niah_medium, + "mk_niah_hard": prepare_mk_niah_hard, + "mv_niah_basic": prepare_mv_niah_basic, + "mv_niah_easy": prepare_mv_niah_easy, + "mv_niah_medium": prepare_mv_niah_medium, + "mv_niah_hard": prepare_mv_niah_hard, + "qa_basic": prepare_qa_basic, + "qa_easy": prepare_qa_easy, + "qa_medium": prepare_qa_medium, + "qa_hard": prepare_qa_hard, +} + + +def prepare_mk_niah_basic(output_folder, tokenizer_type, tokenizer_path, length, dataset_size): + subprocess.run( + f"python -m nemo_skills.dataset.ruler2.prepare_niah " + f"--output_folder {output_folder} " + f"--tokenizer_type ${tokenizer_type} " + f"--tokenizer_path ${tokenizer_path} " + f"--max_seq_length ${length} " + f"--num_samples ${dataset_size} " + f"--random_seed 42 " + f"--num_needle_k 1 " + f"--num_needle_v 1 " + f"--num_needle_q 1 " + f"--type_haystack needle " + f"--type_needle_k words " + f"--type_needle_v numbers " + f"--num_digits_v 10", + shell=True, + check=True, + ) + +def prepare_mk_niah_easy(output_folder, tokenizer_type, tokenizer_path, length, dataset_size): + subprocess.run( + f"python -m nemo_skills.dataset.ruler2.prepare_mmlu " + f"--output_folder {output_folder} " + f"--tokenizer_type ${tokenizer_type} " + f"--tokenizer_path ${tokenizer_path} " + f"--max_seq_length ${length} " + f"--num_samples ${dataset_size} " + f"--random_seed 42 " + f"--dataset mmlu " + f"--fewshot 0 " + f"--prompt_type instruct " + f"--num_order 0 " + f"--task_type retrieve " + f"--algo_type single", + shell=True, + check=True, + ) + +def prepare_mk_niah_medium(output_folder, tokenizer_type, tokenizer_path, length, dataset_size): + subprocess.run( + f"python -m nemo_skills.dataset.ruler2.prepare_mmlu " + f"--output_folder {output_folder} " + f"--tokenizer_type ${tokenizer_type} " + f"--tokenizer_path ${tokenizer_path} " + f"--max_seq_length ${length} " + f"--num_samples ${dataset_size} " + f"--random_seed 42 " + f"--dataset mmlu " + f"--fewshot 5 " + f"--prompt_type instruct " + f"--num_order 0 " + f"--task_type solve " + f"--algo_type 2steps", + shell=True, + check=True, + ) + +def prepare_mk_niah_hard(output_folder, tokenizer_type, tokenizer_path, length, dataset_size): + subprocess.run( + f"python -m nemo_skills.dataset.ruler2.prepare_mmlu " + f"--output_folder {output_folder} " + f"--tokenizer_type ${tokenizer_type} " + f"--tokenizer_path ${tokenizer_path} " + f"--max_seq_length ${length} " + f"--num_samples ${dataset_size} " + f"--random_seed 42 " + f"--dataset mmlu " + f"--fewshot 5 " + f"--prompt_type instruct " + f"--num_order 0 " + f"--task_type solve " + f"--algo_type single", + shell=True, + check=True, + ) + +def prepare_mv_niah_basic(output_folder, tokenizer_type, tokenizer_path, length, dataset_size): + subprocess.run( + f"python -m prepare.py nemo_skills.dataset.ruler2.prepare_niah " + f"--output_folder {output_folder} " + f"--tokenizer_type ${tokenizer_type} " + f"--tokenizer_path ${tokenizer_path} " + f"--max_seq_length ${length} " + f"--num_samples ${dataset_size} " + f"--random_seed 42 " + f"--num_needle_k 1 " + f"--num_needle_v 4 " + f"--num_needle_q 1 " + f"--type_haystack needle " + f"--type_needle_k words " + f"--type_needle_v numbers " + f"--num_digits_v 10", + shell=True, + check=True, + ) + +def prepare_mv_niah_easy(output_folder, tokenizer_type, tokenizer_path, length, dataset_size): + subprocess.run( + f"python -m nemo_skills.dataset.ruler2.prepare_mmlu " + f"--output_folder {output_folder} " + f"--tokenizer_type ${tokenizer_type} " + f"--tokenizer_path ${tokenizer_path} " + f"--max_seq_length ${length} " + f"--num_samples ${dataset_size} " + f"--random_seed 42 " + f"--dataset mmlu " + f"--fewshot 0 " + f"--prompt_type instruct " + f"--num_order 4 " + f"--task_type niah " + f"--algo_type single", + shell=True, + check=True, + ) + +def prepare_mv_niah_medium(output_folder, tokenizer_type, tokenizer_path, length, dataset_size): + subprocess.run( + f"python -m nemo_skills.dataset.ruler2.prepare_mmlu " + f"--output_folder {output_folder} " + f"--tokenizer_type ${tokenizer_type} " + f"--tokenizer_path ${tokenizer_path} " + f"--max_seq_length ${length} " + f"--num_samples ${dataset_size} " + f"--random_seed 42 " + f"--dataset mmlu " + f"--fewshot 0 " + f"--prompt_type instruct " + f"--num_order 4 " + f"--task_type retrieve " + f"--algo_type 2steps", + shell=True, + check=True, + ) + +def prepare_mv_niah_hard(output_folder, tokenizer_type, tokenizer_path, length, dataset_size): + subprocess.run( + f"python -m nemo_skills.dataset.ruler2.prepare_mmlu " + f"--output_folder {output_folder} " + f"--tokenizer_type ${tokenizer_type} " + f"--tokenizer_path ${tokenizer_path} " + f"--max_seq_length ${length} " + f"--num_samples ${dataset_size} " + f"--random_seed 42 " + f"--dataset mmlu " + f"--fewshot 0 " + f"--prompt_type instruct " + f"--num_order 4 " + f"--task_type retrieve " + f"--algo_type single", + shell=True, + check=True, + ) + + +def prepare_qa_basic(output_folder, tokenizer_type, tokenizer_path, length, dataset_size): + subprocess.run( + f"python -m nemo_skills.dataset.ruler2.prepare_qa " + f"--output_folder {output_folder} " + f"--tokenizer_type ${tokenizer_type} " + f"--tokenizer_path ${tokenizer_path} " + f"--max_seq_length ${length} " + f"--num_samples ${dataset_size} " + f"--random_seed 42 " + f"--dataset hotpotqa " + f"--fewshot 0 " + f"--prompt_type instruct " + f"--task_type retrieve " + f"--query_type doc", + shell=True, + check=True, + ) + +def prepare_qa_easy(output_folder, tokenizer_type, tokenizer_path, length, dataset_size): + subprocess.run( + f"python -m nemo_skills.dataset.ruler2.prepare_qa " + f"--output_folder {output_folder} " + f"--tokenizer_type ${tokenizer_type} " + f"--tokenizer_path ${tokenizer_path} " + f"--max_seq_length ${length} " + f"--num_samples ${dataset_size} " + f"--random_seed 42 " + f"--dataset hotpotqa " + f"--fewshot 0 " + f"--prompt_type instruct " + f"--task_type retrieve " + f"--query_type question", + shell=True, + check=True, + ) + + +def prepare_qa_medium(output_folder, tokenizer_type, tokenizer_path, length, dataset_size): + subprocess.run( + f"python -m nemo_skills.dataset.ruler2.prepare_qa " + f"--output_folder {output_folder} " + f"--tokenizer_type ${tokenizer_type} " + f"--tokenizer_path ${tokenizer_path} " + f"--max_seq_length ${length} " + f"--num_samples ${dataset_size} " + f"--random_seed 42 " + f"--dataset hotpotqa " + f"--fewshot 0 " + f"--prompt_type instruct " + f"--task_type solve " + f"--algo_type 2steps", + shell=True, + check=True, + ) + +def prepare_qa_hard(output_folder, tokenizer_type, tokenizer_path, length, dataset_size): + subprocess.run( + f"python -m nemo_skills.dataset.ruler2.prepare_qa " + f"--output_folder {output_folder} " + f"--tokenizer_type ${tokenizer_type} " + f"--tokenizer_path ${tokenizer_path} " + f"--max_seq_length ${length} " + f"--num_samples ${dataset_size} " + f"--random_seed 42 " + f"--dataset hotpotqa " + f"--fewshot 0 " + f"--prompt_type instruct " + f"--task_type solve " + f"--algo_type single", + shell=True, + check=True, + ) + +def prepare_task_for_ns(output_folder): + """Adding proper __init__.py""" + Path(output_folder).mkdir(parents=True, exist_ok=True) + with open(output_folder / "__init__.py", "w", encoding="utf-8") as init_file: + if task in ["mk_niah_medium", "mk_niah_hard"]: + eval_args = "++eval_type=multichoice" + elif task in ["mv_niah_medium"]: + eval_args = "++eval_type=ruler2 ++eval_config.match_type=2steps" + elif "qa" in task: + eval_args = "++eval_type=ruler2 ++eval_config.match_type=part" + else: + eval_args = "++eval_type=ruler2 ++eval_config.match_type=all" + + init_file.write(DEFAULT_SETTINGS.format(eval_args=eval_args)) + +def prepare_dataset(tasks, setup, max_seq_length, tokenizer_type, tokenizer_path, dataset_size): + output_folder = Path(__file__).parent / setup + + # 1. installing necessary packages + subprocess.run(["pip install wonderwords html2text tenacity"], check=True, shell=True) + + for task in tasks: + prepare_task_for_ns(output_folder / task) + + # preparing the datasets based on user options, in parallel + with concurrent.futures.ThreadPoolExecutor() as executor: + futures = [executor.submit(prepare_task[task], + str(output_folder / task), + tokenizer_type, + tokenizer_path, + max_seq_length, + dataset_size + ) for task in tasks] + for future in concurrent.futures.as_completed(futures): + future.result() # Will raise exception if any subprocess fails + + with open(output_folder / "__init__.py", "w", encoding="utf-8") as init_file: + init_file.write("IS_BENCHMARK_GROUP = True\n") + init_file.write("SCORE_MODULE = 'nemo_skills.dataset.ruler2.ruler2_score'\n") + benchmarks = ", ".join(f"'ruler2.{setup}.{task}': {{}}" for task in tasks) + init_file.write(f"BENCHMARKS = {{{benchmarks}}}\n") + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Prepare RULER2 dataset.") + parser.add_argument( + "--tasks", + type=str, + nargs="+", + default=[ + "mk_niah_basic", + "mk_niah_easy", + "mk_niah_medium", + "mk_niah_hard", + "mv_niah_basic", + "mv_niah_easy", + "mv_niah_medium", + "mv_niah_hard", + "qa_basic", + "qa_easy", + "qa_medium", + "qa_hard", + ], + help="List of tasks to prepare for RULER2 dataset.", + ) + parser.add_argument( + "--setup", + type=str, + required=True, + help="Name of the setup for RULER2 dataset. Typically should be _.", + ) + parser.add_argument( + "--max_seq_length", + type=int, + required=True, + help="Sequence length to check with RULER2.", + ) + parser.add_argument( + "--tokenizer_type", + type=str, + default="hf" + help="Type of the tokenizer to use.", + ) + parser.add_argument( + "--tokenizer_path", + type=str, + required=True, + help="Path to the tokenizer to use.", + ) + parser.add_argument( + "--dataset_size", + type=int, + default=100, + help="Number of samples to prepare for RULER2 dataset.", + ) + + args, unknown = parser.parse_known_args() + prepare_dataset( + args.tasks, + args.setup, + args.max_seq_length, + args.tokenizer_type, + args.tokenizer_path, + args.dataset_size, + ) + print("RULER2 dataset preparation completed.") \ No newline at end of file diff --git a/nemo_skills/dataset/ruler2/prepare_mmlu.py b/nemo_skills/dataset/ruler2/prepare_mmlu.py new file mode 100644 index 0000000000..fbd92bbe64 --- /dev/null +++ b/nemo_skills/dataset/ruler2/prepare_mmlu.py @@ -0,0 +1,416 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +import re +import os +import subprocess +import json +import argparse +import random +import numpy as np +from pathlib import Path +from tqdm import tqdm +from datasets import load_dataset +from .tokenizer import select_tokenizer +import logging + +from collections import defaultdict +import math +import inflect +convert = inflect.engine() + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +parser = argparse.ArgumentParser() +# Basic Configurations +parser.add_argument("--output_folder", type=str) +parser.add_argument("--tokenizer_path", type=str, required=True, help='path to the tokenizer model') +parser.add_argument("--tokenizer_type", type=str, default='nemo', help='[Options] nemo, hf, openai.') +parser.add_argument("--max_seq_length", type=int, required=True, help='max sequence length including all input tokens and generated tokens.') +parser.add_argument("--random_seed", type=int, default=42) +parser.add_argument("--insert_position", type=float, default=-1, help='insert position of the true context in the context.') +parser.add_argument("--num_samples", type=int, default=None, help='number of samples to generate') +parser.add_argument("--dataset", type=str, default="gsm8k") +parser.add_argument("--fewshot", type=int, default=0) +parser.add_argument("--prompt_type", type=str, default="chat") +parser.add_argument("--num_order", type=int, default=0) +parser.add_argument("--algo_type", type=str, default="single", choices=["single", "attention","2steps","3steps", "size_2steps", "size_single"]) +parser.add_argument("--task_type", type=str, default="retrieve", choices=["retrieve", "solve", "niah"]) + +args = parser.parse_args() +random.seed(args.random_seed) +np.random.seed(args.random_seed) + +# Load Tokenizer +TOKENIZER = select_tokenizer(args.tokenizer_type, args.tokenizer_path) + +TOTAL_PROMPT = """{context}\n\n{example}{problem}""" +if args.task_type == "retrieve": + CONTEXT_PROMPT = "Below are some questions. I will ask you to copy one of them. Please copy and paste the question you find.\n\n{needles}" + NEEDLE_PROMPT = "Question {i}: {question}" + if args.algo_type == "single": + PROBLEM_PROMPT = "Please copy the {order}Question {i} from the context." + elif args.algo_type == "attention": + PROBLEM_PROMPT = "Please first pay attention to all the Question {i} from the context and then only copy the {order}Question {i} in your response. Do not output any other questions." + elif args.algo_type == "2steps": + # PROBLEM_PROMPT = "Please first find all the Question {i} from the context and then copy the {order}Question {i} at the end." + PROBLEM_PROMPT = "Please first copy all the Question {i} from the context and then copy the {order}Question {i} at the end." + # PROBLEM_PROMPT = "Please first copy all instances of Question {i} from the context in the order in which they appear, and then copy the {order}Question {i} (1-indexed) at the end." + elif args.algo_type == "3steps": + PROBLEM_PROMPT = "Please first find how many Question {i} from the context, list them in order, and then copy the {order}Question {i} at the end." + elif args.algo_type == "size_2steps": + PROBLEM_PROMPT = "Please first find all the" + str(args.num_order) + " Question {i} from the context and then copy the {order}Question {i} at the end." + elif args.algo_type == "size_single": + PROBLEM_PROMPT = "There are " + str(args.num_order) + " Question {i} in the context. Please copy the {order}Question {i} from the context." + + if args.fewshot > 0: + EXAMPLE_PROMPT = PROBLEM_PROMPT + "\nQuestion {i}: {question}" + + if args.prompt_type == "base": + PROBLEM_PROMPT += "\nQuestion {i}:" + +elif args.task_type == "niah": + CONTEXT_PROMPT = "Below are some questions. I will ask you to copy some of them. Please copy and paste the questions you find.\n\n{needles}" + NEEDLE_PROMPT = "Question {i}: {question}" + PROBLEM_PROMPT = "Please find and copy all the Question {i} from the context." + if args.prompt_type == "base": + PROBLEM_PROMPT += "\nQuestion {i}:" + + +elif args.task_type == "solve": + CONTEXT_PROMPT = "Below are some questions. I will ask you to solve one of them. Please solve the question you find and make sure to put the answer (and only answer) inside \\boxed\\{{\\}}.\n\n{needles}" + NEEDLE_PROMPT = "Question {i}: {question}" + if args.dataset == "gsm8k" or args.dataset == "math500": + if args.algo_type == "single": + PROBLEM_PROMPT = "Please solve the Question {i} from the context step by step." + elif args.algo_type == "2steps": + PROBLEM_PROMPT = "Please copy the Question {i} from the context and then solve it step by step." + elif args.dataset == "mmlu": + if args.algo_type == "single": + PROBLEM_PROMPT = "Please solve the Question {i} from the context with an answer from A, B, C, D." + elif args.algo_type == "2steps": + PROBLEM_PROMPT = "Please copy the Question {i} from the context and then solve it with an answer from A, B, C, D." + elif args.dataset == "mbpp": + if args.algo_type == "single": + PROBLEM_PROMPT = "Please solve the Question {i} from the context by generating or completing code.\nYour answer should be in the following format:\n```python\n# Your code here\n```" + elif args.algo_type == "2steps": + PROBLEM_PROMPT = "Please copy the Question {i} from the context and then solve it by generating or completing code.\nYour answer should be in the following format:\n```python\n# Your code here\n```" + + if args.fewshot > 0: + if args.algo_type == "single": + EXAMPLE_PROMPT = PROBLEM_PROMPT + "\nSolution:{solution}" + if args.prompt_type == "base": + PROBLEM_PROMPT += "\nSolution:" + elif args.algo_type == "2steps": + EXAMPLE_PROMPT = PROBLEM_PROMPT + "\nQuestion {i}: {question}\nSolution:{solution}" + if args.prompt_type == "base": + PROBLEM_PROMPT += "\nQuestion {i}:" + +examples = [] +haystack, needle = [], [] +if args.dataset == "gsm8k": + test_dataset = load_dataset("openai/gsm8k", "main") + for d in test_dataset['train']: + solution, answer = d['answer'].split("#### ") + haystack.append({ + "Question": d['question'], + "Solution": " " + solution + f"So the answer is \\boxed{{{answer}}}.", + "Answer": answer, + }) + for d in test_dataset['test']: + solution, answer = d['answer'].split("#### ") + needle.append({ + "Question": d['question'], + "Solution": " " + solution + f"So the answer is \\boxed{{{answer}}}.", + "Answer": answer, + }) +elif args.dataset == "math500": + questions = set() + test_dataset = load_dataset("HuggingFaceH4/MATH-500") + for d in test_dataset['test']: + needle.append({ + "Question": d['problem'], + "Solution": " " + d['solution'], + "Answer": d['answer'], + }) + questions.add(d['problem']) + + from math_verify import parse + for subject in ['algebra', 'counting_and_probability', 'geometry', 'intermediate_algebra', 'number_theory', 'prealgebra', 'precalculus']: + train_dataset = load_dataset("EleutherAI/hendrycks_math", subject) + for index, d in enumerate(train_dataset['test']): + if d['problem'] not in questions: + haystack.append({ + "Question": d['problem'], + "Solution": " " + d['solution'], + "Answer": parse(d["solution"])[-1], + }) + + +elif args.dataset == "mmlu": + test_dataset = load_dataset("cais/mmlu", "all") + options = ['A', 'B', 'C', 'D'] + haystack = [] + needle = [] + for d in test_dataset['test']: + choices = d["choices"] + item = { + "Question": d['question'] + f'\nA. {choices[0]}\nB. {choices[1]}\nC. {choices[2]}\nD. {choices[3]}', + "Solution": " " + f'\\boxed{{{options[d["answer"]]}}}' + "Answer": options[d['answer']], + } + needle.append(item) + + for d in test_dataset['auxiliary_train']: + choices = d["choices"] + item = { + "Question": d['question'] + f'\nA. {choices[0]}\nB. {choices[1]}\nC. {choices[2]}\nD. {choices[3]}', + "Solution": " " + f'\\boxed{{{options[d["answer"]]}}}' + "Answer": options[d['answer']], + } + haystack.append(item) + + +elif args.dataset == "mbpp": + test_dataset = load_dataset("evalplus/mbppplus") + for d in test_dataset['test']: + prompt = d['prompt'].replace(' ', '\t').strip() + assertion = d['test_list'][0] + needle.append({ + "task_id": f'Mbpp/{d["task_id"]}', + "Question": f"{prompt}\n{assertion}", + "Solution": f"\n```python\n{d['code'].strip()}\n```", + "canonical_solution": f"\n{d['code'].strip()}\n", + "assertion": "\n".join(d['test_list']), + }) + + train_dataset = load_dataset("google-research-datasets/mbpp", "full") + for d in train_dataset['train']: + prompt = d['text'].replace(' ', '\t').strip() + assertion = d['test_list'][0] + haystack.append({ + "Question": f"{prompt}\n{assertion}", + "Solution": f"\n```python\n{d['code'].strip()}\n```", + "canonical_solution": f"\n{d['code'].strip()}\n", + "assertion": "\n".join(d['test_list']), + }) + for d in train_dataset['validation']: + prompt = d['text'].replace(' ', '\t').strip() + assertion = d['test_list'][0] + haystack.append({ + "Question": f"{prompt}\n{assertion}", + "Solution": f"\n```python\n{d['code'].strip()}\n```", + "canonical_solution": f"\n{d['code'].strip()}\n", + "assertion": "\n".join(d['test_list']), + }) + for d in train_dataset['test']: + prompt = d['text'].replace(' ', '\t').strip() + assertion = d['test_list'][0] + haystack.append({ + "Question": f"{prompt}\n{assertion}", + "Solution": f"\n```python\n{d['code'].strip()}\n```", + "canonical_solution": f"\n{d['code'].strip()}\n", + "assertion": "\n".join(d['test_list']), + }) +else: + raise ValueError(f"Dataset {args.dataset} is not supported.") + +for item in needle: + item["Question"] = re.sub(r'\s+', ' ', item["Question"]) +for item in haystack: + item["Question"] = re.sub(r'\s+', ' ', item["Question"]) + +logger.info(f'Dataset size: {len(needle)}') + +def generate_random_number(num_digits=7): + lower_bound = 10**(num_digits - 1) + upper_bound = 10**num_digits - 1 + return str(random.randint(lower_bound, upper_bound)) + +def generate_input_output(index, num_qs): + if num_qs > len(haystack): + repeats = (num_qs + len(haystack) - 1) // len(haystack) # Ceiling division + else: + repeats = 1 + + curr_context = random.sample([item for item in haystack for _ in range(repeats)], num_qs) + + if args.num_order > 0: + random_numbers = [generate_random_number() for _ in range(math.ceil((num_qs + 1) / args.num_order))] + random_numbers = random_numbers * args.num_order + else: + random_numbers = [generate_random_number() for _ in range(num_qs + 1)] + + random.shuffle(random_numbers) + random_numbers = random_numbers[:num_qs+1] + for i,q in enumerate(curr_context): + q["random_index"] = random_numbers[i] + + random.shuffle(curr_context) + examples = random.sample(curr_context, args.fewshot) + + true_context = needle[index] + true_context["random_index"] = random_numbers[-1] + if args.insert_position < 0: + insert_position = random.randint(0, len(curr_context)) + else: + insert_position = int(args.insert_position * len(curr_context)) + curr_context.insert(insert_position,true_context) + + counts = defaultdict(int) + for i,q in enumerate(curr_context): + counts[q["random_index"]] += 1 + if args.num_order > 0: + q["order"] = convert.ordinal(counts[q["random_index"]]) + " (1 indexed) " + else: + q["order"] = "" + + needles = '\n\n'.join([NEEDLE_PROMPT.format(i=q["random_index"], question=q["Question"]) for i, q in enumerate(curr_context)]) + if args.task_type == "niah": + problem = PROBLEM_PROMPT.format(i=true_context["random_index"]) + else: + problem = PROBLEM_PROMPT.format(i=true_context["random_index"], order=true_context["order"]) + + if args.fewshot > 0: + if args.task_type == "retrieve": + example = '\n\n'.join([EXAMPLE_PROMPT.format(i=q["random_index"], question=q["Question"], order=q["order"]) for q in examples]) + elif args.task_type == "solve": + if args.algo_type == "single": + example = '\n\n'.join([EXAMPLE_PROMPT.format(i=q["random_index"], solution=q["Solution"]) for q in examples]) + elif args.algo_type == "2steps": + example = '\n\n'.join([EXAMPLE_PROMPT.format(i=q["random_index"], question=q["Question"], solution=q["Solution"]) for q in examples]) + + if args.prompt_type == "base": + example = f"{example}\n\n" + else: + example = f"Here are some examples to help you understand the task:\n\n{example}\n\nHere is the actual task you need to solve:\n\n" + else: + example = "" + + + context = CONTEXT_PROMPT.format(needles=needles) + input_text = TOTAL_PROMPT.format( + context=context, + problem=problem, + example=example, + ) + + if args.task_type == "retrieve": + expected_answer = { + "expected_answer" : [true_context["Question"]] + } + elif args.task_type == "niah": + expected_answer = { + "expected_answer" : [c["Question"] for c in curr_context if c["random_index"] == true_context["random_index"]] + } + elif args.task_type == "solve": + if args.dataset == "mbpp": + expected_answer = { + "task_id": true_context["task_id"], + "assertion": true_context["assertion"], + "canonical_solution": true_context["canonical_solution"], + } + else: + expected_answer = { + "expected_answer" : true_context["Answer"] + } + + save_dict = { + "index": index, + "question": f"{context}\n\n{example}{problem}", + **expected_answer, + } + return input_text, save_dict + + +def generate_samples(max_seq_length: int, incremental: int = 10): + + write_jsons = [] + + # Estimate tokens per question to determine reasonable upper bound + sample_input_text, _ = generate_input_output(0, incremental) + sample_tokens = len(TOKENIZER.text_to_tokens(sample_input_text)) + tokens_per_question = sample_tokens / incremental + + # Let's do 3x to allow for some slack since we can get unlucky due to sampling. + # NOTE: We should test this for really large sequence lengths to make sure it's reasonable. + estimated_max_questions = int((max_seq_length / tokens_per_question) * 3) + + # Binary search for optimal haystack size + lower_bound = incremental + upper_bound = max(estimated_max_questions, incremental * 2) # Ensure upper_bound is reasonable + + optimal_num_qs = None + + logger.info(f"Estimated {tokens_per_question:.1f} tokens per question") + logger.info(f"Starting binary search with bounds: {lower_bound} to {upper_bound}") + + while lower_bound <= upper_bound: + mid = (lower_bound + upper_bound) // 2 + input_text, save_dict = generate_input_output(0, mid) + total_tokens = len(TOKENIZER.text_to_tokens(input_text)) + + logger.info(f"Testing haystack size: {mid}, resulting tokens: {total_tokens}/{max_seq_length}") + + if total_tokens <= max_seq_length: + # This size works, can we go larger? + optimal_num_qs = mid + lower_bound = mid + 1 + else: + # Too large, need to go smaller + upper_bound = mid - 1 + + num_qs = optimal_num_qs if optimal_num_qs is not None else incremental + logger.info(f'Final optimal haystack size (number of questions): {num_qs}') + + if args.num_samples is not None: + needle_sample = random.sample(list(range(len(needle))), min(len(needle), args.num_samples)) + else: + needle_sample = list(range(len(needle))) + + # Generate samples + for index in tqdm(needle_sample): + used_qs = num_qs + while(True): + try: + input_text, save_dict = generate_input_output(index, used_qs) + length = len(TOKENIZER.text_to_tokens(input_text)) + assert length <= max_seq_length, f"{length} exceeds max_seq_length." + break + except: + if used_qs > incremental: + used_qs -= incremental + + save_dict["length"] = length + formatted_output = save_dict + write_jsons.append(formatted_output) + + return write_jsons + + +def main(): + output_file = str(args.output_folder / "test.jsonl") + + write_jsons = generate_samples( + max_seq_length=args.max_seq_length, + incremental=max(10, args.fewshot) + ) + + with open(output_file, "wt", encoding="utf-8") as fout: + for entry in write_jsons: + fout.write(json.dumps(entry) + "\n") + +if __name__=="__main__": + main() diff --git a/nemo_skills/dataset/ruler2/prepare_niah.py b/nemo_skills/dataset/ruler2/prepare_niah.py new file mode 100644 index 0000000000..0dd9de76df --- /dev/null +++ b/nemo_skills/dataset/ruler2/prepare_niah.py @@ -0,0 +1,271 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +import os +import re +import json +import uuid +import argparse +import random +import nltk +import math +import numpy as np +import wonderwords +from pathlib import Path +from tqdm import tqdm +from .tokenizer import select_tokenizer +from nltk.tokenize import sent_tokenize +try: + nltk.data.find('tokenizers/punkt') + nltk.data.find('tokenizers/punkt_tab') +except LookupError: + nltk.download('punkt') + nltk.download('punkt_tab') +import logging + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +parser = argparse.ArgumentParser() +parser.add_argument("--output_folder", type=str) +parser.add_argument("--tokenizer_type", type=str, default='nemo', help='[Options] nemo, hf, openai.') +parser.add_argument("--tokenizer_path", type=str, required=True, help='path to the tokenizer model') +parser.add_argument("--max_seq_length", type=int, required=True, help='max sequence length including all input tokens and generated tokens.') +parser.add_argument("--num_samples", type=int, required=True, help='number of samples to generate') +parser.add_argument("--random_seed", type=int, default=42) + +# Complexity Configurations +parser.add_argument("--num_needle_k", type=int, default=1) +parser.add_argument("--num_needle_v", type=int, default=1) +parser.add_argument("--num_needle_q", type=int, default=1) +parser.add_argument("--type_haystack", type=str, default='essay', help='[Options] noise, essay, needle.') +parser.add_argument("--type_needle_k", type=str, default='words', help='[Options] numbers, words, uuids.') +parser.add_argument("--type_needle_v", type=str, default='numbers', help='[Options] numbers, words, uuids.') +parser.add_argument("--num_digits_k", type=int, default=7) +parser.add_argument("--num_digits_v", type=int, default=7) + +args = parser.parse_args() +random.seed(args.random_seed) +np.random.seed(args.random_seed) +args.num_needle_k = max(args.num_needle_k, args.num_needle_q) + +# Load Tokenizer +TOKENIZER = select_tokenizer(args.tokenizer_type, args.tokenizer_path) + +TEMPLATE_SINGLE = """A special magic {type_needle_v} is hidden within the following text. Make sure to memorize it. I will quiz you about the {type_needle_v} afterwards.\n{context}\nWhat is the special magic {type_needle_v} for {query} mentioned in the provided text? The special magic {type_needle_v} for {query} mentioned in the provided text is""" +TEMPLATE_MULTIPLE = """Some special magic {type_needle_v} are hidden within the following text. Make sure to memorize them. I will quiz you about the {type_needle_v} afterwards.\n{context}\nWhat are all the special magic {type_needle_v} for {query} mentioned in the provided text? The special magic {type_needle_v} for {query} mentioned in the provided text are""" + +# Define Needle/Haystack Format +needle = "One of the special magic {type_needle_v} for {key} is: {value}." +if args.type_haystack == 'needle': + haystack = needle +else: + raise NotImplementedError(f'{args.type_haystack} is not implemented.') + +# Words +nouns = wonderwords.random_word._get_words_from_text_file("nounlist.txt") +adjs = wonderwords.random_word._get_words_from_text_file("adjectivelist.txt") +words = [f"{adj}-{noun}" for adj in adjs for noun in nouns] +words = sorted(list(set(words))) + +# Positions +DEPTHS = list(np.round(np.linspace(0, 100, num=40, endpoint=True)).astype(int)) + +def generate_random_number(num_digits=7): + lower_bound = 10**(num_digits - 1) + upper_bound = 10**num_digits - 1 + return str(random.randint(lower_bound, upper_bound)) + +def generate_random_word(): + word = random.choice(words) + return word + +def generate_random_uuid(): + return str(uuid.UUID(int=random.getrandbits(128), version=4)) + +def generate_random(type_needle: str, digits: int = None): + if type_needle == 'numbers': + return generate_random_number(digits) + elif type_needle == 'words': + return generate_random_word() + elif type_needle == 'uuids': + return generate_random_uuid() + else: + raise NotImplementedError(f'{args.type_needle} is not implemented.') + +def generate_input_output(num_haystack): + keys, values, needles = [], [], [] + for _ in range(args.num_needle_k): + keys.append(generate_random(args.type_needle_k, args.num_digits_k)) + value = [] + for _ in range(args.num_needle_v): + value.append(generate_random(args.type_needle_v, args.num_digits_v)) + needles.append(needle.format( + type_needle_v=args.type_needle_v, + key=keys[-1], + value=value[-1], + )) + values.append(value) + + random.Random(args.random_seed).shuffle(needles) + + # Context + if args.type_haystack == 'essay': + if num_haystack <= len(haystack): + text = " ".join(haystack[:num_haystack]) + else: + repeats = (num_haystack + len(haystack) - 1) // len(haystack) # Ceiling division + text = " ".join((haystack * repeats)[:num_haystack]) + document_sents = sent_tokenize(text.strip()) + insertion_positions = [0] + \ + sorted([int(len(document_sents) * (depth / 100)) for depth in random.sample(DEPTHS, len(needles))]) + \ + [len(document_sents)] + document_sents_list = [] + for i in range(1,len(insertion_positions)): + last_pos = insertion_positions[i-1] + next_pos = insertion_positions[i] + document_sents_list.append(" ".join(document_sents[last_pos:next_pos])) + if i-1 < len(needles): + document_sents_list.append(needles[i-1]) + context = " ".join(document_sents_list) + + else: + if args.type_haystack == 'noise': + sentences = [haystack] * num_haystack + elif args.type_haystack == 'needle': + if args.num_needle_v == 1: + sentences = [haystack.format( + type_needle_v=args.type_needle_v, + key=generate_random(args.type_needle_k, args.num_digits_k), + value=generate_random(args.type_needle_v, args.num_digits_v), + ) for _ in range(num_haystack)] + else: + haystack_values = [generate_random(args.type_needle_v, args.num_digits_v) for _ in range(num_haystack)] + haystack_keys = ([generate_random(args.type_needle_k, args.num_digits_k) for _ in range(math.ceil(num_haystack / args.num_needle_v))] * args.num_needle_v)[:num_haystack] + sentences = [haystack.format( + type_needle_v=args.type_needle_v, + key=haystack_keys[i], + value=haystack_values[i], + ) for i in range(num_haystack)] + random.shuffle(sentences) + + indexes = sorted(random.sample(range(num_haystack), len(needles)), reverse=True) + for index, element in zip(indexes, needles): + sentences.insert(index, element) + context = "\n".join(sentences) + + + ## Query and Answer + indices = random.sample(range(args.num_needle_k), args.num_needle_q) + queries = [keys[i] for i in indices] + answers = [a for i in indices for a in values[i]] + query = ', '.join(queries[:-1]) + ', and ' + queries[-1] if len(queries) > 1 else queries[0] + + if args.num_needle_q * args.num_needle_v == 1: + template = TEMPLATE_SINGLE + type_needle_v = args.type_needle_v[:-1] # remove "s" + else: + template = TEMPLATE_MULTIPLE + type_needle_v = args.type_needle_v + + input_text = template.format( + type_needle_v=type_needle_v, + context=context, + query=query, + ) + + return input_text, answers + + +def generate_samples(num_samples: int, max_seq_length: int, incremental: int = 500): + write_jsons = [] + + if args.type_haystack == 'needle': + incremental = max(5, args.num_needle_v * args.num_needle_k) + + if args.max_seq_length < 4096: + incremental = 5 + + # Estimate tokens per question to determine reasonable upper bound + sample_input_text, _ = generate_input_output(incremental) + sample_tokens = len(TOKENIZER.text_to_tokens(sample_input_text)) + tokens_per_haystack = sample_tokens / incremental + + # Let's do 3x to allow for some slack since we can get unlucky due to sampling. + # NOTE: We should test this for really large sequence lengths to make sure it's reasonable. + estimated_max_questions = int((max_seq_length / tokens_per_haystack) * 3) + + # Binary search for optimal haystack size + lower_bound = incremental + upper_bound = max(estimated_max_questions, incremental * 2) # Ensure upper_bound is reasonable + + optimal_num_haystack = None + + logger.info(f"Estimated {tokens_per_haystack:.1f} tokens per haystack") + logger.info(f"Starting binary search with bounds: {lower_bound} to {upper_bound}") + while lower_bound <= upper_bound: + mid = (lower_bound + upper_bound) // 2 + input_text, save_dict = generate_input_output(mid) + total_tokens = len(TOKENIZER.text_to_tokens(input_text)) + + logger.info(f"Testing haystack size: {mid}, resulting tokens: {total_tokens}/{max_seq_length}") + + if total_tokens <= max_seq_length: + # This size works, can we go larger? + optimal_num_haystack = mid + lower_bound = mid + 1 + else: + # Too large, need to go smaller + upper_bound = mid - 1 + + num_haystack = optimal_num_haystack if optimal_num_haystack is not None else incremental + logger.info(f'Final optimal haystack size (number of haystack): {num_haystack}') + + # Generate samples + for index in tqdm(range(num_samples)): + used_haystack = num_haystack + while(True): + try: + input_text, answer = generate_input_output(used_haystack) + length = len(TOKENIZER.text_to_tokens(input_text)) + assert length <= max_seq_length, f"{length} exceeds max_seq_length." + break + except: + if used_haystack > incremental: + used_haystack -= incremental + + formatted_output = { + 'index': index, + "question": input_text, + "expected_answer": answer, + "length": length, + } + write_jsons.append(formatted_output) + + return write_jsons + + +def main(): + output_file = str(args.output_folder / "test.jsonl") + + write_jsons = generate_samples( + num_samples=args.num_samples, + max_seq_length=args.max_seq_length, + ) + with open(output_file, "wt", encoding="utf-8") as fout: + for entry in write_jsons: + fout.write(json.dumps(entry) + "\n") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/nemo_skills/dataset/ruler2/prepare_qa.py b/nemo_skills/dataset/ruler2/prepare_qa.py new file mode 100644 index 0000000000..1287e4ab31 --- /dev/null +++ b/nemo_skills/dataset/ruler2/prepare_qa.py @@ -0,0 +1,340 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +import os +import re +import json +import argparse +from pathlib import Path +from tqdm import tqdm +import random +import numpy as np +import subprocess +from datasets import load_dataset +from collections import defaultdict +from .tokenizer import select_tokenizer +import logging + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +parser = argparse.ArgumentParser() +# Basic Configurations +parser.add_argument("--output_folder", type=str) +parser.add_argument("--tokenizer_type", type=str, default='nemo', help='[Options] nemo, hf, openai.') +parser.add_argument("--tokenizer_path", type=str, required=True, help='path to the tokenizer model') +parser.add_argument("--max_seq_length", type=int, required=True, help='max sequence length including all input tokens and generated tokens.') +parser.add_argument("--random_seed", type=int, default=42) +parser.add_argument("--num_samples", type=int, default=500) +parser.add_argument("--dataset", type=str, required=True, help='dataset file') +parser.add_argument("--fewshot", type=int, default=0) +parser.add_argument("--prompt_type", type=str, default="chat") +parser.add_argument("--query_type", type=str, default="id", choices=["id", "doc", "question"]) +parser.add_argument("--algo_type", type=str, default="single", choices=["single", "2steps"]) +parser.add_argument("--task_type", type=str, default="retrieve", choices=["retrieve", "solve"]) + +args = parser.parse_args() +random.seed(args.random_seed) +np.random.seed(args.random_seed) + +# Load Tokenizer +TOKENIZER = select_tokenizer(args.tokenizer_type, args.tokenizer_path) + + +TOTAL_PROMPT = """{context}\n\n{example}{problem}""" +NEEDLE_PROMPT = "Document {i}:\n{text}" +if args.task_type == "retrieve": + if args.query_type == "id": + CONTEXT_PROMPT = """Below are some documents. I will ask you to copy one of them. Please copy and paste the document you find.\n\n{needles}""" + PROBLEM_PROMPT = "Please copy the Document {i} from the context." + if args.fewshot > 0: + EXAMPLE_PROMPT = PROBLEM_PROMPT + "\nDocument {i}:\n{text}" + if args.prompt_type == "base": + PROBLEM_PROMPT += "\nDocument {i}:" + + elif args.query_type == "doc" or args.query_type == "question": + if args.query_type == "doc": + CONTEXT_PROMPT = """Below are some documents. I will give you a text at the end. Please find the document index of the text. Only give me the index without any document contents.\n\n{needles}""" + PROBLEM_PROMPT = "Text: {text}\nMost relevant document index:" + elif args.query_type == "question": + # CONTEXT_PROMPT = """Below are some documents. I will give you a text at the end. Please find the document index most relevant to the text. Only give me the index without any document contents.\n\n{needles}""" + # PROBLEM_PROMPT = "Text: {question}\nMost relevant document index:" + CONTEXT_PROMPT = """Below are some documents. I will give you a question at the end. Please find the index of the most relevant document that can help answer the question. Only give me the index without any document contents.\n\n{needles}""" + PROBLEM_PROMPT = "Question: {question}\nIndex of the most relevant document that can help answer the question:" + if args.fewshot > 0: + EXAMPLE_PROMPT = PROBLEM_PROMPT + " {i}" +elif args.task_type == "solve": + CONTEXT_PROMPT = """Below are some documents. I will ask you to answer a question based on the documents. Please answer the question.\n\n{needles}""" + if args.algo_type == "single": + PROBLEM_PROMPT = "Please answer the following question based on the documents.\n\nQuestion: {question}" + # PROBLEM_PROMPT = "Please answer the following question based on the documents. If the question is a yes/no question, please only answer yes or no. If your answer can be extracted from the documents, please directly copy the answer span as short as possible. Please put your answer inside \\boxed{{}}.\n\nQuestion: {question}" + elif args.algo_type == "2steps": + PROBLEM_PROMPT = "Please first find and copy paste the documents relevant to the following question and then answer it based on the documents you find.\n\nQuestion: {question}" + if args.fewshot > 0: + EXAMPLE_PROMPT = PROBLEM_PROMPT + "\nAnswer: {answer}" + if args.prompt_type == "base": + PROBLEM_PROMPT += "\nAnswer:" + + +# Read SQuAD QA dataset +def read_squad(): + data = load_dataset("squad_v2")["train"] + haystack = [d['context'] for d in data] + haystack = list(set(haystack)) + haystack = [{ + "text": d + } for d in haystack] + + + data = load_dataset("squad_v2")["validation"] + title2context = defaultdict(set) + for d in data: + title2context[d['title']].add(d['context']) + + needle = [{ + "question": d['question'], + "answer": d['answers']['text'], + "context": [{"text": d['context']}], + "distractor": [{"text": t} for t in title2context[d['title']] if t != d['context']] + } for d in data] + needle = [n for n in needle if len(n["answer"]) > 0] + + return haystack, needle + +# Read Hotpot QA dataset +def read_hotpotqa(): + data = load_dataset(f"hotpotqa/hotpot_qa", "distractor")["train"] + haystack = [f"{t}\n{''.join(s)}" for d in data for t, s in zip(d['context']['title'], d['context']['sentences'])] + haystack = list(set(haystack)) + haystack = [{ + "text": d + } for d in haystack] + + data = load_dataset(f"hotpotqa/hotpot_qa", "distractor")["validation"] + needle = [{ + "question": d['question'], + "answer": [d['answer']], + "context": [{"text": f"{t}\n{''.join(s)}"} for t, s in zip(d['context']['title'], d['context']['sentences']) if t in d['supporting_facts']['title']], + "distractor": [{"text": f"{t}\n{''.join(s)}"} for t, s in zip(d['context']['title'], d['context']['sentences']) if t not in d['supporting_facts']['title']] + } for d in data] + needle = [n for n in needle if len(n["answer"]) > 0] + + return haystack, needle + + +def read_musique(): + data = load_dataset("dgslibisey/MuSiQue")["train"] + haystack = [f"{p['title']}\n{p['paragraph_text']}" for d in data for p in d['paragraphs']] + haystack = list(set(haystack)) + haystack = [{ + "text": d + } for d in haystack] + + data = load_dataset("dgslibisey/MuSiQue")["validation"] + needle = [{ + "question": d['question'], + "answer": [d['answer']] + d['answer_aliases'], + "context": [{"text": f"{p['title']}\n{p['paragraph_text']}"} for p in d['paragraphs'] if p['is_supporting']], + "distractor": [{"text": f"{p['title']}\n{p['paragraph_text']}"} for p in d['paragraphs'] if not p['is_supporting']] + } for d in data if d['answerable']] + needle = [n for n in needle if len(n["answer"]) > 0] + + return haystack, needle + +# Download dataset +if args.dataset == 'squad': + haystack, needle = read_squad() +elif args.dataset == 'hotpotqa': + haystack, needle = read_hotpotqa() +elif args.dataset == 'musique': + haystack, needle = read_musique() +else: + raise NotImplementedError(f'{args.dataset} is not implemented.') + + +def generate_random_number(num_digits=7): + lower_bound = 10**(num_digits - 1) + upper_bound = 10**num_digits - 1 + return str(random.randint(lower_bound, upper_bound)) + + +def generate_input_output(index, num_docs): + + curr_needle = needle[index] + curr_needle["context"] = [{**c, "random_index": generate_random_number()} for c in curr_needle["context"]] + curr_needle["distractor"] = [{**c, "random_index": generate_random_number()} for c in curr_needle["distractor"]] + + if args.fewshot > 0: + fewshot_examples = random.sample([i for i in range(len(needle)) if i != index], args.fewshot) + fewshot_examples = [needle[i] for i in fewshot_examples] + for e in fewshot_examples: + e["context"] = [{**c, "random_index": generate_random_number()} for c in e['context']] + e["distractor"] = [{**c, "random_index": generate_random_number()} for c in e['distractor']] + else: + fewshot_examples = [] + + remaining_haystack_size = len(haystack) - len(set([c["text"] for c in (curr_needle["context"] + curr_needle["distractor"])] + [f["text"] for e in fewshot_examples for f in (e["context"] + e["distractor"])])) + if num_docs > remaining_haystack_size: + repeats = (num_docs + remaining_haystack_size - 1) // remaining_haystack_size # Ceiling division + else: + repeats = 1 + + curr_context = random.sample([i for i in range(len(haystack)) for _ in range(repeats)], num_docs) + curr_context = [{**haystack[i], "random_index": generate_random_number()} for i in curr_context] + curr_context = curr_context + curr_needle["context"] + [item for example in fewshot_examples for item in example["context"]] + if num_docs > 0: + curr_context = curr_context + curr_needle["distractor"] + [item for example in fewshot_examples for item in example["distractor"]] + random.shuffle(curr_context) + + needles = '\n\n'.join([NEEDLE_PROMPT.format(i=c["random_index"], text=c["text"]) for c in curr_context]) + if args.task_type == "retrieve": + if args.query_type == "id": + problem = PROBLEM_PROMPT.format(i=curr_needle["context"][0]["random_index"]) + elif args.query_type == "doc": + problem = PROBLEM_PROMPT.format(text=curr_needle["context"][0]["text"]) + elif args.query_type == "question": + problem = PROBLEM_PROMPT.format(question=curr_needle["question"]) + elif args.task_type == "solve": + problem = PROBLEM_PROMPT.format(question=curr_needle["question"]) + + + if args.fewshot > 0: + if args.task_type == "retrieve": + if args.query_type == "id": + example = '\n\n'.join([EXAMPLE_PROMPT.format(i=e["context"][0]["random_index"], text=e["context"][0]["text"]) for e in fewshot_examples]) + elif args.query_type == "doc": + example = '\n\n'.join([EXAMPLE_PROMPT.format(i=e["context"][0]["random_index"], text=e["context"][0]["text"]) for e in fewshot_examples]) + elif args.query_type == "question": + example = '\n\n'.join([EXAMPLE_PROMPT.format(i=random.sample(e["context"], 1)[0]["random_index"], question=e["question"]) for e in fewshot_examples]) + + elif args.task_type == "solve": + example = '\n\n'.join([EXAMPLE_PROMPT.format(answer=random.sample(e["answer"], 1)[0], question=e["question"]) for e in fewshot_examples]) + + if args.prompt_type == "base": + example = f"{example}\n\n" + else: + example = f"Here are some examples to help you understand the task:\n\n{example}\n\nHere is the actual task you need to solve:\n\n" + + else: + example = "" + + + context = CONTEXT_PROMPT.format(needles=needles) + input_text = TOTAL_PROMPT.format( + context=context, + problem=problem, + example=example + ) + if args.task_type == "retrieve": + if args.query_type == "id": + expected_answer = { + "expected_answer" : [curr_needle["context"][0]["text"]] + } + elif args.query_type == "doc": + expected_answer = { + "expected_answer" : [curr_needle["context"][0]["random_index"]] + } + elif args.query_type == "question": + expected_answer = { + "expected_answer" : [c["random_index"] for c in curr_needle["context"]] + } + elif args.task_type == "solve": + expected_answer = { + "expected_answer" : curr_needle["answer"] + } + + save_dict = { + "index": index, + "question": f"{context}\n\n{example}{problem}", + **expected_answer, + } + return input_text, save_dict + +def generate_samples(num_samples: int, max_seq_length: int, incremental: int = 5): + + write_jsons = [] + + # Estimate tokens per question to determine reasonable upper bound + sample_input_text, _ = generate_input_output(0, incremental) + sample_tokens = len(TOKENIZER.text_to_tokens(sample_input_text)) + tokens_per_doc = sample_tokens / incremental + + if max_seq_length > 0: + # Let's do 3x to allow for some slack since we can get unlucky due to sampling. + # NOTE: We should test this for really large sequence lengths to make sure it's reasonable. + estimated_max_docs = int((max_seq_length / tokens_per_doc) * 3) + + # Binary search for optimal haystack size + lower_bound = incremental + upper_bound = max(estimated_max_docs, incremental * 2) # Ensure upper_bound is reasonable + + optimal_num_docs = None + + logger.info(f"Estimated {tokens_per_doc:.1f} tokens per doc") + logger.info(f"Starting binary search with bounds: {lower_bound} to {upper_bound}") + + while lower_bound <= upper_bound: + mid = (lower_bound + upper_bound) // 2 + input_text, save_dict = generate_input_output(0, mid) + total_tokens = len(TOKENIZER.text_to_tokens(input_text)) + + logger.info(f"Testing haystack size: {mid}, resulting tokens: {total_tokens}/{max_seq_length}") + + if total_tokens <= max_seq_length: + # This size works, can we go larger? + optimal_num_docs = mid + lower_bound = mid + 1 + else: + # Too large, need to go smaller + upper_bound = mid - 1 + + num_docs = optimal_num_docs if optimal_num_docs is not None else incremental + logger.info(f'Final optimal haystack size (number of docs): {num_docs}') + else: + num_docs = 0 + + # Generate samples + for index in tqdm(range(num_samples)): + used_docs = num_docs + while(True): + try: + input_text, save_dict = generate_input_output(index, used_docs) + length = len(TOKENIZER.text_to_tokens(input_text)) + if max_seq_length > 0: + assert length <= max_seq_length, f"{length} exceeds max_seq_length." + break + except: + if used_docs > incremental: + used_docs -= incremental + + save_dict["length"] = length + formatted_output = save_dict + write_jsons.append(formatted_output) + + return write_jsons + + +def main(): + output_file = str(args.output_folder / "test.jsonl") + + write_jsons = generate_samples( + num_samples=args.num_samples, + max_seq_length=args.max_seq_length, + ) + with open(output_file, "wt", encoding="utf-8") as fout: + for entry in write_jsons: + fout.write(json.dumps(entry) + "\n") + +if __name__=="__main__": + main() diff --git a/nemo_skills/dataset/ruler2/ruler2_score.py b/nemo_skills/dataset/ruler2/ruler2_score.py new file mode 100644 index 0000000000..7e34448bda --- /dev/null +++ b/nemo_skills/dataset/ruler2/ruler2_score.py @@ -0,0 +1,42 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def compute_score(metrics: dict): + # just an average of all metrics. Here we assume that all tasks are present. + # if that's not the case, users shouldn't run as a group + tasks = [ + "mk_niah_basic", + "mk_niah_easy", + "mk_niah_medium", + "mk_niah_hard", + "mv_niah_basic", + "mv_niah_easy", + "mv_niah_medium", + "mv_niah_hard", + "qa_basic", + "qa_easy", + "qa_medium", + "qa_hard", + + ] + setup = list(metrics.keys())[0].rsplit(".", 1)[0] + metrics[setup] = {} + + for aggregation in metrics[f"{setup}.mk_niah_basic"]: + metrics[setup][aggregation] = { + "accuracy": sum(metrics[f"{setup}.{task}"][aggregation]["accuracy"] for task in tasks) / len(tasks) + } + + return metrics diff --git a/nemo_skills/dataset/ruler2/tokenizer.py b/nemo_skills/dataset/ruler2/tokenizer.py new file mode 100644 index 0000000000..567ec8ef06 --- /dev/null +++ b/nemo_skills/dataset/ruler2/tokenizer.py @@ -0,0 +1,127 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +from typing import List +from tenacity import ( + retry, + stop_after_attempt, + wait_fixed, + wait_random, +) + + +def select_tokenizer(tokenizer_type, tokenizer_path): + if tokenizer_type == 'nemo': + if '.model' in tokenizer_path: + return NeMoSentencePieceTokenizer(model_path=tokenizer_path) + elif '.json' in tokenizer_path: + return NeMoTikTokenTokenizer(vocab_file=tokenizer_path) + else: + raise ValueError(f"Unknown tokenizer file format {tokenizer_path}") + elif tokenizer_type == 'hf': + return HFTokenizer(model_path=tokenizer_path) + elif tokenizer_type == 'openai': + return OpenAITokenizer(model_path=tokenizer_path) + elif tokenizer_type == 'gemini': + return GeminiTokenizer(model_path=tokenizer_path) + else: + raise ValueError(f"Unknown tokenizer_type {tokenizer_type}") + + +class NeMoTikTokenTokenizer: + """ + Tokenizer from NeMo TiktokenTokenizer + """ + def __init__(self, vocab_file) -> None: + from nemo.collections.common.tokenizers.tiktoken_tokenizer import TiktokenTokenizer + self.tokenizer = TiktokenTokenizer(vocab_file=vocab_file) + + def text_to_tokens(self, text: str) -> List[str]: + tokens = self.tokenizer.text_to_tokens(text) + return tokens + + def tokens_to_text(self, tokens: List[int]) -> str: + text = self.tokenizer.tokens_to_text(tokens) + return text + + +class NeMoSentencePieceTokenizer: + """ + Tokenizer from NeMo SentencePieceTokenizer + """ + def __init__(self, model_path) -> None: + from nemo.collections.common.tokenizers.sentencepiece_tokenizer import SentencePieceTokenizer + self.tokenizer = SentencePieceTokenizer(model_path=model_path) + + def text_to_tokens(self, text: str) -> List[str]: + tokens = self.tokenizer.text_to_tokens(text) + return tokens + + def tokens_to_text(self, tokens: List[int]) -> str: + text = self.tokenizer.tokens_to_text(tokens) + return text + + +class HFTokenizer: + """ + Tokenizer from HF models + """ + def __init__(self, model_path) -> None: + from transformers import AutoTokenizer + self.tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True) + + def text_to_tokens(self, text: str) -> List[str]: + tokens = self.tokenizer.tokenize(text) + return tokens + + def tokens_to_text(self, tokens: List[int]) -> str: + text = self.tokenizer.convert_tokens_to_string(tokens) + return text + + +class OpenAITokenizer: + """ + Tokenizer from tiktoken + """ + def __init__(self, model_path="cl100k_base") -> None: + import tiktoken + self.tokenizer = tiktoken.get_encoding(model_path) + + def text_to_tokens(self, text: str) -> List[int]: + tokens = self.tokenizer.encode(text) + return tokens + + def tokens_to_text(self, tokens: List[int]) -> str: + text = self.tokenizer.decode(tokens) + return text + + +class GeminiTokenizer: + """ + Tokenizer from gemini + """ + def __init__(self, model_path="gemini-1.5-pro-latest") -> None: + import google.generativeai as genai + genai.configure(api_key=os.environ["GEMINI_API_KEY"]) + self.model = genai.GenerativeModel(model_path) + + @retry(wait=wait_fixed(60) + wait_random(0, 10), stop=stop_after_attempt(3)) + def text_to_tokens(self, text: str) -> List[int]: + tokens = list(range(self.model.count_tokens(text).total_tokens)) + return tokens + + def tokens_to_text(self, tokens: List[int]) -> str: + pass diff --git a/nemo_skills/evaluation/evaluator/__init__.py b/nemo_skills/evaluation/evaluator/__init__.py index 21f8a0e3d2..a6cd083c71 100644 --- a/nemo_skills/evaluation/evaluator/__init__.py +++ b/nemo_skills/evaluation/evaluator/__init__.py @@ -37,7 +37,7 @@ from nemo_skills.evaluation.evaluator.mcq import eval_mcq from nemo_skills.evaluation.evaluator.mmau_pro import eval_mmau_pro from nemo_skills.evaluation.evaluator.mrcr import eval_mrcr -from nemo_skills.evaluation.evaluator.ruler import eval_ruler +from nemo_skills.evaluation.evaluator.ruler import eval_ruler, eval_ruler2 from nemo_skills.evaluation.evaluator.scicode import eval_scicode EVALUATOR_MAP = { @@ -48,6 +48,7 @@ "bfcl": eval_bfcl, "multichoice": eval_mcq, "ruler": eval_ruler, + "ruler2": eval_ruler2, "livecodebench": eval_livecodebench, "livebench_coding": eval_livebench_coding, "livecodebench_pro": eval_livecodebench_pro, diff --git a/nemo_skills/evaluation/evaluator/ruler.py b/nemo_skills/evaluation/evaluator/ruler.py index b43393675a..ec6802d998 100644 --- a/nemo_skills/evaluation/evaluator/ruler.py +++ b/nemo_skills/evaluation/evaluator/ruler.py @@ -66,6 +66,7 @@ def string_match_part_single(preds, refs): "part": string_match_part_single, } +<<<<<<< HEAD jsonl_file = eval_config.input_file with open(jsonl_file, "rt", encoding="utf-8") as fin: data = [json.loads(line) for line in fin] @@ -81,3 +82,87 @@ def string_match_part_single(preds, refs): fout.write(json.dumps(sample) + "\n") os.replace(jsonl_file + "-tmp", jsonl_file) +======= + for file in unroll_files(cfg.input_files): + with open(file, "rt", encoding="utf-8") as fin: + data = [json.loads(line) for line in fin] + with open(file, "wt", encoding="utf-8") as fout: + for sample in tqdm(data): + parse_result = parse_funcs[eval_config.parse_func](sample["generation"]) + sample["is_correct"] = match_type_funcs[eval_config.match_type]( + sample["generation"], sample["expected_answer"] + ) + sample["predicted_answer"] = parse_result + fout.write(json.dumps(sample) + "\n") + + +def eval_ruler2(cfg): + def default_parse(prediction): + prediction = prediction.strip() + # Remove all non-printable characters + np_pattern = re.compile(r"[\x00-\x1f]") + pp_predict = np_pattern.sub("\n", prediction).strip() + return pp_predict + + def post_process_preds(preds): + if "" in preds: + preds = preds.split("")[-1] + + if "Answer:" in preds: + preds = preds.split("Answer:")[-1] + return preds + + def string_match_all_single(preds, refs): + """the metric function with input (predictions: [str], references: [[str]]) to compute score.""" + preds = post_process_preds(preds) + preds = [preds] + refs = [refs] + score = [ + sum([1.0 if r.lower() in pred.lower() else 0.0 for r in ref]) / len(ref) for pred, ref in zip(preds, refs) + ][0] + return score + + def string_match_2steps_single(preds, refs): + preds = post_process_preds(preds) + preds = preds.split("\n\n")[-1] + preds = [preds] + refs = [refs] + score = [ + sum([1.0 if r.lower() in pred.lower() else 0.0 for r in ref]) / len(ref) for pred, ref in zip(preds, refs) + ][0] + return score + + def string_match_part_single(preds, refs): + preds = post_process_preds(preds) + preds = re.sub(r'Document \d+:(?:.*\n)+?\n', '', preds) + + preds = [preds] + refs = [refs] + score = [ + sum([max([1.0 if r.lower() in pred.lower() else 0.0 for r in ref]) for pred, ref in zip(preds, refs)]) + ][0] + return score + + eval_config = RulerEvaluatorConfig(**cfg.eval_config) + + parse_funcs = { + "default": default_parse, + } + match_type_funcs = { + "all": string_match_all_single, + "part": string_match_part_single, + "2steps": string_match_2steps_single, + } + + for file in unroll_files(cfg.input_files): + with open(file, "rt", encoding="utf-8") as fin: + data = [json.loads(line) for line in fin] + with open(file, "wt", encoding="utf-8") as fout: + for sample in tqdm(data): + parse_result = parse_funcs[eval_config.parse_func](sample["generation"]) + sample["is_correct"] = match_type_funcs[eval_config.match_type]( + sample["generation"], sample["expected_answer"] + ) + sample["predicted_answer"] = parse_result + fout.write(json.dumps(sample) + "\n") +>>>>>>> 2f7a3a33 (add initial ruler2) From e970ab2a0ca6b7803c2aa99914617f8477733f92 Mon Sep 17 00:00:00 2001 From: Cheng-Ping Hsieh Date: Wed, 17 Sep 2025 16:26:18 -0700 Subject: [PATCH 02/88] fix syntax error Signed-off-by: Cheng-Ping Hsieh --- nemo_skills/dataset/ruler2/prepare.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nemo_skills/dataset/ruler2/prepare.py b/nemo_skills/dataset/ruler2/prepare.py index 5b8b955b72..62acad9707 100644 --- a/nemo_skills/dataset/ruler2/prepare.py +++ b/nemo_skills/dataset/ruler2/prepare.py @@ -351,7 +351,7 @@ def prepare_dataset(tasks, setup, max_seq_length, tokenizer_type, tokenizer_path parser.add_argument( "--tokenizer_type", type=str, - default="hf" + default="hf", help="Type of the tokenizer to use.", ) parser.add_argument( From f62aa6b2fd7ecb989e842c71602173dfe28c2b54 Mon Sep 17 00:00:00 2001 From: Cheng-Ping Hsieh Date: Wed, 17 Sep 2025 16:30:18 -0700 Subject: [PATCH 03/88] fix Signed-off-by: Cheng-Ping Hsieh --- nemo_skills/dataset/ruler2/prepare.py | 33 +++++++++++++++------------ 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/nemo_skills/dataset/ruler2/prepare.py b/nemo_skills/dataset/ruler2/prepare.py index 62acad9707..f5451de9d2 100644 --- a/nemo_skills/dataset/ruler2/prepare.py +++ b/nemo_skills/dataset/ruler2/prepare.py @@ -28,21 +28,6 @@ ) """ -prepare_task = { - "mk_niah_basic": prepare_mk_niah_basic, - "mk_niah_easy": prepare_mk_niah_easy, - "mk_niah_medium": prepare_mk_niah_medium, - "mk_niah_hard": prepare_mk_niah_hard, - "mv_niah_basic": prepare_mv_niah_basic, - "mv_niah_easy": prepare_mv_niah_easy, - "mv_niah_medium": prepare_mv_niah_medium, - "mv_niah_hard": prepare_mv_niah_hard, - "qa_basic": prepare_qa_basic, - "qa_easy": prepare_qa_easy, - "qa_medium": prepare_qa_medium, - "qa_hard": prepare_qa_hard, -} - def prepare_mk_niah_basic(output_folder, tokenizer_type, tokenizer_path, length, dataset_size): subprocess.run( @@ -272,6 +257,8 @@ def prepare_qa_hard(output_folder, tokenizer_type, tokenizer_path, length, datas check=True, ) + + def prepare_task_for_ns(output_folder): """Adding proper __init__.py""" Path(output_folder).mkdir(parents=True, exist_ok=True) @@ -288,6 +275,22 @@ def prepare_task_for_ns(output_folder): init_file.write(DEFAULT_SETTINGS.format(eval_args=eval_args)) def prepare_dataset(tasks, setup, max_seq_length, tokenizer_type, tokenizer_path, dataset_size): + prepare_task = { + "mk_niah_basic": prepare_mk_niah_basic, + "mk_niah_easy": prepare_mk_niah_easy, + "mk_niah_medium": prepare_mk_niah_medium, + "mk_niah_hard": prepare_mk_niah_hard, + "mv_niah_basic": prepare_mv_niah_basic, + "mv_niah_easy": prepare_mv_niah_easy, + "mv_niah_medium": prepare_mv_niah_medium, + "mv_niah_hard": prepare_mv_niah_hard, + "qa_basic": prepare_qa_basic, + "qa_easy": prepare_qa_easy, + "qa_medium": prepare_qa_medium, + "qa_hard": prepare_qa_hard, + } + + output_folder = Path(__file__).parent / setup # 1. installing necessary packages From 0d290fdc10774904768f21698f6f48d9d52fa388 Mon Sep 17 00:00:00 2001 From: Cheng-Ping Hsieh Date: Wed, 17 Sep 2025 16:32:39 -0700 Subject: [PATCH 04/88] fix Signed-off-by: Cheng-Ping Hsieh --- nemo_skills/dataset/ruler2/prepare.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nemo_skills/dataset/ruler2/prepare.py b/nemo_skills/dataset/ruler2/prepare.py index f5451de9d2..2ac6ad8eb5 100644 --- a/nemo_skills/dataset/ruler2/prepare.py +++ b/nemo_skills/dataset/ruler2/prepare.py @@ -259,8 +259,9 @@ def prepare_qa_hard(output_folder, tokenizer_type, tokenizer_path, length, datas -def prepare_task_for_ns(output_folder): +def prepare_task_for_ns(output_folder, task): """Adding proper __init__.py""" + output_folder = Path(output_folder) / task Path(output_folder).mkdir(parents=True, exist_ok=True) with open(output_folder / "__init__.py", "w", encoding="utf-8") as init_file: if task in ["mk_niah_medium", "mk_niah_hard"]: @@ -297,7 +298,7 @@ def prepare_dataset(tasks, setup, max_seq_length, tokenizer_type, tokenizer_path subprocess.run(["pip install wonderwords html2text tenacity"], check=True, shell=True) for task in tasks: - prepare_task_for_ns(output_folder / task) + prepare_task_for_ns(output_folder, task) # preparing the datasets based on user options, in parallel with concurrent.futures.ThreadPoolExecutor() as executor: From b9eb3d47d487d0b7c3c1c5636fee5941ac38cf07 Mon Sep 17 00:00:00 2001 From: Cheng-Ping Hsieh Date: Wed, 17 Sep 2025 16:38:25 -0700 Subject: [PATCH 05/88] fix Signed-off-by: Cheng-Ping Hsieh --- nemo_skills/dataset/ruler2/prepare.py | 3 ++- nemo_skills/dataset/ruler2/prepare_mmlu.py | 4 ++-- nemo_skills/dataset/ruler2/prepare_niah.py | 2 +- nemo_skills/dataset/ruler2/prepare_qa.py | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/nemo_skills/dataset/ruler2/prepare.py b/nemo_skills/dataset/ruler2/prepare.py index 2ac6ad8eb5..254ddd3084 100644 --- a/nemo_skills/dataset/ruler2/prepare.py +++ b/nemo_skills/dataset/ruler2/prepare.py @@ -302,7 +302,8 @@ def prepare_dataset(tasks, setup, max_seq_length, tokenizer_type, tokenizer_path # preparing the datasets based on user options, in parallel with concurrent.futures.ThreadPoolExecutor() as executor: - futures = [executor.submit(prepare_task[task], + futures = [executor.submit( + prepare_task[task], str(output_folder / task), tokenizer_type, tokenizer_path, diff --git a/nemo_skills/dataset/ruler2/prepare_mmlu.py b/nemo_skills/dataset/ruler2/prepare_mmlu.py index fbd92bbe64..c86ce87911 100644 --- a/nemo_skills/dataset/ruler2/prepare_mmlu.py +++ b/nemo_skills/dataset/ruler2/prepare_mmlu.py @@ -37,7 +37,7 @@ # Basic Configurations parser.add_argument("--output_folder", type=str) parser.add_argument("--tokenizer_path", type=str, required=True, help='path to the tokenizer model') -parser.add_argument("--tokenizer_type", type=str, default='nemo', help='[Options] nemo, hf, openai.') +parser.add_argument("--tokenizer_type", type=str, default='hf', help='[Options] nemo, hf, openai.') parser.add_argument("--max_seq_length", type=int, required=True, help='max sequence length including all input tokens and generated tokens.') parser.add_argument("--random_seed", type=int, default=42) parser.add_argument("--insert_position", type=float, default=-1, help='insert position of the true context in the context.') @@ -168,7 +168,7 @@ choices = d["choices"] item = { "Question": d['question'] + f'\nA. {choices[0]}\nB. {choices[1]}\nC. {choices[2]}\nD. {choices[3]}', - "Solution": " " + f'\\boxed{{{options[d["answer"]]}}}' + "Solution": " " + f'\\boxed{{{options[d["answer"]]}}}', "Answer": options[d['answer']], } needle.append(item) diff --git a/nemo_skills/dataset/ruler2/prepare_niah.py b/nemo_skills/dataset/ruler2/prepare_niah.py index 0dd9de76df..f0bbc083b6 100644 --- a/nemo_skills/dataset/ruler2/prepare_niah.py +++ b/nemo_skills/dataset/ruler2/prepare_niah.py @@ -39,7 +39,7 @@ parser = argparse.ArgumentParser() parser.add_argument("--output_folder", type=str) -parser.add_argument("--tokenizer_type", type=str, default='nemo', help='[Options] nemo, hf, openai.') +parser.add_argument("--tokenizer_type", type=str, default='hf', help='[Options] nemo, hf, openai.') parser.add_argument("--tokenizer_path", type=str, required=True, help='path to the tokenizer model') parser.add_argument("--max_seq_length", type=int, required=True, help='max sequence length including all input tokens and generated tokens.') parser.add_argument("--num_samples", type=int, required=True, help='number of samples to generate') diff --git a/nemo_skills/dataset/ruler2/prepare_qa.py b/nemo_skills/dataset/ruler2/prepare_qa.py index 1287e4ab31..1ffaee0488 100644 --- a/nemo_skills/dataset/ruler2/prepare_qa.py +++ b/nemo_skills/dataset/ruler2/prepare_qa.py @@ -32,7 +32,7 @@ parser = argparse.ArgumentParser() # Basic Configurations parser.add_argument("--output_folder", type=str) -parser.add_argument("--tokenizer_type", type=str, default='nemo', help='[Options] nemo, hf, openai.') +parser.add_argument("--tokenizer_type", type=str, default='hf', help='[Options] nemo, hf, openai.') parser.add_argument("--tokenizer_path", type=str, required=True, help='path to the tokenizer model') parser.add_argument("--max_seq_length", type=int, required=True, help='max sequence length including all input tokens and generated tokens.') parser.add_argument("--random_seed", type=int, default=42) From e7211abdef8bb9ca61722fc1a7735029833d6f0a Mon Sep 17 00:00:00 2001 From: Cheng-Ping Hsieh Date: Wed, 17 Sep 2025 16:39:40 -0700 Subject: [PATCH 06/88] fix Signed-off-by: Cheng-Ping Hsieh --- nemo_skills/dataset/ruler2/prepare_mmlu.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nemo_skills/dataset/ruler2/prepare_mmlu.py b/nemo_skills/dataset/ruler2/prepare_mmlu.py index c86ce87911..cfabe1ed89 100644 --- a/nemo_skills/dataset/ruler2/prepare_mmlu.py +++ b/nemo_skills/dataset/ruler2/prepare_mmlu.py @@ -36,8 +36,8 @@ parser = argparse.ArgumentParser() # Basic Configurations parser.add_argument("--output_folder", type=str) -parser.add_argument("--tokenizer_path", type=str, required=True, help='path to the tokenizer model') parser.add_argument("--tokenizer_type", type=str, default='hf', help='[Options] nemo, hf, openai.') +parser.add_argument("--tokenizer_path", type=str, required=True, help='path to the tokenizer model') parser.add_argument("--max_seq_length", type=int, required=True, help='max sequence length including all input tokens and generated tokens.') parser.add_argument("--random_seed", type=int, default=42) parser.add_argument("--insert_position", type=float, default=-1, help='insert position of the true context in the context.') From 16e7e210aac19cd54a33ad7ecf8b06a4796f36c7 Mon Sep 17 00:00:00 2001 From: Cheng-Ping Hsieh Date: Wed, 17 Sep 2025 16:43:45 -0700 Subject: [PATCH 07/88] fix Signed-off-by: Cheng-Ping Hsieh --- nemo_skills/dataset/ruler2/prepare.py | 96 +++++++++++----------- nemo_skills/dataset/ruler2/prepare_mmlu.py | 2 +- 2 files changed, 49 insertions(+), 49 deletions(-) diff --git a/nemo_skills/dataset/ruler2/prepare.py b/nemo_skills/dataset/ruler2/prepare.py index 254ddd3084..dc80ef2e73 100644 --- a/nemo_skills/dataset/ruler2/prepare.py +++ b/nemo_skills/dataset/ruler2/prepare.py @@ -33,10 +33,10 @@ def prepare_mk_niah_basic(output_folder, tokenizer_type, tokenizer_path, length, subprocess.run( f"python -m nemo_skills.dataset.ruler2.prepare_niah " f"--output_folder {output_folder} " - f"--tokenizer_type ${tokenizer_type} " - f"--tokenizer_path ${tokenizer_path} " - f"--max_seq_length ${length} " - f"--num_samples ${dataset_size} " + f"--tokenizer_type {tokenizer_type} " + f"--tokenizer_path {tokenizer_path} " + f"--max_seq_length {length} " + f"--num_samples {dataset_size} " f"--random_seed 42 " f"--num_needle_k 1 " f"--num_needle_v 1 " @@ -53,10 +53,10 @@ def prepare_mk_niah_easy(output_folder, tokenizer_type, tokenizer_path, length, subprocess.run( f"python -m nemo_skills.dataset.ruler2.prepare_mmlu " f"--output_folder {output_folder} " - f"--tokenizer_type ${tokenizer_type} " - f"--tokenizer_path ${tokenizer_path} " - f"--max_seq_length ${length} " - f"--num_samples ${dataset_size} " + f"--tokenizer_type {tokenizer_type} " + f"--tokenizer_path {tokenizer_path} " + f"--max_seq_length {length} " + f"--num_samples {dataset_size} " f"--random_seed 42 " f"--dataset mmlu " f"--fewshot 0 " @@ -72,10 +72,10 @@ def prepare_mk_niah_medium(output_folder, tokenizer_type, tokenizer_path, length subprocess.run( f"python -m nemo_skills.dataset.ruler2.prepare_mmlu " f"--output_folder {output_folder} " - f"--tokenizer_type ${tokenizer_type} " - f"--tokenizer_path ${tokenizer_path} " - f"--max_seq_length ${length} " - f"--num_samples ${dataset_size} " + f"--tokenizer_type {tokenizer_type} " + f"--tokenizer_path {tokenizer_path} " + f"--max_seq_length {length} " + f"--num_samples {dataset_size} " f"--random_seed 42 " f"--dataset mmlu " f"--fewshot 5 " @@ -91,10 +91,10 @@ def prepare_mk_niah_hard(output_folder, tokenizer_type, tokenizer_path, length, subprocess.run( f"python -m nemo_skills.dataset.ruler2.prepare_mmlu " f"--output_folder {output_folder} " - f"--tokenizer_type ${tokenizer_type} " - f"--tokenizer_path ${tokenizer_path} " - f"--max_seq_length ${length} " - f"--num_samples ${dataset_size} " + f"--tokenizer_type {tokenizer_type} " + f"--tokenizer_path {tokenizer_path} " + f"--max_seq_length {length} " + f"--num_samples {dataset_size} " f"--random_seed 42 " f"--dataset mmlu " f"--fewshot 5 " @@ -110,10 +110,10 @@ def prepare_mv_niah_basic(output_folder, tokenizer_type, tokenizer_path, length, subprocess.run( f"python -m prepare.py nemo_skills.dataset.ruler2.prepare_niah " f"--output_folder {output_folder} " - f"--tokenizer_type ${tokenizer_type} " - f"--tokenizer_path ${tokenizer_path} " - f"--max_seq_length ${length} " - f"--num_samples ${dataset_size} " + f"--tokenizer_type {tokenizer_type} " + f"--tokenizer_path {tokenizer_path} " + f"--max_seq_length {length} " + f"--num_samples {dataset_size} " f"--random_seed 42 " f"--num_needle_k 1 " f"--num_needle_v 4 " @@ -130,10 +130,10 @@ def prepare_mv_niah_easy(output_folder, tokenizer_type, tokenizer_path, length, subprocess.run( f"python -m nemo_skills.dataset.ruler2.prepare_mmlu " f"--output_folder {output_folder} " - f"--tokenizer_type ${tokenizer_type} " - f"--tokenizer_path ${tokenizer_path} " - f"--max_seq_length ${length} " - f"--num_samples ${dataset_size} " + f"--tokenizer_type {tokenizer_type} " + f"--tokenizer_path {tokenizer_path} " + f"--max_seq_length {length} " + f"--num_samples {dataset_size} " f"--random_seed 42 " f"--dataset mmlu " f"--fewshot 0 " @@ -149,10 +149,10 @@ def prepare_mv_niah_medium(output_folder, tokenizer_type, tokenizer_path, length subprocess.run( f"python -m nemo_skills.dataset.ruler2.prepare_mmlu " f"--output_folder {output_folder} " - f"--tokenizer_type ${tokenizer_type} " - f"--tokenizer_path ${tokenizer_path} " - f"--max_seq_length ${length} " - f"--num_samples ${dataset_size} " + f"--tokenizer_type {tokenizer_type} " + f"--tokenizer_path {tokenizer_path} " + f"--max_seq_length {length} " + f"--num_samples {dataset_size} " f"--random_seed 42 " f"--dataset mmlu " f"--fewshot 0 " @@ -168,10 +168,10 @@ def prepare_mv_niah_hard(output_folder, tokenizer_type, tokenizer_path, length, subprocess.run( f"python -m nemo_skills.dataset.ruler2.prepare_mmlu " f"--output_folder {output_folder} " - f"--tokenizer_type ${tokenizer_type} " - f"--tokenizer_path ${tokenizer_path} " - f"--max_seq_length ${length} " - f"--num_samples ${dataset_size} " + f"--tokenizer_type {tokenizer_type} " + f"--tokenizer_path {tokenizer_path} " + f"--max_seq_length {length} " + f"--num_samples {dataset_size} " f"--random_seed 42 " f"--dataset mmlu " f"--fewshot 0 " @@ -188,10 +188,10 @@ def prepare_qa_basic(output_folder, tokenizer_type, tokenizer_path, length, data subprocess.run( f"python -m nemo_skills.dataset.ruler2.prepare_qa " f"--output_folder {output_folder} " - f"--tokenizer_type ${tokenizer_type} " - f"--tokenizer_path ${tokenizer_path} " - f"--max_seq_length ${length} " - f"--num_samples ${dataset_size} " + f"--tokenizer_type {tokenizer_type} " + f"--tokenizer_path {tokenizer_path} " + f"--max_seq_length {length} " + f"--num_samples {dataset_size} " f"--random_seed 42 " f"--dataset hotpotqa " f"--fewshot 0 " @@ -206,10 +206,10 @@ def prepare_qa_easy(output_folder, tokenizer_type, tokenizer_path, length, datas subprocess.run( f"python -m nemo_skills.dataset.ruler2.prepare_qa " f"--output_folder {output_folder} " - f"--tokenizer_type ${tokenizer_type} " - f"--tokenizer_path ${tokenizer_path} " - f"--max_seq_length ${length} " - f"--num_samples ${dataset_size} " + f"--tokenizer_type {tokenizer_type} " + f"--tokenizer_path {tokenizer_path} " + f"--max_seq_length {length} " + f"--num_samples {dataset_size} " f"--random_seed 42 " f"--dataset hotpotqa " f"--fewshot 0 " @@ -225,10 +225,10 @@ def prepare_qa_medium(output_folder, tokenizer_type, tokenizer_path, length, dat subprocess.run( f"python -m nemo_skills.dataset.ruler2.prepare_qa " f"--output_folder {output_folder} " - f"--tokenizer_type ${tokenizer_type} " - f"--tokenizer_path ${tokenizer_path} " - f"--max_seq_length ${length} " - f"--num_samples ${dataset_size} " + f"--tokenizer_type {tokenizer_type} " + f"--tokenizer_path {tokenizer_path} " + f"--max_seq_length {length} " + f"--num_samples {dataset_size} " f"--random_seed 42 " f"--dataset hotpotqa " f"--fewshot 0 " @@ -243,10 +243,10 @@ def prepare_qa_hard(output_folder, tokenizer_type, tokenizer_path, length, datas subprocess.run( f"python -m nemo_skills.dataset.ruler2.prepare_qa " f"--output_folder {output_folder} " - f"--tokenizer_type ${tokenizer_type} " - f"--tokenizer_path ${tokenizer_path} " - f"--max_seq_length ${length} " - f"--num_samples ${dataset_size} " + f"--tokenizer_type {tokenizer_type} " + f"--tokenizer_path {tokenizer_path} " + f"--max_seq_length {length} " + f"--num_samples {dataset_size} " f"--random_seed 42 " f"--dataset hotpotqa " f"--fewshot 0 " diff --git a/nemo_skills/dataset/ruler2/prepare_mmlu.py b/nemo_skills/dataset/ruler2/prepare_mmlu.py index cfabe1ed89..471cdb9a1e 100644 --- a/nemo_skills/dataset/ruler2/prepare_mmlu.py +++ b/nemo_skills/dataset/ruler2/prepare_mmlu.py @@ -177,7 +177,7 @@ choices = d["choices"] item = { "Question": d['question'] + f'\nA. {choices[0]}\nB. {choices[1]}\nC. {choices[2]}\nD. {choices[3]}', - "Solution": " " + f'\\boxed{{{options[d["answer"]]}}}' + "Solution": " " + f'\\boxed{{{options[d["answer"]]}}}', "Answer": options[d['answer']], } haystack.append(item) From f567774ab0f17ec7513b9f777c39e2c842cd43a8 Mon Sep 17 00:00:00 2001 From: Cheng-Ping Hsieh Date: Wed, 17 Sep 2025 16:48:27 -0700 Subject: [PATCH 08/88] fix Signed-off-by: Cheng-Ping Hsieh --- nemo_skills/dataset/ruler2/prepare_mmlu.py | 2 +- nemo_skills/dataset/ruler2/prepare_niah.py | 2 +- nemo_skills/dataset/ruler2/prepare_qa.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/nemo_skills/dataset/ruler2/prepare_mmlu.py b/nemo_skills/dataset/ruler2/prepare_mmlu.py index 471cdb9a1e..4f69dfaf58 100644 --- a/nemo_skills/dataset/ruler2/prepare_mmlu.py +++ b/nemo_skills/dataset/ruler2/prepare_mmlu.py @@ -401,7 +401,7 @@ def generate_samples(max_seq_length: int, incremental: int = 10): def main(): - output_file = str(args.output_folder / "test.jsonl") + output_file = Path(args.output_folder) / "test.jsonl" write_jsons = generate_samples( max_seq_length=args.max_seq_length, diff --git a/nemo_skills/dataset/ruler2/prepare_niah.py b/nemo_skills/dataset/ruler2/prepare_niah.py index f0bbc083b6..5596582bf8 100644 --- a/nemo_skills/dataset/ruler2/prepare_niah.py +++ b/nemo_skills/dataset/ruler2/prepare_niah.py @@ -257,7 +257,7 @@ def generate_samples(num_samples: int, max_seq_length: int, incremental: int = 5 def main(): - output_file = str(args.output_folder / "test.jsonl") + output_file = Path(args.output_folder) / "test.jsonl" write_jsons = generate_samples( num_samples=args.num_samples, diff --git a/nemo_skills/dataset/ruler2/prepare_qa.py b/nemo_skills/dataset/ruler2/prepare_qa.py index 1ffaee0488..a72f2ff0c2 100644 --- a/nemo_skills/dataset/ruler2/prepare_qa.py +++ b/nemo_skills/dataset/ruler2/prepare_qa.py @@ -326,7 +326,7 @@ def generate_samples(num_samples: int, max_seq_length: int, incremental: int = 5 def main(): - output_file = str(args.output_folder / "test.jsonl") + output_file = Path(args.output_folder) / "test.jsonl" write_jsons = generate_samples( num_samples=args.num_samples, From c30d3f353d85c30c993c34885d16cf3cd3ba3cd4 Mon Sep 17 00:00:00 2001 From: Cheng-Ping Hsieh Date: Wed, 17 Sep 2025 16:51:31 -0700 Subject: [PATCH 09/88] fix Signed-off-by: Cheng-Ping Hsieh --- nemo_skills/dataset/ruler2/prepare.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nemo_skills/dataset/ruler2/prepare.py b/nemo_skills/dataset/ruler2/prepare.py index dc80ef2e73..8e46e9fe14 100644 --- a/nemo_skills/dataset/ruler2/prepare.py +++ b/nemo_skills/dataset/ruler2/prepare.py @@ -108,7 +108,7 @@ def prepare_mk_niah_hard(output_folder, tokenizer_type, tokenizer_path, length, def prepare_mv_niah_basic(output_folder, tokenizer_type, tokenizer_path, length, dataset_size): subprocess.run( - f"python -m prepare.py nemo_skills.dataset.ruler2.prepare_niah " + f"python -m nemo_skills.dataset.ruler2.prepare_niah " f"--output_folder {output_folder} " f"--tokenizer_type {tokenizer_type} " f"--tokenizer_path {tokenizer_path} " From 810f5f6250db5f967206e5e4007d06ac5f98a8a6 Mon Sep 17 00:00:00 2001 From: Cheng-Ping Hsieh Date: Wed, 17 Sep 2025 17:15:17 -0700 Subject: [PATCH 10/88] add ruler2 metrics Signed-off-by: Cheng-Ping Hsieh --- nemo_skills/evaluation/metrics/map_metrics.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nemo_skills/evaluation/metrics/map_metrics.py b/nemo_skills/evaluation/metrics/map_metrics.py index 34dd0192e6..83e2495b9f 100644 --- a/nemo_skills/evaluation/metrics/map_metrics.py +++ b/nemo_skills/evaluation/metrics/map_metrics.py @@ -54,6 +54,7 @@ "icpc": ICPCMetrics, "multichoice": MathMetrics, "ruler": RulerMetrics, + "ruler2": RulerMetrics, "livecodebench": LiveCodeBenchMetrics, "swe-bench": SweBenchMetrics, "scicode": SciCodeMetrics, From 4877ea89b7722b064cc60f0e15c11329cf7a81cf Mon Sep 17 00:00:00 2001 From: Cheng-Ping Hsieh Date: Wed, 17 Sep 2025 17:26:55 -0700 Subject: [PATCH 11/88] fix Signed-off-by: Cheng-Ping Hsieh --- nemo_skills/dataset/ruler2/prepare.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/nemo_skills/dataset/ruler2/prepare.py b/nemo_skills/dataset/ruler2/prepare.py index 8e46e9fe14..e09a8e4d7b 100644 --- a/nemo_skills/dataset/ruler2/prepare.py +++ b/nemo_skills/dataset/ruler2/prepare.py @@ -21,7 +21,7 @@ DEFAULT_SETTINGS = """ DATASET_GROUP = "long-context" -METRICS_TYPE = "ruler2" +METRICS_TYPE = "{metrics_type}" EVAL_ARGS = "{eval_args}" GENERATION_ARGS = ( "++prompt_config=generic/default " @@ -265,15 +265,19 @@ def prepare_task_for_ns(output_folder, task): Path(output_folder).mkdir(parents=True, exist_ok=True) with open(output_folder / "__init__.py", "w", encoding="utf-8") as init_file: if task in ["mk_niah_medium", "mk_niah_hard"]: + metrics_type = "multichoice" eval_args = "++eval_type=multichoice" elif task in ["mv_niah_medium"]: + metrics_type = "ruler2" eval_args = "++eval_type=ruler2 ++eval_config.match_type=2steps" elif "qa" in task: + metrics_type = "ruler2" eval_args = "++eval_type=ruler2 ++eval_config.match_type=part" else: + metrics_type = "ruler2" eval_args = "++eval_type=ruler2 ++eval_config.match_type=all" - init_file.write(DEFAULT_SETTINGS.format(eval_args=eval_args)) + init_file.write(DEFAULT_SETTINGS.format(metrics_type=metrics_type, eval_args=eval_args)) def prepare_dataset(tasks, setup, max_seq_length, tokenizer_type, tokenizer_path, dataset_size): prepare_task = { From 0de020f0262f911a475395f365b2673a09e97146 Mon Sep 17 00:00:00 2001 From: Cheng-Ping Hsieh Date: Wed, 17 Sep 2025 18:02:19 -0700 Subject: [PATCH 12/88] fix Signed-off-by: Cheng-Ping Hsieh --- nemo_skills/dataset/ruler2/ruler2_score.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nemo_skills/dataset/ruler2/ruler2_score.py b/nemo_skills/dataset/ruler2/ruler2_score.py index 7e34448bda..14d4ed6d97 100644 --- a/nemo_skills/dataset/ruler2/ruler2_score.py +++ b/nemo_skills/dataset/ruler2/ruler2_score.py @@ -36,7 +36,7 @@ def compute_score(metrics: dict): for aggregation in metrics[f"{setup}.mk_niah_basic"]: metrics[setup][aggregation] = { - "accuracy": sum(metrics[f"{setup}.{task}"][aggregation]["accuracy"] for task in tasks) / len(tasks) + "accuracy": sum(metrics[f"{setup}.{task}"][aggregation].get("accuracy", (metrics[f"{setup}.{task}"][aggregation].get("symbolic_correct", 0))) for task in tasks) / len(tasks) } return metrics From aeb325e0379b752645835a36702929638c35dfc9 Mon Sep 17 00:00:00 2001 From: Cheng-Ping Hsieh Date: Wed, 17 Sep 2025 18:35:34 -0700 Subject: [PATCH 13/88] add wer Signed-off-by: Cheng-Ping Hsieh --- nemo_skills/evaluation/evaluator/ruler.py | 26 +++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/nemo_skills/evaluation/evaluator/ruler.py b/nemo_skills/evaluation/evaluator/ruler.py index ec6802d998..5cf58fa765 100644 --- a/nemo_skills/evaluation/evaluator/ruler.py +++ b/nemo_skills/evaluation/evaluator/ruler.py @@ -16,6 +16,7 @@ import logging import os import re +import editdistance from tqdm import tqdm @@ -112,6 +113,27 @@ def post_process_preds(preds): preds = preds.split("Answer:")[-1] return preds + + def wer(hypotheses: list[str], references: list[str]) -> float: + scores = 0 + words = 0 + if len(hypotheses) != len(references): + raise ValueError( + "In word error rate calculation, hypotheses and reference" + " lists must have the same number of elements. But I got:" + "{0} and {1} correspondingly".format(len(hypotheses), len(references)) + ) + for h, r in zip(hypotheses, references): + h_list = h.split() + r_list = r.split() + words += len(r_list) + scores += editdistance.eval(h_list, r_list) + if words != 0: + wer = 1.0 * scores / words + else: + wer = float('inf') + return wer + def string_match_all_single(preds, refs): """the metric function with input (predictions: [str], references: [[str]]) to compute score.""" preds = post_process_preds(preds) @@ -128,7 +150,7 @@ def string_match_2steps_single(preds, refs): preds = [preds] refs = [refs] score = [ - sum([1.0 if r.lower() in pred.lower() else 0.0 for r in ref]) / len(ref) for pred, ref in zip(preds, refs) + sum([max(1.0 if r.lower() in pred.lower() else 0.0, 1 - wer([pred], [r])) for r in ref]) / len(ref) for pred, ref in zip(preds, refs) ][0] return score @@ -139,7 +161,7 @@ def string_match_part_single(preds, refs): preds = [preds] refs = [refs] score = [ - sum([max([1.0 if r.lower() in pred.lower() else 0.0 for r in ref]) for pred, ref in zip(preds, refs)]) + sum([max([max(1.0 if r.lower() in pred.lower() else 0.0, 1 - wer([pred], [r])) for r in ref]) for pred, ref in zip(preds, refs)]) ][0] return score From 39e08c753bc50271b4758707b2d91ca7e698e0b5 Mon Sep 17 00:00:00 2001 From: Wasi Ahmad Date: Thu, 18 Sep 2025 12:28:55 -0700 Subject: [PATCH 14/88] Evaluation on LiveBench-Coding (#821) Signed-off-by: wasiahmad Signed-off-by: Cheng-Ping Hsieh --- docs/evaluation/code.md | 9 +++++++++ nemo_skills/dataset/livebench-coding/__init__.py | 5 +++++ nemo_skills/evaluation/evaluator/__init__.py | 5 +++++ 3 files changed, 19 insertions(+) diff --git a/docs/evaluation/code.md b/docs/evaluation/code.md index 5a8d634bcc..1d128b63a9 100644 --- a/docs/evaluation/code.md +++ b/docs/evaluation/code.md @@ -340,11 +340,16 @@ Due to variance between runs, you can automatically repeat the evaluation and av ### bigcodebench +<<<<<<< HEAD - Benchmark is defined in [`nemo_skills/dataset/bigcodebench/__init__.py`](https://github.com/NVIDIA-NeMo/Skills/blob/main/nemo_skills/dataset/bigcodebench/__init__.py) +======= +- Benchmark is defined in [`nemo_skills/dataset/bigcodebench/__init__.py`](https://github.com/NVIDIA/NeMo-Skills/blob/main/nemo_skills/dataset/bigcodebench/__init__.py) +>>>>>>> 0b6e9d4b (Evaluation on LiveBench-Coding (#821)) - Original benchmark source is [here](https://github.com/bigcode-project/bigcodebench). ### livebench-coding +<<<<<<< HEAD - Benchmark is defined in [`nemo_skills/dataset/livebench-coding/__init__.py`](https://github.com/NVIDIA-NeMo/Skills/blob/main/nemo_skills/dataset/livebench-coding/__init__.py) - Original benchmark source is [here](https://huggingface.co/datasets/livebench/coding). @@ -352,3 +357,7 @@ Due to variance between runs, you can automatically repeat the evaluation and av - Benchmark is defined in [`nemo_skills/dataset/human-eval-infilling/__init__.py`](https://github.com/NVIDIA-NeMo/Skills/blob/main/nemo_skills/dataset/human-eval-infilling/__init__.py) - Original benchmark source is [here](https://github.com/openai/human-eval-infilling). +======= +- Benchmark is defined in [`nemo_skills/dataset/livebench-coding/__init__.py`](https://github.com/NVIDIA/NeMo-Skills/blob/main/nemo_skills/dataset/livebench-coding/__init__.py) +- Original benchmark source is [here](https://huggingface.co/datasets/livebench/coding). +>>>>>>> 0b6e9d4b (Evaluation on LiveBench-Coding (#821)) diff --git a/nemo_skills/dataset/livebench-coding/__init__.py b/nemo_skills/dataset/livebench-coding/__init__.py index 523301c188..57073fa2d1 100644 --- a/nemo_skills/dataset/livebench-coding/__init__.py +++ b/nemo_skills/dataset/livebench-coding/__init__.py @@ -16,4 +16,9 @@ DATASET_GROUP = "code" METRICS_TYPE = "livecodebench" EVAL_SPLIT = "test" +<<<<<<< HEAD GENERATION_ARGS = "++prompt_config=generic/default ++eval_type=livebench_coding" +======= +EVAL_ARGS = "++eval_type=livebench_coding" +GENERATION_ARGS = "++prompt_config=generic/default" +>>>>>>> 0b6e9d4b (Evaluation on LiveBench-Coding (#821)) diff --git a/nemo_skills/evaluation/evaluator/__init__.py b/nemo_skills/evaluation/evaluator/__init__.py index a6cd083c71..ac2c1e0d4c 100644 --- a/nemo_skills/evaluation/evaluator/__init__.py +++ b/nemo_skills/evaluation/evaluator/__init__.py @@ -21,8 +21,13 @@ CodeExecEvaluator, eval_bigcodebench, eval_evalplus, +<<<<<<< HEAD eval_human_eval_infilling, eval_livebench_coding, +======= + eval_livebench_coding, + eval_livecodebench, +>>>>>>> 0b6e9d4b (Evaluation on LiveBench-Coding (#821)) eval_livecodebench_pro, ) from nemo_skills.evaluation.evaluator.icpc import ICPCEvaluator From fe6d7071888922bcd2235165d1a50dd3b16808af Mon Sep 17 00:00:00 2001 From: Cheng-Ping Hsieh Date: Wed, 17 Sep 2025 16:21:36 -0700 Subject: [PATCH 15/88] add initial ruler2 Signed-off-by: Cheng-Ping Hsieh --- nemo_skills/dataset/ruler2/prepare_mmlu.py | 13 +++++++++++++ nemo_skills/dataset/ruler2/prepare_niah.py | 4 ++++ nemo_skills/dataset/ruler2/prepare_qa.py | 4 ++++ 3 files changed, 21 insertions(+) diff --git a/nemo_skills/dataset/ruler2/prepare_mmlu.py b/nemo_skills/dataset/ruler2/prepare_mmlu.py index 4f69dfaf58..6516987978 100644 --- a/nemo_skills/dataset/ruler2/prepare_mmlu.py +++ b/nemo_skills/dataset/ruler2/prepare_mmlu.py @@ -36,8 +36,13 @@ parser = argparse.ArgumentParser() # Basic Configurations parser.add_argument("--output_folder", type=str) +<<<<<<< HEAD parser.add_argument("--tokenizer_type", type=str, default='hf', help='[Options] nemo, hf, openai.') parser.add_argument("--tokenizer_path", type=str, required=True, help='path to the tokenizer model') +======= +parser.add_argument("--tokenizer_path", type=str, required=True, help='path to the tokenizer model') +parser.add_argument("--tokenizer_type", type=str, default='nemo', help='[Options] nemo, hf, openai.') +>>>>>>> fc830c89 (add initial ruler2) parser.add_argument("--max_seq_length", type=int, required=True, help='max sequence length including all input tokens and generated tokens.') parser.add_argument("--random_seed", type=int, default=42) parser.add_argument("--insert_position", type=float, default=-1, help='insert position of the true context in the context.') @@ -168,7 +173,11 @@ choices = d["choices"] item = { "Question": d['question'] + f'\nA. {choices[0]}\nB. {choices[1]}\nC. {choices[2]}\nD. {choices[3]}', +<<<<<<< HEAD "Solution": " " + f'\\boxed{{{options[d["answer"]]}}}', +======= + "Solution": " " + f'\\boxed{{{options[d["answer"]]}}}' +>>>>>>> fc830c89 (add initial ruler2) "Answer": options[d['answer']], } needle.append(item) @@ -177,7 +186,11 @@ choices = d["choices"] item = { "Question": d['question'] + f'\nA. {choices[0]}\nB. {choices[1]}\nC. {choices[2]}\nD. {choices[3]}', +<<<<<<< HEAD "Solution": " " + f'\\boxed{{{options[d["answer"]]}}}', +======= + "Solution": " " + f'\\boxed{{{options[d["answer"]]}}}' +>>>>>>> fc830c89 (add initial ruler2) "Answer": options[d['answer']], } haystack.append(item) diff --git a/nemo_skills/dataset/ruler2/prepare_niah.py b/nemo_skills/dataset/ruler2/prepare_niah.py index 5596582bf8..9c7c984bc0 100644 --- a/nemo_skills/dataset/ruler2/prepare_niah.py +++ b/nemo_skills/dataset/ruler2/prepare_niah.py @@ -39,7 +39,11 @@ parser = argparse.ArgumentParser() parser.add_argument("--output_folder", type=str) +<<<<<<< HEAD parser.add_argument("--tokenizer_type", type=str, default='hf', help='[Options] nemo, hf, openai.') +======= +parser.add_argument("--tokenizer_type", type=str, default='nemo', help='[Options] nemo, hf, openai.') +>>>>>>> fc830c89 (add initial ruler2) parser.add_argument("--tokenizer_path", type=str, required=True, help='path to the tokenizer model') parser.add_argument("--max_seq_length", type=int, required=True, help='max sequence length including all input tokens and generated tokens.') parser.add_argument("--num_samples", type=int, required=True, help='number of samples to generate') diff --git a/nemo_skills/dataset/ruler2/prepare_qa.py b/nemo_skills/dataset/ruler2/prepare_qa.py index a72f2ff0c2..9cfac0ada3 100644 --- a/nemo_skills/dataset/ruler2/prepare_qa.py +++ b/nemo_skills/dataset/ruler2/prepare_qa.py @@ -32,7 +32,11 @@ parser = argparse.ArgumentParser() # Basic Configurations parser.add_argument("--output_folder", type=str) +<<<<<<< HEAD parser.add_argument("--tokenizer_type", type=str, default='hf', help='[Options] nemo, hf, openai.') +======= +parser.add_argument("--tokenizer_type", type=str, default='nemo', help='[Options] nemo, hf, openai.') +>>>>>>> fc830c89 (add initial ruler2) parser.add_argument("--tokenizer_path", type=str, required=True, help='path to the tokenizer model') parser.add_argument("--max_seq_length", type=int, required=True, help='max sequence length including all input tokens and generated tokens.') parser.add_argument("--random_seed", type=int, default=42) From 7d2cc35d2d146fedaf3f35de41d0cb1088498f8c Mon Sep 17 00:00:00 2001 From: Cheng-Ping Hsieh Date: Wed, 8 Oct 2025 13:56:41 -0700 Subject: [PATCH 16/88] fix wer Signed-off-by: Cheng-Ping Hsieh --- nemo_skills/evaluation/evaluator/ruler.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nemo_skills/evaluation/evaluator/ruler.py b/nemo_skills/evaluation/evaluator/ruler.py index 5cf58fa765..2fc0379e9a 100644 --- a/nemo_skills/evaluation/evaluator/ruler.py +++ b/nemo_skills/evaluation/evaluator/ruler.py @@ -140,7 +140,7 @@ def string_match_all_single(preds, refs): preds = [preds] refs = [refs] score = [ - sum([1.0 if r.lower() in pred.lower() else 0.0 for r in ref]) / len(ref) for pred, ref in zip(preds, refs) + sum([max(1.0 if r.lower() in pred.lower() else 0.0, 1 - wer([pred.lower()], [r.lower()])) for r in ref]) / len(ref) for pred, ref in zip(preds, refs) ][0] return score @@ -150,7 +150,7 @@ def string_match_2steps_single(preds, refs): preds = [preds] refs = [refs] score = [ - sum([max(1.0 if r.lower() in pred.lower() else 0.0, 1 - wer([pred], [r])) for r in ref]) / len(ref) for pred, ref in zip(preds, refs) + sum([max(1.0 if r.lower() in pred.lower() else 0.0, 1 - wer([pred.lower()], [r.lower()])) for r in ref]) / len(ref) for pred, ref in zip(preds, refs) ][0] return score @@ -161,7 +161,7 @@ def string_match_part_single(preds, refs): preds = [preds] refs = [refs] score = [ - sum([max([max(1.0 if r.lower() in pred.lower() else 0.0, 1 - wer([pred], [r])) for r in ref]) for pred, ref in zip(preds, refs)]) + sum([max([max(1.0 if r.lower() in pred.lower() else 0.0, 1 - wer([pred.lower()], [r.lower()])) for r in ref]) for pred, ref in zip(preds, refs)]) ][0] return score From e63de3c28373e9aa3afb5ac0d3094307db2b13aa Mon Sep 17 00:00:00 2001 From: Cheng-Ping Hsieh Date: Wed, 8 Oct 2025 16:43:57 -0700 Subject: [PATCH 17/88] fix conflict Signed-off-by: Cheng-Ping Hsieh --- nemo_skills/dataset/ruler2/prepare_mmlu.py | 5 ----- nemo_skills/dataset/ruler2/prepare_niah.py | 4 ---- nemo_skills/dataset/ruler2/prepare_qa.py | 4 ---- 3 files changed, 13 deletions(-) diff --git a/nemo_skills/dataset/ruler2/prepare_mmlu.py b/nemo_skills/dataset/ruler2/prepare_mmlu.py index 6516987978..429ce5f642 100644 --- a/nemo_skills/dataset/ruler2/prepare_mmlu.py +++ b/nemo_skills/dataset/ruler2/prepare_mmlu.py @@ -36,13 +36,8 @@ parser = argparse.ArgumentParser() # Basic Configurations parser.add_argument("--output_folder", type=str) -<<<<<<< HEAD parser.add_argument("--tokenizer_type", type=str, default='hf', help='[Options] nemo, hf, openai.') parser.add_argument("--tokenizer_path", type=str, required=True, help='path to the tokenizer model') -======= -parser.add_argument("--tokenizer_path", type=str, required=True, help='path to the tokenizer model') -parser.add_argument("--tokenizer_type", type=str, default='nemo', help='[Options] nemo, hf, openai.') ->>>>>>> fc830c89 (add initial ruler2) parser.add_argument("--max_seq_length", type=int, required=True, help='max sequence length including all input tokens and generated tokens.') parser.add_argument("--random_seed", type=int, default=42) parser.add_argument("--insert_position", type=float, default=-1, help='insert position of the true context in the context.') diff --git a/nemo_skills/dataset/ruler2/prepare_niah.py b/nemo_skills/dataset/ruler2/prepare_niah.py index 9c7c984bc0..5596582bf8 100644 --- a/nemo_skills/dataset/ruler2/prepare_niah.py +++ b/nemo_skills/dataset/ruler2/prepare_niah.py @@ -39,11 +39,7 @@ parser = argparse.ArgumentParser() parser.add_argument("--output_folder", type=str) -<<<<<<< HEAD parser.add_argument("--tokenizer_type", type=str, default='hf', help='[Options] nemo, hf, openai.') -======= -parser.add_argument("--tokenizer_type", type=str, default='nemo', help='[Options] nemo, hf, openai.') ->>>>>>> fc830c89 (add initial ruler2) parser.add_argument("--tokenizer_path", type=str, required=True, help='path to the tokenizer model') parser.add_argument("--max_seq_length", type=int, required=True, help='max sequence length including all input tokens and generated tokens.') parser.add_argument("--num_samples", type=int, required=True, help='number of samples to generate') diff --git a/nemo_skills/dataset/ruler2/prepare_qa.py b/nemo_skills/dataset/ruler2/prepare_qa.py index 9cfac0ada3..a72f2ff0c2 100644 --- a/nemo_skills/dataset/ruler2/prepare_qa.py +++ b/nemo_skills/dataset/ruler2/prepare_qa.py @@ -32,11 +32,7 @@ parser = argparse.ArgumentParser() # Basic Configurations parser.add_argument("--output_folder", type=str) -<<<<<<< HEAD parser.add_argument("--tokenizer_type", type=str, default='hf', help='[Options] nemo, hf, openai.') -======= -parser.add_argument("--tokenizer_type", type=str, default='nemo', help='[Options] nemo, hf, openai.') ->>>>>>> fc830c89 (add initial ruler2) parser.add_argument("--tokenizer_path", type=str, required=True, help='path to the tokenizer model') parser.add_argument("--max_seq_length", type=int, required=True, help='max sequence length including all input tokens and generated tokens.') parser.add_argument("--random_seed", type=int, default=42) From d4f31eef469f894101c8c802dac5d97035ed0248 Mon Sep 17 00:00:00 2001 From: Cheng-Ping Hsieh Date: Wed, 8 Oct 2025 16:44:43 -0700 Subject: [PATCH 18/88] fix conflict Signed-off-by: Cheng-Ping Hsieh --- nemo_skills/dataset/ruler2/prepare_mmlu.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/nemo_skills/dataset/ruler2/prepare_mmlu.py b/nemo_skills/dataset/ruler2/prepare_mmlu.py index 429ce5f642..4f69dfaf58 100644 --- a/nemo_skills/dataset/ruler2/prepare_mmlu.py +++ b/nemo_skills/dataset/ruler2/prepare_mmlu.py @@ -168,11 +168,7 @@ choices = d["choices"] item = { "Question": d['question'] + f'\nA. {choices[0]}\nB. {choices[1]}\nC. {choices[2]}\nD. {choices[3]}', -<<<<<<< HEAD "Solution": " " + f'\\boxed{{{options[d["answer"]]}}}', -======= - "Solution": " " + f'\\boxed{{{options[d["answer"]]}}}' ->>>>>>> fc830c89 (add initial ruler2) "Answer": options[d['answer']], } needle.append(item) @@ -181,11 +177,7 @@ choices = d["choices"] item = { "Question": d['question'] + f'\nA. {choices[0]}\nB. {choices[1]}\nC. {choices[2]}\nD. {choices[3]}', -<<<<<<< HEAD "Solution": " " + f'\\boxed{{{options[d["answer"]]}}}', -======= - "Solution": " " + f'\\boxed{{{options[d["answer"]]}}}' ->>>>>>> fc830c89 (add initial ruler2) "Answer": options[d['answer']], } haystack.append(item) From 5189ccadb35d3521ed798075769e44982d067a86 Mon Sep 17 00:00:00 2001 From: Cheng-Ping Hsieh Date: Fri, 5 Dec 2025 07:26:30 -0800 Subject: [PATCH 19/88] resolve conflict Signed-off-by: Cheng-Ping Hsieh --- nemo_skills/dataset/ruler2/prepare.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nemo_skills/dataset/ruler2/prepare.py b/nemo_skills/dataset/ruler2/prepare.py index e09a8e4d7b..f10ae8d705 100644 --- a/nemo_skills/dataset/ruler2/prepare.py +++ b/nemo_skills/dataset/ruler2/prepare.py @@ -22,10 +22,10 @@ DEFAULT_SETTINGS = """ DATASET_GROUP = "long-context" METRICS_TYPE = "{metrics_type}" -EVAL_ARGS = "{eval_args}" GENERATION_ARGS = ( "++prompt_config=generic/default " -) + "{eval_args} " +} """ From 770d781dcfc2d981742d7e0d6e4b0b3841137308 Mon Sep 17 00:00:00 2001 From: Cheng-Ping Hsieh Date: Fri, 5 Dec 2025 07:28:30 -0800 Subject: [PATCH 20/88] resolve conflict Signed-off-by: Cheng-Ping Hsieh --- nemo_skills/evaluation/evaluator/__init__.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/nemo_skills/evaluation/evaluator/__init__.py b/nemo_skills/evaluation/evaluator/__init__.py index ac2c1e0d4c..a6cd083c71 100644 --- a/nemo_skills/evaluation/evaluator/__init__.py +++ b/nemo_skills/evaluation/evaluator/__init__.py @@ -21,13 +21,8 @@ CodeExecEvaluator, eval_bigcodebench, eval_evalplus, -<<<<<<< HEAD eval_human_eval_infilling, eval_livebench_coding, -======= - eval_livebench_coding, - eval_livecodebench, ->>>>>>> 0b6e9d4b (Evaluation on LiveBench-Coding (#821)) eval_livecodebench_pro, ) from nemo_skills.evaluation.evaluator.icpc import ICPCEvaluator From d73fcd081999a03054ea9a68dc1ca36a59cc6158 Mon Sep 17 00:00:00 2001 From: Cheng-Ping Hsieh Date: Fri, 5 Dec 2025 07:30:47 -0800 Subject: [PATCH 21/88] resolve conflict Signed-off-by: Cheng-Ping Hsieh --- nemo_skills/evaluation/evaluator/ruler.py | 41 +++++++++-------------- 1 file changed, 16 insertions(+), 25 deletions(-) diff --git a/nemo_skills/evaluation/evaluator/ruler.py b/nemo_skills/evaluation/evaluator/ruler.py index 2fc0379e9a..ffdc7913bb 100644 --- a/nemo_skills/evaluation/evaluator/ruler.py +++ b/nemo_skills/evaluation/evaluator/ruler.py @@ -67,7 +67,6 @@ def string_match_part_single(preds, refs): "part": string_match_part_single, } -<<<<<<< HEAD jsonl_file = eval_config.input_file with open(jsonl_file, "rt", encoding="utf-8") as fin: data = [json.loads(line) for line in fin] @@ -83,18 +82,6 @@ def string_match_part_single(preds, refs): fout.write(json.dumps(sample) + "\n") os.replace(jsonl_file + "-tmp", jsonl_file) -======= - for file in unroll_files(cfg.input_files): - with open(file, "rt", encoding="utf-8") as fin: - data = [json.loads(line) for line in fin] - with open(file, "wt", encoding="utf-8") as fout: - for sample in tqdm(data): - parse_result = parse_funcs[eval_config.parse_func](sample["generation"]) - sample["is_correct"] = match_type_funcs[eval_config.match_type]( - sample["generation"], sample["expected_answer"] - ) - sample["predicted_answer"] = parse_result - fout.write(json.dumps(sample) + "\n") def eval_ruler2(cfg): @@ -176,15 +163,19 @@ def string_match_part_single(preds, refs): "2steps": string_match_2steps_single, } - for file in unroll_files(cfg.input_files): - with open(file, "rt", encoding="utf-8") as fin: - data = [json.loads(line) for line in fin] - with open(file, "wt", encoding="utf-8") as fout: - for sample in tqdm(data): - parse_result = parse_funcs[eval_config.parse_func](sample["generation"]) - sample["is_correct"] = match_type_funcs[eval_config.match_type]( - sample["generation"], sample["expected_answer"] - ) - sample["predicted_answer"] = parse_result - fout.write(json.dumps(sample) + "\n") ->>>>>>> 2f7a3a33 (add initial ruler2) + + jsonl_file = eval_config.input_file + with open(jsonl_file, "rt", encoding="utf-8") as fin: + data = [json.loads(line) for line in fin] + for sample in tqdm(data): + parse_result = parse_funcs[eval_config.parse_func](sample["generation"]) + sample["is_correct"] = match_type_funcs[eval_config.match_type]( + sample["generation"], sample["expected_answer"] + ) + sample["predicted_answer"] = parse_result + + with open(jsonl_file + "-tmp", "wt", encoding="utf-8") as fout: + for sample in data: + fout.write(json.dumps(sample) + "\n") + + os.replace(jsonl_file + "-tmp", jsonl_file) From f16b8fb2554e7c90a6a44ce78badbc763d9ccdb6 Mon Sep 17 00:00:00 2001 From: Cheng-Ping Hsieh Date: Fri, 5 Dec 2025 07:49:43 -0800 Subject: [PATCH 22/88] fix Signed-off-by: Cheng-Ping Hsieh --- nemo_skills/dataset/ruler2/prepare.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nemo_skills/dataset/ruler2/prepare.py b/nemo_skills/dataset/ruler2/prepare.py index f10ae8d705..26f9825862 100644 --- a/nemo_skills/dataset/ruler2/prepare.py +++ b/nemo_skills/dataset/ruler2/prepare.py @@ -25,7 +25,7 @@ GENERATION_ARGS = ( "++prompt_config=generic/default " "{eval_args} " -} +) """ From d05b538503278a07051648edd960e015bc01aad9 Mon Sep 17 00:00:00 2001 From: Cheng-Ping Hsieh Date: Fri, 5 Dec 2025 12:00:24 -0800 Subject: [PATCH 23/88] fix Signed-off-by: Cheng-Ping Hsieh --- nemo_skills/evaluation/evaluator/ruler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nemo_skills/evaluation/evaluator/ruler.py b/nemo_skills/evaluation/evaluator/ruler.py index ffdc7913bb..13f6f74c85 100644 --- a/nemo_skills/evaluation/evaluator/ruler.py +++ b/nemo_skills/evaluation/evaluator/ruler.py @@ -152,7 +152,7 @@ def string_match_part_single(preds, refs): ][0] return score - eval_config = RulerEvaluatorConfig(**cfg.eval_config) + eval_config = RulerEvaluatorConfig(**cfg) parse_funcs = { "default": default_parse, From 44cf7fd8a832f92c86dbe03edbb742b3342a4058 Mon Sep 17 00:00:00 2001 From: Cheng-Ping Hsieh Date: Fri, 5 Dec 2025 12:54:43 -0800 Subject: [PATCH 24/88] fix conflict Signed-off-by: Cheng-Ping Hsieh --- docs/evaluation/code.md | 9 --------- nemo_skills/dataset/livebench-coding/__init__.py | 5 ----- 2 files changed, 14 deletions(-) diff --git a/docs/evaluation/code.md b/docs/evaluation/code.md index 1d128b63a9..5a8d634bcc 100644 --- a/docs/evaluation/code.md +++ b/docs/evaluation/code.md @@ -340,16 +340,11 @@ Due to variance between runs, you can automatically repeat the evaluation and av ### bigcodebench -<<<<<<< HEAD - Benchmark is defined in [`nemo_skills/dataset/bigcodebench/__init__.py`](https://github.com/NVIDIA-NeMo/Skills/blob/main/nemo_skills/dataset/bigcodebench/__init__.py) -======= -- Benchmark is defined in [`nemo_skills/dataset/bigcodebench/__init__.py`](https://github.com/NVIDIA/NeMo-Skills/blob/main/nemo_skills/dataset/bigcodebench/__init__.py) ->>>>>>> 0b6e9d4b (Evaluation on LiveBench-Coding (#821)) - Original benchmark source is [here](https://github.com/bigcode-project/bigcodebench). ### livebench-coding -<<<<<<< HEAD - Benchmark is defined in [`nemo_skills/dataset/livebench-coding/__init__.py`](https://github.com/NVIDIA-NeMo/Skills/blob/main/nemo_skills/dataset/livebench-coding/__init__.py) - Original benchmark source is [here](https://huggingface.co/datasets/livebench/coding). @@ -357,7 +352,3 @@ Due to variance between runs, you can automatically repeat the evaluation and av - Benchmark is defined in [`nemo_skills/dataset/human-eval-infilling/__init__.py`](https://github.com/NVIDIA-NeMo/Skills/blob/main/nemo_skills/dataset/human-eval-infilling/__init__.py) - Original benchmark source is [here](https://github.com/openai/human-eval-infilling). -======= -- Benchmark is defined in [`nemo_skills/dataset/livebench-coding/__init__.py`](https://github.com/NVIDIA/NeMo-Skills/blob/main/nemo_skills/dataset/livebench-coding/__init__.py) -- Original benchmark source is [here](https://huggingface.co/datasets/livebench/coding). ->>>>>>> 0b6e9d4b (Evaluation on LiveBench-Coding (#821)) diff --git a/nemo_skills/dataset/livebench-coding/__init__.py b/nemo_skills/dataset/livebench-coding/__init__.py index 57073fa2d1..523301c188 100644 --- a/nemo_skills/dataset/livebench-coding/__init__.py +++ b/nemo_skills/dataset/livebench-coding/__init__.py @@ -16,9 +16,4 @@ DATASET_GROUP = "code" METRICS_TYPE = "livecodebench" EVAL_SPLIT = "test" -<<<<<<< HEAD GENERATION_ARGS = "++prompt_config=generic/default ++eval_type=livebench_coding" -======= -EVAL_ARGS = "++eval_type=livebench_coding" -GENERATION_ARGS = "++prompt_config=generic/default" ->>>>>>> 0b6e9d4b (Evaluation on LiveBench-Coding (#821)) From 24b62739a93b0348baf4519bff77f8b35c2ddf04 Mon Sep 17 00:00:00 2001 From: Cheng-Ping Hsieh Date: Fri, 5 Dec 2025 15:57:44 -0800 Subject: [PATCH 25/88] rulerv1 reasoning Signed-off-by: Cheng-Ping Hsieh --- nemo_skills/dataset/ruler/prepare.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/nemo_skills/dataset/ruler/prepare.py b/nemo_skills/dataset/ruler/prepare.py index 2a1ffe6df1..6db25721fe 100644 --- a/nemo_skills/dataset/ruler/prepare.py +++ b/nemo_skills/dataset/ruler/prepare.py @@ -27,10 +27,10 @@ METRICS_TYPE = "ruler" GENERATION_ARGS = ( "++prompt_config=generic/default " - "++inference.tokens_to_generate={tokens_to_generate} " + # "++inference.tokens_to_generate={tokens_to_generate} " # ruler is adding prefix for assistant response, so it has to go through completions api - "++start_assistant_response_key=generation " - "++inference.endpoint_type=text " + # "++start_assistant_response_key=generation " + # "++inference.endpoint_type=text " "++eval_type=ruler ++eval_config.match_type={match_type} " ) """ @@ -48,10 +48,10 @@ def prepare_task_for_ns(task, data_dir, setup): original_entry = json.loads(line) new_entry = { "index": original_entry["index"], - "question": original_entry["input"], + "question": original_entry["input"] + original_entry["answer_prefix"], "expected_answer": original_entry["outputs"], "length": original_entry["length"], - "generation": original_entry["answer_prefix"].strip(), + # "generation": original_entry["answer_prefix"].strip(), } fout.write(json.dumps(new_entry) + "\n") @@ -60,7 +60,7 @@ def prepare_task_for_ns(task, data_dir, setup): init_file.write( DEFAULT_SETTINGS.format( match_type=MATCH_TYPE[short_name], - tokens_to_generate=TOKENS_TO_GENERATE[short_name], + # tokens_to_generate=TOKENS_TO_GENERATE[short_name], ) ) From 0d78226de70d886569ca7ad1688ba74601455c4b Mon Sep 17 00:00:00 2001 From: Cheng-Ping Hsieh Date: Fri, 5 Dec 2025 17:04:33 -0800 Subject: [PATCH 26/88] save Signed-off-by: Cheng-Ping Hsieh --- nemo_skills/dataset/ruler/prepare.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/nemo_skills/dataset/ruler/prepare.py b/nemo_skills/dataset/ruler/prepare.py index 6db25721fe..80320b3035 100644 --- a/nemo_skills/dataset/ruler/prepare.py +++ b/nemo_skills/dataset/ruler/prepare.py @@ -27,10 +27,6 @@ METRICS_TYPE = "ruler" GENERATION_ARGS = ( "++prompt_config=generic/default " - # "++inference.tokens_to_generate={tokens_to_generate} " - # ruler is adding prefix for assistant response, so it has to go through completions api - # "++start_assistant_response_key=generation " - # "++inference.endpoint_type=text " "++eval_type=ruler ++eval_config.match_type={match_type} " ) """ @@ -115,7 +111,7 @@ def prepare_task(task): subprocess.run( f"python prepare.py --save_dir {tmpdirname}/ruler_data --benchmark synthetic " f" --subset test --task {task} --tokenizer_type hf --model_template_type base --prepare_for_ns " - f" --num_samples 500 --max_seq_length {max_seq_length} {ruler_prepare_args}", + f" --num_samples 100 --max_seq_length {max_seq_length} {ruler_prepare_args}", shell=True, check=True, cwd=Path(tmpdirname) / "RULER" / "scripts" / "data", From cac7836b49ab5c30adb51bb783cde55b75997617 Mon Sep 17 00:00:00 2001 From: Cheng-Ping Hsieh Date: Fri, 5 Dec 2025 19:20:07 -0800 Subject: [PATCH 27/88] remove prefix Signed-off-by: Cheng-Ping Hsieh --- nemo_skills/dataset/ruler/prepare.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nemo_skills/dataset/ruler/prepare.py b/nemo_skills/dataset/ruler/prepare.py index 80320b3035..ad6aadb72a 100644 --- a/nemo_skills/dataset/ruler/prepare.py +++ b/nemo_skills/dataset/ruler/prepare.py @@ -44,7 +44,7 @@ def prepare_task_for_ns(task, data_dir, setup): original_entry = json.loads(line) new_entry = { "index": original_entry["index"], - "question": original_entry["input"] + original_entry["answer_prefix"], + "question": original_entry["input"], "expected_answer": original_entry["outputs"], "length": original_entry["length"], # "generation": original_entry["answer_prefix"].strip(), From 413ec3b1d24a3584007aa5200cb86417792c264e Mon Sep 17 00:00:00 2001 From: Cheng-Ping Hsieh Date: Fri, 12 Dec 2025 13:03:36 -0800 Subject: [PATCH 28/88] revert rulerv1 Signed-off-by: Cheng-Ping Hsieh --- nemo_skills/dataset/ruler/prepare.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/nemo_skills/dataset/ruler/prepare.py b/nemo_skills/dataset/ruler/prepare.py index ad6aadb72a..71e4a5872a 100644 --- a/nemo_skills/dataset/ruler/prepare.py +++ b/nemo_skills/dataset/ruler/prepare.py @@ -27,6 +27,10 @@ METRICS_TYPE = "ruler" GENERATION_ARGS = ( "++prompt_config=generic/default " + "++inference.tokens_to_generate={tokens_to_generate} " + # ruler is adding prefix for assistant response, so it has to go through completions api + "++start_assistant_response_key=generation " + "++inference.endpoint_type=text " "++eval_type=ruler ++eval_config.match_type={match_type} " ) """ @@ -47,7 +51,7 @@ def prepare_task_for_ns(task, data_dir, setup): "question": original_entry["input"], "expected_answer": original_entry["outputs"], "length": original_entry["length"], - # "generation": original_entry["answer_prefix"].strip(), + "generation": original_entry["answer_prefix"].strip(), } fout.write(json.dumps(new_entry) + "\n") @@ -56,7 +60,7 @@ def prepare_task_for_ns(task, data_dir, setup): init_file.write( DEFAULT_SETTINGS.format( match_type=MATCH_TYPE[short_name], - # tokens_to_generate=TOKENS_TO_GENERATE[short_name], + tokens_to_generate=TOKENS_TO_GENERATE[short_name], ) ) @@ -111,7 +115,7 @@ def prepare_task(task): subprocess.run( f"python prepare.py --save_dir {tmpdirname}/ruler_data --benchmark synthetic " f" --subset test --task {task} --tokenizer_type hf --model_template_type base --prepare_for_ns " - f" --num_samples 100 --max_seq_length {max_seq_length} {ruler_prepare_args}", + f" --num_samples 500 --max_seq_length {max_seq_length} {ruler_prepare_args}", shell=True, check=True, cwd=Path(tmpdirname) / "RULER" / "scripts" / "data", @@ -205,4 +209,4 @@ def prepare_task(task): ruler_prepare_args, tmp_data_dir=args.tmp_data_dir, ) - print("RULER dataset preparation completed.") + print("RULER dataset preparation completed.") \ No newline at end of file From 2c6b9e32bbb1c40d822f6f8a79b7ef317ce43310 Mon Sep 17 00:00:00 2001 From: Cheng-Ping Hsieh Date: Fri, 12 Dec 2025 13:06:28 -0800 Subject: [PATCH 29/88] remove pred postprocess Signed-off-by: Cheng-Ping Hsieh --- nemo_skills/evaluation/evaluator/ruler.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/nemo_skills/evaluation/evaluator/ruler.py b/nemo_skills/evaluation/evaluator/ruler.py index 13f6f74c85..e3e9e1173b 100644 --- a/nemo_skills/evaluation/evaluator/ruler.py +++ b/nemo_skills/evaluation/evaluator/ruler.py @@ -93,11 +93,6 @@ def default_parse(prediction): return pp_predict def post_process_preds(preds): - if "" in preds: - preds = preds.split("")[-1] - - if "Answer:" in preds: - preds = preds.split("Answer:")[-1] return preds From 6ae14d9131f3d92498e8c0ecf917bd294d745cdd Mon Sep 17 00:00:00 2001 From: Ivan Date: Mon, 8 Dec 2025 22:12:26 +0500 Subject: [PATCH 30/88] Add apex-shortlist dataset (#1080) Signed-off-by: i-vainn Co-authored-by: George Armstrong Signed-off-by: Cheng-Ping Hsieh --- .../dataset/apex-shortlist/__init__.py | 18 ++++++++++ nemo_skills/dataset/apex-shortlist/prepare.py | 35 +++++++++++++++++++ 2 files changed, 53 insertions(+) create mode 100644 nemo_skills/dataset/apex-shortlist/__init__.py create mode 100644 nemo_skills/dataset/apex-shortlist/prepare.py diff --git a/nemo_skills/dataset/apex-shortlist/__init__.py b/nemo_skills/dataset/apex-shortlist/__init__.py new file mode 100644 index 0000000000..22521d5317 --- /dev/null +++ b/nemo_skills/dataset/apex-shortlist/__init__.py @@ -0,0 +1,18 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# settings that define how evaluation should be done by default (all can be changed from cmdline) +DATASET_GROUP = "math" +METRICS_TYPE = "math" +GENERATION_ARGS = "++prompt_config=generic/math ++eval_type=math" diff --git a/nemo_skills/dataset/apex-shortlist/prepare.py b/nemo_skills/dataset/apex-shortlist/prepare.py new file mode 100644 index 0000000000..ffccd25e54 --- /dev/null +++ b/nemo_skills/dataset/apex-shortlist/prepare.py @@ -0,0 +1,35 @@ +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +from pathlib import Path + +from datasets import load_dataset +from tqdm import tqdm + + +def write_data_to_file(output_file, data): + with open(output_file, "wt", encoding="utf-8") as fout: + for entry in tqdm(data, desc=f"Writing {output_file.name}"): + entry["expected_answer"] = entry.pop("answer") + json.dump(entry, fout) + fout.write("\n") + + +if __name__ == "__main__": + dataset = load_dataset("MathArena/apex-shortlist")["train"] + data_dir = Path(__file__).absolute().parent + data_dir.mkdir(exist_ok=True) + output_file = data_dir / "test.jsonl" + write_data_to_file(output_file, dataset) From 8a103ab171fe588d2f10254d78cffada3dbb1d4a Mon Sep 17 00:00:00 2001 From: Wojciech Prazuch Date: Tue, 9 Dec 2025 18:25:13 +0100 Subject: [PATCH 31/88] Introduce regex for small differences of formatting from judge (#1082) Signed-off-by: George Armstrong Co-authored-by: George Armstrong Signed-off-by: Cheng-Ping Hsieh --- nemo_skills/evaluation/metrics/utils.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/nemo_skills/evaluation/metrics/utils.py b/nemo_skills/evaluation/metrics/utils.py index d23f483a55..f5f804d5b7 100644 --- a/nemo_skills/evaluation/metrics/utils.py +++ b/nemo_skills/evaluation/metrics/utils.py @@ -13,6 +13,7 @@ import json import logging +import re from typing import Union from nemo_skills.utils import get_logger_name @@ -34,8 +35,10 @@ def read_predictions(predictions, line_idx, file_handles): def is_correct_judgement(judgement, return_none=False) -> Union[bool, None]: - if "Judgement:" in judgement: - verdict = judgement.split("Judgement:")[-1].strip() + # Match both plain "Judgement:" and markdown bold "**Judgement**:" formats, this happens for gpt-4o which is AA Judge model. + match = re.search(r"\*{0,2}Judgement\*{0,2}\s*:", judgement, re.IGNORECASE) + if match: + verdict = judgement[match.end() :].strip() if verdict.lower().startswith("yes"): return True elif verdict.lower().startswith("no"): From 25e53b4d4fdcb9c4a297b497c18d7e9df6746c4c Mon Sep 17 00:00:00 2001 From: gnalbandyan <153070076+gnalbandyan@users.noreply.github.com> Date: Tue, 9 Dec 2025 22:47:13 +0400 Subject: [PATCH 32/88] Add LCB Prompts, fix regex bug in robust_eval, remove CR, make summarize_robustness generic for more benchmarks, update docstrings. (#1079) Signed-off-by: Grigor Nalbandyan Signed-off-by: Cheng-Ping Hsieh --- docs/evaluation/robustness.md | 33 +++---- nemo_skills/evaluation/evaluator/mcq.py | 2 +- nemo_skills/evaluation/math_grader.py | 4 +- nemo_skills/pipeline/robust_eval.py | 7 +- nemo_skills/pipeline/summarize_robustness.py | 98 +++++-------------- .../robustness/code_prompts/aai_prompt.yaml | 10 ++ .../robustness/code_prompts/code_1.yaml | 3 + .../robustness/code_prompts/code_2.yaml | 6 ++ .../robustness/code_prompts/code_3.yaml | 8 ++ .../robustness/code_prompts/code_4.yaml | 19 ++++ .../code_prompts/ns_gen_codegen.yaml | 12 +++ .../code_prompts/ns_python_codegen.yaml | 7 ++ .../config/robustness/prompt_set_config.yaml | 10 ++ 13 files changed, 121 insertions(+), 98 deletions(-) create mode 100644 nemo_skills/prompt/config/robustness/code_prompts/aai_prompt.yaml create mode 100644 nemo_skills/prompt/config/robustness/code_prompts/code_1.yaml create mode 100644 nemo_skills/prompt/config/robustness/code_prompts/code_2.yaml create mode 100644 nemo_skills/prompt/config/robustness/code_prompts/code_3.yaml create mode 100644 nemo_skills/prompt/config/robustness/code_prompts/code_4.yaml create mode 100644 nemo_skills/prompt/config/robustness/code_prompts/ns_gen_codegen.yaml create mode 100644 nemo_skills/prompt/config/robustness/code_prompts/ns_python_codegen.yaml diff --git a/docs/evaluation/robustness.md b/docs/evaluation/robustness.md index a06de02842..c4c6a9fee1 100644 --- a/docs/evaluation/robustness.md +++ b/docs/evaluation/robustness.md @@ -70,12 +70,10 @@ The following metrics are calculated: * **Aggregated Benchmark Statistics**: For each benchmark across all prompts and seeds, the script calculates: - `min`, `max`, `avg`, `std`: Statistical metrics across all runs per benchmark. - - `CR` (Consistency Rate): The average rate of agreement of model predictions on the same datapoint across different runs. - `prompt_sensitivity`: The standard deviation of the average scores across different prompts, which measures how sensitive the model's accuracy is to prompt variations. * **Per-Prompt Statistics**: For each prompt across all random seeds, the script calculates: - `min`, `max`, `avg`, `std`: Statistical metrics for a single prompt across seeds. - - `CR` (Consistency Rate): The average rate of agreement of model predictions on the same question across different runs. - `no_answer`: The proportion of questions for which no answer was extracted from the generation, either due to a wrong answer format or no answer at all (can be used to find prompts that break the model predictions). @@ -84,34 +82,29 @@ First, for each benchmark, metrics are aggregated across all prompts and seeds. All calculated metrics are also saved to `output_dir/metrics.json`.
``` -dataset | min | max | avg | std | CR | prompt_sensitivity -------------------------------------------------------------------------------------------- -comp-math-24-25@80 | 48.05 | 53.91 | 51.10 | 1.60 | 55.25 | 0.34 -gpqa@80 | 50.51 | 60.61 | 55.51 | 2.44 | 65.15 | 0.77 +dataset | min | max | avg | std | prompt_sensitivity +---------------------------------------------------------------------------------- +comp-math-24-25@80 | 48.05 | 53.91 | 51.10 | 1.60 | 0.34 +gpqa@80 | 50.51 | 60.61 | 55.51 | 2.44 | 0.77 -------------------------------------- comp-math-24-25 ------------------------------------- -prompt@8 | min | max | avg | std | CR | no_answer +------------------------------------- comp-math-24-25 ---------------------------- +prompt@8 | min | max | avg | std | no_answer ---------------------------------------------------------------------------------- -prompt_1 | 48.05 | 53.91 | 50.76 | 1.61 | 55.48 | 1.56 +prompt_1 | 48.05 | 53.91 | 50.76 | 1.61 | 1.56 ... -prompt_10 | 48.44 | 53.91 | 51.44 | 1.52 | 55.13 | 1.66 +prompt_10 | 48.44 | 53.91 | 51.44 | 1.52 | 1.66 -------------------------------------- gpqa -------------------------------------- -prompt@8 | min | max | avg | std | CR | no_answer +prompt@8 | min | max | avg | std | no_answer ---------------------------------------------------------------------------------- -prompt_1 | 50.51 | 60.61 | 54.73 | 2.68 | 64.20 | 3.03 +prompt_1 | 50.51 | 60.61 | 54.73 | 2.68 | 3.03 ... -prompt_10 | 53.54 | 60.10 | 56.28 | 1.88 | 66.59 | 2.78 +prompt_10 | 53.54 | 60.10 | 56.28 | 1.88 | 2.78 ``` -### Consistency Rate -For each datapoint, collect all predictions and calculate the similarity between all possible pairs of predictions. -The consistency rate is the number of pairs of equivalent prediction pairs divided by the total number of prediction pairs (N choose 2).
-Example: For a datapoint with predictions [A, A, C] across 3 files, it will compare pairs (A, A), (A, C), and (A, C), and the consistency rate will be 1/3 = 33.33%.
-Consistency rate is proposed in [Improving the Robustness of Large Language Models via Consistency Alignment](https://arxiv.org/abs/2403.14221). ## Notes on Usage -- There are 10 Math and 10 MCQ prompts in the `prompt/config/robustness` folder, along with the prompt_set_config.yaml. Those prompts vary by prompt wording and problem placement. MCQ prompts also vary by answer formatting instruction, while Math prompts use only \boxed{} format. These prompts can be used with any Math (AIME, comp-math-24-25, etc) and MCQ (GPQA, MMLU-Pro, etc) benchmarks. -- robust_eval can be used with any dataset that Nemo-Skills supports, but summarize_robustness works on Math and MCQ datasets (for now). If you need evaluations on multiple prompts, you can still use robust_eval. However, the `summarize_robustness` part won't work. +- There are 10 Math, 10 MCQ and 7 LiveCodeBench prompts in the `prompt/config/robustness` folder, along with the prompt_set_config.yaml. Those prompts vary by prompt wording and problem placement. MCQ prompts also vary by answer formatting instruction, while Math prompts use only \boxed{} format. `prompt/config/robustness/math_prompts` can be used for any Math (AIME, comp-math-24-25, etc) benchmarks, `prompt/config/robustness/mcq_prompts` for any MCQ (GPQA, MMLU-Pro, etc) benchmarks. +- robust_eval can be used with any dataset that Nemo-Skills supports, but summarize_robustness works on Math, MCQ, LiveCodeBench datasets and any dataset with judge evaluation (for now). If you need evaluations on multiple prompts, you can still use robust_eval. However, the `summarize_robustness` part won't work. diff --git a/nemo_skills/evaluation/evaluator/mcq.py b/nemo_skills/evaluation/evaluator/mcq.py index 2b0eeea10c..821f1a47f8 100644 --- a/nemo_skills/evaluation/evaluator/mcq.py +++ b/nemo_skills/evaluation/evaluator/mcq.py @@ -31,7 +31,7 @@ class MCQEvaluatorConfig(BaseEvaluatorConfig): # only used if extract_from_boxed is False extract_regex: str = r"The final answer is (.+)$" # if relaxed is True: - # extract from boxed FIRST, if not found, extract from regex + # extract from regex FIRST, if not found, extract from boxed # if relaxed is False: # if extract_from_boxed is True -> extract from boxed{} ONLY # else extract from regex ONLY diff --git a/nemo_skills/evaluation/math_grader.py b/nemo_skills/evaluation/math_grader.py index dd75529c4b..4000265374 100644 --- a/nemo_skills/evaluation/math_grader.py +++ b/nemo_skills/evaluation/math_grader.py @@ -103,11 +103,11 @@ def extract_answer( string: str, extract_from_boxed: bool = True, extract_regex: str = r"The final answer is (.+)$", relaxed=False ): """Extract Answer String from \\boxed expression or based on regex - If relaxed=True: try both methods, boxed first. + If relaxed=True: try both methods, regex first. If relaxed=False: use only one method based on extract_from_boxed flag. """ if relaxed: - return search_boxed(string) or search_regex(string, extract_regex) + return search_regex(string, extract_regex) or search_boxed(string) if extract_from_boxed: return search_boxed(string) diff --git a/nemo_skills/pipeline/robust_eval.py b/nemo_skills/pipeline/robust_eval.py index 0ea2f820b1..1959f12ae7 100644 --- a/nemo_skills/pipeline/robust_eval.py +++ b/nemo_skills/pipeline/robust_eval.py @@ -13,6 +13,7 @@ # limitations under the License. import inspect import logging +import shlex from copy import deepcopy from dataclasses import dataclass from pathlib import Path @@ -115,7 +116,11 @@ def robust_eval( prompt_context = deepcopy(ctx) prompt = PromptConfig(**prompt) if prompt.extract_regex: - prompt_context.args.append(f"++eval_config.extract_regex='\"{prompt.extract_regex}\"'") + hydra_arg = f"++eval_config.extract_regex='{prompt.extract_regex}'" + # Quote properly for it to be correct passed to ns eval in terminal + shell_safe_arg = shlex.quote(shlex.quote(hydra_arg)) + prompt_context.args.append(shell_safe_arg) + prompt_context.args.append(f"++prompt_config={prompt.prompt_config}") prompt_kwargs = deepcopy(ns_eval_kwargs) diff --git a/nemo_skills/pipeline/summarize_robustness.py b/nemo_skills/pipeline/summarize_robustness.py index bc7828b7d1..d36d4bc184 100644 --- a/nemo_skills/pipeline/summarize_robustness.py +++ b/nemo_skills/pipeline/summarize_robustness.py @@ -17,8 +17,6 @@ import logging import os import tempfile -from collections import defaultdict -from itertools import combinations from pathlib import Path from typing import List, Optional @@ -58,7 +56,7 @@ def get_metrics(prediction_files: List[str]) -> List[float] | List[float]: per_file_metrics = [] no_answer = [] for pred_file in prediction_files: - metrics_calculator = ComputeMetrics(benchmark="custom", metric_type="math", max_samples=-1) + metrics_calculator = ComputeMetrics(benchmark=Path(pred_file).parent.name, max_samples=-1) metrics_calculator.calculator = metrics_calculator.get_metrics_calculator() with open(pred_file, "rt", encoding="utf-8") as f: @@ -66,60 +64,18 @@ def get_metrics(prediction_files: List[str]) -> List[float] | List[float]: data = read_predictions([line], idx, [f]) metrics_calculator.calculator.update(data) metrics = metrics_calculator.calculator.get_metrics() - per_file_metrics.append(metrics["pass@1"]["symbolic_correct"]) - no_answer.append(metrics["pass@1"]["no_answer"]) + for acc_key in ["judge_correct", "symbolic_correct", "accuracy"]: + if acc_key in metrics["pass@1"]: + per_file_metrics.append(metrics["pass@1"][acc_key]) + break + else: + LOG.warning(f"Could not find accuracy metric in {pred_file}, setting to -1.") + per_file_metrics.append(-1) + no_answer.append(metrics["pass@1"].get("no_answer", -1)) return per_file_metrics, no_answer -def calculate_similarity(answer1: str | None, answer2: str | None) -> float: - if answer1 is None and answer2 is None: - return 0 - return 1 if answer1 == answer2 else 0 - - -def calculate_consistency_rate(input_files: List[str]) -> float: - """Calculate the consistency rate across multiple input files. - Metric proposed in https://arxiv.org/abs/2403.14221 - - Args: - input_files: List of file paths containing predictions - - Returns: - float: Average consistency rate as a percentage (0-100) - - For each datapoint, collect all predictions, and - calculate similarity between all possible pairs of predictions. - The consistency rate is the number of pairs of equivalent prediction pairs - divided by the total number of prediction pairs (N choose 2). - - Example: - If datapoint i has predictions [A, A, C] across 3 files, it will - compare pairs (A,A), (A, C) and (A, C) and consistency rate will be 1/3 = 33.33%. - - """ - per_idx_preds = defaultdict(list) - for inp_f in input_files: - with open(inp_f, "rt", encoding="utf-8") as f: - for idx, line in enumerate(f): - data = read_predictions([line], idx, [f]) - per_idx_preds[idx].append(data[0]["predicted_answer"]) - responses = per_idx_preds.values() - total_similarity = 0 - total_combinations = 0 - - for response_set in responses: - if len(response_set) < 2: - continue - for answer1, answer2 in combinations(response_set, 2): - total_similarity += calculate_similarity(answer1, answer2) - total_combinations += 1 - - if total_combinations == 0: - return 100.0 - return round(total_similarity / total_combinations * 100, 2) - - @app.command() @typer_unpacker def summarize_robustness( @@ -168,7 +124,6 @@ def summarize_robustness( Calculates the following both per benchmark across prompts and per prompt across random seeds: - Statistical metrics: min, max, average, standard deviation - - Consistency Rate (CR): Agreement between different model runs - No-answer rate: Proportion of questions without answers - Cross-prompt standard deviation of averages @@ -216,8 +171,10 @@ def summarize_robustness( if benchmarks_paths: # Ascertain that the benchmarks_paths are valid for benchmark_path in benchmarks_paths: - # Valid benchmark_path should contain output*jsonl files - if len(glob.glob(f"{benchmark_path}/**/output*jsonl", recursive=True)) == 0: + # Valid benchmark_path should contain output*jsonl files excluding output.jsonl and chunked files + pred_files = glob.glob(f"{benchmark_path}/**/eval-results/*/output*jsonl", recursive=True) + pred_files = [f for f in pred_files if Path(f).name != "output.jsonl" and "_chunk_" not in Path(f).name] + if len(pred_files) == 0: raise ValueError(f"The benchmark directory {benchmark_path} lacks output*jsonl files.") else: print(f"No benchmarks found in {results_dir}") @@ -227,15 +184,13 @@ def summarize_robustness( print("Calculating robustness metrics for benchmarks found:", benchmarks_paths) for benchmark_path in sorted(benchmarks_paths): # sorting to ensure consistent order benchmark = str(Path(benchmark_path).name) - if not Path(benchmark_path).is_dir(): - continue - metrics_to_print[benchmark] = dict() # calculate metrics per prompt all_eval_metrics = [] for prompt_dir in sorted(glob.glob(f"{benchmark_path}/*")): prompt_name = str(Path(prompt_dir).name) - input_files = glob.glob(f"{prompt_dir}/**/output-rs*.jsonl", recursive=True) + input_files = glob.glob(f"{prompt_dir}/**/eval-results/*/output-rs*.jsonl", recursive=True) + input_files = [f for f in input_files if Path(f).name != "output.jsonl" and "_chunk_" not in Path(f).name] if not input_files: print("No input files found for prompt", prompt_dir) continue @@ -249,9 +204,6 @@ def summarize_robustness( "num_seeds": len(per_file_metrics), } all_eval_metrics.extend(per_file_metrics) - # calculate consistency rate per prompt - consistency_rate = calculate_consistency_rate(input_files) - metrics_to_print[benchmark][prompt_name]["CR"] = consistency_rate # calculate metrics across all prompts and seeds metrics_to_print[benchmark]["aggregated"] = { @@ -262,16 +214,13 @@ def summarize_robustness( "num_runs": len(all_eval_metrics), } - input_files = glob.glob(f"{benchmark_path}/**/output-rs*.jsonl", recursive=True) - consistency_rate = calculate_consistency_rate(input_files) - metrics_to_print[benchmark]["aggregated"]["CR"] = consistency_rate - # calculate the std of prompt averages for benchmark, metrics in metrics_to_print.items(): prompt_avgs = [m["avg"] for k, m in metrics.items() if k != "aggregated"] - metrics_to_print[benchmark]["aggregated"]["prompt_sensitivity"] = np.std(prompt_avgs) + prompt_std = np.std(prompt_avgs) + metrics_to_print[benchmark]["aggregated"]["prompt_sensitivity"] = prompt_std - header_fields = ["min", "max", "avg", "std", "CR", "prompt_sensitivity"] + header_fields = ["min", "max", "avg", "std", "prompt_sensitivity"] header = f"{'dataset':<20} | " header += " | ".join(f"{stat}".center(7) for stat in header_fields) print(header) @@ -279,14 +228,15 @@ def summarize_robustness( # Print aggregated stats for benchmark in metrics_to_print.keys(): bench_runs = f"{benchmark}@{metrics_to_print[benchmark]['aggregated']['num_runs']}" - row = f"{bench_runs:<20} | " + " | ".join( - f"{metrics_to_print[benchmark]['aggregated'][stat]:.2f}".center(7) for stat in header_fields - ) - print(row) + row = f"{bench_runs:<20} | " + for stat in header_fields: + value = metrics_to_print[benchmark]["aggregated"][stat] + row += f"{value:.2f}".center(max(len(stat), 7)) + " | " + print(row[:-3]) print("\n") # Print stats per prompt - header_fields = ["min", "max", "avg", "std", "CR", "no_answer"] + header_fields = ["min", "max", "avg", "std", "no_answer"] for benchmark, metrics in metrics_to_print.items(): print(f" {benchmark} ".center(len(header), "-")) num_seeds = metrics["aggregated"]["num_runs"] // (len(metrics) - 1) # excluding aggregated diff --git a/nemo_skills/prompt/config/robustness/code_prompts/aai_prompt.yaml b/nemo_skills/prompt/config/robustness/code_prompts/aai_prompt.yaml new file mode 100644 index 0000000000..51b474eee9 --- /dev/null +++ b/nemo_skills/prompt/config/robustness/code_prompts/aai_prompt.yaml @@ -0,0 +1,10 @@ +# https://github.com/NVIDIA-NeMo/Skills/blob/main/nemo_skills/prompt/config/eval/aai/livecodebench.yaml +# almost identical to https://artificialanalysis.ai/methodology/intelligence-benchmarking#intelligence-index-evaluation-suite-overview +# except we don't add ### before Format. +# The starter code and formatting instructions are already included inside question by prepare.py + +user: |- + ### Question: + {question} + + ### Answer: (use the provided format with backticks) diff --git a/nemo_skills/prompt/config/robustness/code_prompts/code_1.yaml b/nemo_skills/prompt/config/robustness/code_prompts/code_1.yaml new file mode 100644 index 0000000000..006f8d1bda --- /dev/null +++ b/nemo_skills/prompt/config/robustness/code_prompts/code_1.yaml @@ -0,0 +1,3 @@ +user: |- + {question} + Generate a correct Python program that matches the specification and passes all tests for this question. diff --git a/nemo_skills/prompt/config/robustness/code_prompts/code_2.yaml b/nemo_skills/prompt/config/robustness/code_prompts/code_2.yaml new file mode 100644 index 0000000000..bd4cea64b3 --- /dev/null +++ b/nemo_skills/prompt/config/robustness/code_prompts/code_2.yaml @@ -0,0 +1,6 @@ +user: |- + {question} + Write an executable Python solution. The output should look like: + ```python + [Insert the Python code here.] + ``` diff --git a/nemo_skills/prompt/config/robustness/code_prompts/code_3.yaml b/nemo_skills/prompt/config/robustness/code_prompts/code_3.yaml new file mode 100644 index 0000000000..70cd6953a7 --- /dev/null +++ b/nemo_skills/prompt/config/robustness/code_prompts/code_3.yaml @@ -0,0 +1,8 @@ +user: |- + You are a helpful and harmless assistant. Analyze the following problem, think step-by-step before solving the problem below. + {question} + Please use python programming language only. + You must use ```python for just the final solution code block with the following format: + ```python + # Your code here + ``` diff --git a/nemo_skills/prompt/config/robustness/code_prompts/code_4.yaml b/nemo_skills/prompt/config/robustness/code_prompts/code_4.yaml new file mode 100644 index 0000000000..930127019e --- /dev/null +++ b/nemo_skills/prompt/config/robustness/code_prompts/code_4.yaml @@ -0,0 +1,19 @@ +user: |- + PROBLEM DESCRIPTION: + You will be provided with the description of a python coding problem. Your task is to solve the problem step by step. + + RESPONSE GUIDELINES: + 1. Start with the knowledge required for the solution and the plan on how you will solve the problem.. + 2. Then write the complete and executable Python program. + 3. Your response should be correct and pass all tests. + 4. DO NOT include example usage or test code in your response. + 5. Ensure your response is in the format of ```python``` and includes the necessary background as a comment at the top. + + Example: + ```python + # Background: [Here, insert the necessary knowledge required for the solution and the plan on how you will solve the problem.] + + [Insert the Python code here.] + ``` + + {question} diff --git a/nemo_skills/prompt/config/robustness/code_prompts/ns_gen_codegen.yaml b/nemo_skills/prompt/config/robustness/code_prompts/ns_gen_codegen.yaml new file mode 100644 index 0000000000..dba1c7a91d --- /dev/null +++ b/nemo_skills/prompt/config/robustness/code_prompts/ns_gen_codegen.yaml @@ -0,0 +1,12 @@ +# default prompt for all python based code benchmark evaluations +# https://github.com/NVIDIA-NeMo/Skills/blob/main/nemo_skills/prompt/config/generic/codegen.yaml +user: |- + Here is a problem for which you need to generate/complete code: + {question} + + Please continue to complete the function with python programming language. You are not allowed to modify the given code and do the completion only. + + The solution should be in the following format: + ```python + # Your code here + ``` diff --git a/nemo_skills/prompt/config/robustness/code_prompts/ns_python_codegen.yaml b/nemo_skills/prompt/config/robustness/code_prompts/ns_python_codegen.yaml new file mode 100644 index 0000000000..c7337420a3 --- /dev/null +++ b/nemo_skills/prompt/config/robustness/code_prompts/ns_python_codegen.yaml @@ -0,0 +1,7 @@ +# /home/gnalbandyan/grigor/code/ns_pr/Skills/nemo_skills/prompt/config/eval/livecodebench/python_codegen.yaml +# default prompt for livecodebench Python + +user: |- + Here is a problem for which you need to generate an executable code in python programming language. + + {question} diff --git a/nemo_skills/prompt/config/robustness/prompt_set_config.yaml b/nemo_skills/prompt/config/robustness/prompt_set_config.yaml index 9bc260d059..f00486bd2f 100644 --- a/nemo_skills/prompt/config/robustness/prompt_set_config.yaml +++ b/nemo_skills/prompt/config/robustness/prompt_set_config.yaml @@ -28,3 +28,13 @@ comp-math-24-25: - prompt_config: robustness/math_prompts/boxed_8 - prompt_config: robustness/math_prompts/boxed_aai - prompt_config: robustness/math_prompts/boxed_general + + +livecodebench: + - prompt_config: robustness/code_prompts/aai_prompt + - prompt_config: robustness/code_prompts/ns_gen_codegen + - prompt_config: robustness/code_prompts/ns_python_codegen + - prompt_config: robustness/code_prompts/code_1 + - prompt_config: robustness/code_prompts/code_2 + - prompt_config: robustness/code_prompts/code_3 + - prompt_config: robustness/code_prompts/code_4 From 47d9312522e23a8138a743275ca89c7807f8bc23 Mon Sep 17 00:00:00 2001 From: George Armstrong Date: Wed, 10 Dec 2025 11:40:05 -0800 Subject: [PATCH 33/88] MAINT pin nemo-evaluator (#1095) Signed-off-by: George Armstrong Signed-off-by: Cheng-Ping Hsieh --- requirements/main.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/main.txt b/requirements/main.txt index bb6790dde5..f140a97b50 100644 --- a/requirements/main.txt +++ b/requirements/main.txt @@ -33,7 +33,7 @@ language-data litellm[caching] == 1.77.5 # Requires patching the logging worker (See nemo_skills/inference/patch_litellm_logging.py) math-verify[antlr4_9_3] mcp -nemo-evaluator-launcher +nemo-evaluator-launcher<0.1.47 nemo_run @ git+https://github.com/NVIDIA-NeMo/Run numpy openai From 8bd2125fd9cae3c978873d07d43e01f3a88019ce Mon Sep 17 00:00:00 2001 From: George Armstrong Date: Wed, 10 Dec 2025 16:25:51 -0800 Subject: [PATCH 34/88] Update issue templates Signed-off-by: Cheng-Ping Hsieh --- .github/ISSUE_TEMPLATE/custom.md | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/custom.md diff --git a/.github/ISSUE_TEMPLATE/custom.md b/.github/ISSUE_TEMPLATE/custom.md new file mode 100644 index 0000000000..8bd8fc143b --- /dev/null +++ b/.github/ISSUE_TEMPLATE/custom.md @@ -0,0 +1,10 @@ +--- +name: Custom issue template +about: Default Template +title: '' +labels: '' +assignees: '' + +--- + + From 3c030135968907142abd4c30919f193ee457530a Mon Sep 17 00:00:00 2001 From: George Armstrong Date: Wed, 10 Dec 2025 16:27:22 -0800 Subject: [PATCH 35/88] Delete .github/ISSUE_TEMPLATE directory Signed-off-by: George Armstrong Signed-off-by: Cheng-Ping Hsieh --- .github/ISSUE_TEMPLATE/custom.md | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 .github/ISSUE_TEMPLATE/custom.md diff --git a/.github/ISSUE_TEMPLATE/custom.md b/.github/ISSUE_TEMPLATE/custom.md deleted file mode 100644 index 8bd8fc143b..0000000000 --- a/.github/ISSUE_TEMPLATE/custom.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -name: Custom issue template -about: Default Template -title: '' -labels: '' -assignees: '' - ---- - - From d1449e04af3f2daf1fdcf6562c1e7e9e48eb4256 Mon Sep 17 00:00:00 2001 From: George Armstrong Date: Wed, 10 Dec 2025 16:32:36 -0800 Subject: [PATCH 36/88] enable blank issues (#1096) Signed-off-by: George Armstrong Signed-off-by: Cheng-Ping Hsieh --- .github/ISSUE_TEMPLATE/config.yml | 1 + 1 file changed, 1 insertion(+) create mode 100644 .github/ISSUE_TEMPLATE/config.yml diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000000..0086358db1 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1 @@ +blank_issues_enabled: true From 7f5753e42011144bcff19bbbf038b2a6dc081fb6 Mon Sep 17 00:00:00 2001 From: Minho Ryu Date: Wed, 10 Dec 2025 17:17:31 -0800 Subject: [PATCH 37/88] Fix input_file path handling when executor is "none" (#1089) Signed-off-by: bzantium Signed-off-by: Cheng-Ping Hsieh --- nemo_skills/pipeline/utils/eval.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nemo_skills/pipeline/utils/eval.py b/nemo_skills/pipeline/utils/eval.py index dd44ea0f83..736d1c7cb6 100644 --- a/nemo_skills/pipeline/utils/eval.py +++ b/nemo_skills/pipeline/utils/eval.py @@ -91,7 +91,7 @@ def get_benchmark_args_from_module( split = get_arg_from_module_or_dict(benchmark_module, "EVAL_SPLIT", "test", override_dict) if not is_on_cluster: - if pipeline_utils.is_mounted_filepath(cluster_config, data_path): + if pipeline_utils.is_mounted_filepath(cluster_config, data_path) or cluster_config["executor"] == "none": input_file = f"{data_path}/{benchmark.replace('.', '/')}/{split}.jsonl" unmounted_input_file = pipeline_utils.get_unmounted_path(cluster_config, input_file) unmounted_path = str(Path(__file__).parents[3] / unmounted_input_file.replace("/nemo_run/code/", "")) From ab87b77a8268b92beb922b72663636b25b82d12f Mon Sep 17 00:00:00 2001 From: George Armstrong Date: Wed, 10 Dec 2025 17:39:47 -0800 Subject: [PATCH 38/88] TST for #1089 (#1097) Signed-off-by: George Armstrong Signed-off-by: Cheng-Ping Hsieh --- tests/test_configs.py | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/tests/test_configs.py b/tests/test_configs.py index 9494a2f369..e99ac3a979 100644 --- a/tests/test_configs.py +++ b/tests/test_configs.py @@ -13,10 +13,12 @@ # limitations under the License. import subprocess +from types import SimpleNamespace import pytest from nemo_skills.pipeline.utils import get_mounted_path +from nemo_skills.pipeline.utils.eval import get_benchmark_args_from_module def test_error_on_extra_params(): @@ -78,3 +80,42 @@ def test_get_mounted_path(mount_source, mount_dest, input_path, expected): result = get_mounted_path(cluster_config, input_path) assert result == expected + + +def test_get_benchmark_args_input_file_should_be_local_path_for_executor_none(tmp_path): + """For executor='none', input_file should be a real local path, not a container path.""" + # Setup: create a local data file + benchmark_dir = tmp_path / "gsm8k" + benchmark_dir.mkdir() + (benchmark_dir / "test.jsonl").write_text('{"problem": "test"}\n') + + cluster_config = {"executor": "none", "containers": {}} + mock_module = SimpleNamespace( + EVAL_SPLIT="test", + PROMPT_CONFIG="", + GENERATION_ARGS="", + EVAL_ARGS="", + REQUIRES_SANDBOX=False, + KEEP_MOUNTS_FOR_SANDBOX=False, + GENERATION_MODULE="nemo_skills.inference.generate", + JUDGE_PIPELINE_ARGS={}, + JUDGE_ARGS="", + NUM_SAMPLES=0, + NUM_CHUNKS=0, + ) + + result = get_benchmark_args_from_module( + benchmark_module=mock_module, + benchmark="gsm8k", + split="test", + cluster_config=cluster_config, + data_path=str(tmp_path), # local path like /tmp/pytest-xxx + is_on_cluster=False, + eval_requires_judge=False, + ) + + # For executor='none' (no container), input_file should be the actual local path + expected_input_file = str(tmp_path / "gsm8k" / "test.jsonl") + assert result.input_file == expected_input_file, ( + f"Expected local path {expected_input_file}, got {result.input_file}" + ) From 70cc6bfc7ea363d0a8f71735378901c2435570bd Mon Sep 17 00:00:00 2001 From: Stephen Ge Date: Wed, 10 Dec 2025 20:58:59 -0500 Subject: [PATCH 39/88] Stepheng/prover cleanup (#1078) Signed-off-by: Stephen Ge Co-authored-by: George Armstrong Signed-off-by: Cheng-Ping Hsieh --- nemo_skills/code_execution/proof_utils.py | 184 ++++++++ nemo_skills/inference/prover.py | 426 ++++++++++++++++++ ...mal-proof-deepseek-prover-v2-nemotron.yaml | 29 ++ .../lean4/goedel-prover-v2-nemotron.yaml | 28 ++ .../goedel-prover-v2-refinement-nemotron.yaml | 28 ++ .../lean4/goedel-prover-v2-refinement.yaml | 24 + .../prompt/config/lean4/goedel-prover-v2.yaml | 27 ++ 7 files changed, 746 insertions(+) create mode 100644 nemo_skills/inference/prover.py create mode 100644 nemo_skills/prompt/config/lean4/formal-proof-deepseek-prover-v2-nemotron.yaml create mode 100644 nemo_skills/prompt/config/lean4/goedel-prover-v2-nemotron.yaml create mode 100644 nemo_skills/prompt/config/lean4/goedel-prover-v2-refinement-nemotron.yaml create mode 100644 nemo_skills/prompt/config/lean4/goedel-prover-v2-refinement.yaml create mode 100644 nemo_skills/prompt/config/lean4/goedel-prover-v2.yaml diff --git a/nemo_skills/code_execution/proof_utils.py b/nemo_skills/code_execution/proof_utils.py index fdf84a87cf..33c92891db 100644 --- a/nemo_skills/code_execution/proof_utils.py +++ b/nemo_skills/code_execution/proof_utils.py @@ -14,12 +14,16 @@ """Shared utilities for proof processing and evaluation.""" +import logging import re from dataclasses import dataclass from typing import Any, Dict from nemo_skills.code_execution.utils import clean_formal_generation from nemo_skills.dataset.utils import get_lean4_header +from nemo_skills.utils import get_logger_name + +LOG = logging.getLogger(get_logger_name(__file__)) @dataclass @@ -192,3 +196,183 @@ def prepare_predicted_proof_from_line_dict( return build_lean4_proof( generation=line_dict["generation"], data_point=line_dict, config=config, answer_format=answer_format ) + + +# ------------------------------------------------------------------------------------------------ +# The following code is adapted from https://github.com/Goedel-LM/Goedel-Prover-V2 +# Used for multi-turn proof refinement in the lean4_prover workflow +# ------------------------------------------------------------------------------------------------ + + +def remove_comments(text): + # First remove all /- ... -/ blocks + text = re.sub(r"/-.*?-/", "", text, flags=re.DOTALL) + # Then remove -- comments from each line + lines = text.split("\n") + cleaned_lines = [] + for line in lines: + cleaned_line = line.split("--", 1)[0] + if cleaned_line.strip() == "": + continue + cleaned_lines.append(cleaned_line) + # Join back together and remove excessive empty lines + cleaned_text = "\n".join(cleaned_lines) + return cleaned_text.strip() + + +def move_imports_to_beginning(input_string): + lines = input_string.split("\n") + import_lines = [line for line in lines if line.startswith("import")] + other_lines = [line for line in lines if not line.startswith("import")] + return "\n".join(import_lines + other_lines) + + +def return_theorem_to_prove(text): + # Pattern that matches from 'theorem' or 'lemma' to ':= by sorry' with any content in between + pattern = r"((?:theorem).*?:=\s*by\s*sorry)" + match = re.search(pattern, text, re.DOTALL) + return match.span() if match else None + + +def return_theorem_to_replace(text): + # Pattern that matches from 'theorem' or 'lemma' to ':= by sorry' with any content in between + pattern = r"((?:^|\s)theorem\s+.*?:=\s*by)" + match = re.search(pattern, text, re.DOTALL) + return match.span() if match else None + + +def replace_statement_in_proof(statement, proof): + if ("apply?" in proof) or ("exact?" in proof): + return "**Error**, 'apply?' or 'exact?' is used, which is not allowed." + stats_re = remove_comments(statement) + stats_span_ = return_theorem_to_prove(stats_re) + if stats_span_ is None: + error_app = "\n".join(["\n"] + ["-- " + x for x in statement.split("\n")]) + return f"**Error**, can not find 'theorem' and ':= sorry' in {error_app}" + proof_str = remove_comments(proof) + span = return_theorem_to_replace(proof_str) + if span is None: + error_app = "\n".join(["\n"] + ["-- " + x for x in proof.split("\n")]) + return f"**Error**, can not find 'theorem' and ':=' in {error_app}" + return stats_re[: stats_span_[1]].replace("sorry", "") + proof_str[span[1] :] + + +def refine_by_sorry(text): + # Define the regular expression pattern + target_pattern = r":=\s*(?:by\s*)?(?:sorry\s*)?" + replacement = ":= by sorry" # The new text we want to insert + # We construct the pattern with two capturing groups + # (group 1: the part from 'theorem' to just before our target) + # (group 2: the target pattern itself) + combined_pattern = r"(theorem.*?)(" + target_pattern + r")" + # Find the first match + match = re.search(combined_pattern, text, re.DOTALL) + if match: + # The part of the string BEFORE the target we want to replace + # We use match.start(2) which is the start of the second group (our target) + prefix = text[: match.start(2)] + # Concatenate the prefix with the replacement to get the final, truncated string + final_text = prefix + replacement + else: + final_text = text + return final_text + + +def extract_code(inputs): + import_head = ( + "import Mathlib\nimport Aesop\n\nset_option maxHeartbeats 0\n\nopen BigOperators Real Nat Topology Rat\n\n" + ) + pattern = r"```lean4\n(.*?)\n```" + matches = re.findall(pattern, inputs, re.DOTALL) + if matches: + return import_head + matches[-1] + pattern = r"```lean4\n(.*?)```" + matches = re.findall(pattern, inputs, re.DOTALL) + if matches: + return import_head + matches[-1] + pattern = r"```lean\n(.*?)```" + matches = re.findall(pattern, inputs, re.DOTALL) + if matches: + return import_head + matches[-1] + return "None" + + +def parse_error(log_string): + """Parse Lean4 compiler error messages from log output.""" + error_pattern = re.compile( + r"(/lean4/my_project/.*?:\d+:\d+: error:.*?)(?=\n/lean4/my_project|\Z)", + re.DOTALL, + ) + errors = error_pattern.findall(log_string) + pattern = re.compile(r":(\d+):(\d+):") + error_list = [] + for error in errors: + match = pattern.search(error) + error_list.append( + { + "pos": {"line": int(match.group(1)), "column": int(match.group(2))}, + "endPos": None, + "data": error.split("error:")[1], + } + ) + + return error_list + + +def get_error_str(code, errors, error_thres=True): + """Format compiler errors with code context for display.""" + err_str = "" + code_lines = code.split("\n") + error_num_thres = 8 if error_thres else len(errors) + + for i, error in enumerate(errors[:error_num_thres]): + start_line = error["pos"]["line"] - 1 + start_col = error["pos"]["column"] + if start_line >= len(code_lines): + LOG.warning( + "Error line %d exceeds code length %d. Errors: %s, Code: %s", start_line, len(code_lines), errors, code + ) + continue + if error["endPos"] is None: + end_line = start_line + end_col = len(code_lines[start_line]) + else: + end_line = error["endPos"]["line"] - 1 + end_col = error["endPos"]["column"] + + err_str += f"\nError {i + 1}:\n" + err_str += "\nCorresponding Code:\n```lean4\n" + error_code = "" + for ii in range(-4, 0): + if start_line + ii >= 0: + error_code += f"{code_lines[start_line + ii]}\n" + if start_line != end_line: + error_code += code_lines[start_line][:start_col] + "" + code_lines[start_line][start_col:] + "\n" + if not error_thres: + for j in range(start_line + 1, end_line): + error_code += f"{code_lines[j]}\n" + else: + show_line = 6 + for j in range(start_line + 1, min(end_line, start_line + show_line)): + error_code += f"{code_lines[j]}\n" + if end_line > start_line + show_line: + leading_spaces = len(code_lines[j]) - len(code_lines[j].lstrip(" ")) + error_code += "\n" + " " * leading_spaces + "... --[Truncated]-- ...\n" + error_code += code_lines[end_line][:end_col] + "" + code_lines[end_line][end_col:] + "\n" + else: + error_code += ( + code_lines[start_line][:start_col] + + "" + + code_lines[start_line][start_col:end_col] + + "" + + code_lines[start_line][end_col:] + + "\n" + ) + if end_line + 1 < len(code_lines): + error_code += f"{code_lines[end_line + 1]}\n" + err_str += error_code + err_str += "\n```\n" + err_str += f"\nError Message: {error['data']}\n" + if len(errors) > error_num_thres: + err_str += f"\n... [Omitted {len(errors) - error_num_thres} more errors] ...\n" + return err_str diff --git a/nemo_skills/inference/prover.py b/nemo_skills/inference/prover.py new file mode 100644 index 0000000000..848c7c70dd --- /dev/null +++ b/nemo_skills/inference/prover.py @@ -0,0 +1,426 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import re +import sys +from copy import deepcopy +from dataclasses import asdict, is_dataclass + +import hydra +from transformers import AutoTokenizer + +from nemo_skills.code_execution.proof_utils import ( + extract_code, + get_error_str, + parse_error, + refine_by_sorry, + replace_statement_in_proof, +) +from nemo_skills.code_execution.sandbox import sandbox_params +from nemo_skills.inference.model import server_params +from nemo_skills.inference.model.base import EndpointType +from nemo_skills.prompt.utils import get_prompt +from nemo_skills.utils import ( + get_help_message, + get_logger_name, + nested_dataclass, + parse_reasoning, + setup_logging, +) + +from .generate import GenerateSolutionsConfig, GenerationTask + +LOG = logging.getLogger(get_logger_name(__file__)) + +reasoning_effort_list = [ + "low", + "medium", + "high", +] # This is only used for adaptive reasoning with gpt-oss models + + +@nested_dataclass(kw_only=True) +class ProverConfig(GenerateSolutionsConfig): + max_tokens: int = 40960 # model max tokens + n_pass: int = 1 # number of passes to run the prover + + # Lean 4 specific parameters + nemotron_refinement: bool = False # whether to use single-turn nemotron-style refinement + refinement: bool = False # whether to refine the code + refinement_max_turns: int = 2 # maximum number of turns for refinement + refinement_prompt_config: str | None = None # prompt for multi-turn refinement feedback + # prompt for single-turn nemotron refinement (used when nemotron_refinement=True) + nemotron_refinement_prompt_config: str | None = None + adaptive_reasoning: bool = False # whether to adapt the reasoning effort + parse_generation: bool = False # whether to parse the generation + remove_cot: bool = False # whether to remove the cot from the generation + # whether to delete the wrong turns from the generation + delete_wrong_turns: bool = False + + def _post_init_validate_params(self): + """Validate that certain parameters are restricted to certain values""" + if self.prompt_format == "openai": + raise ValueError( + "prompt_format='openai' is not supported for lean4_prover. Use prompt_format='ns' with a prompt_config." + ) + if self.prompt_format != "ns": + raise ValueError(f"prompt_format must be 'ns', got '{self.prompt_format}'") + + assert self.prompt_config is not None, "prompt_config is required for lean4_prover" + + for param, default_value in self._get_disallowed_params(): + if getattr(self, param) != default_value: + raise ValueError(f"{param} must be {default_value}") + + if self.n_pass > 32: + LOG.warning( + "n_pass=%d exceeds recommended maximum of 32. Consider using num_random_seeds instead.", self.n_pass + ) + + +cs = hydra.core.config_store.ConfigStore.instance() +cs.store(name="base_prover_config", node=ProverConfig) + + +class ProverTask(GenerationTask): + def __init__(self, cfg: ProverConfig): + """ + Class that represents a generation task. It implements a template of steps to generate solutions using LLMs. + Individual functions can be overriden to customize the behavior of the generation task. + + Args: + cfg: GenerateSolutionsConfig object with the configuration parameters or subclass. + """ + super().__init__(cfg) + + # Initialize tokenizer for chat template application + tokenizer_path = self.cfg.tokenizer or self.cfg.server.get("model") + self.hf_tokenizer = AutoTokenizer.from_pretrained(tokenizer_path) + + if self.cfg.refinement: + self.setup_refine_prompt() + + if self.cfg.delete_wrong_turns: + assert self.cfg.remove_cot, "remove_cot is required when delete_wrong_turns is enabled" + + def log_example_prompt(self, data): + return + + def setup_llm(self): + if self.cfg.code_execution: + raise ValueError("Code execution is not supported for prover. Use sandbox config for Lean4 execution.") + return super().setup_llm() + + def setup_refine_prompt(self): + assert self.cfg.refinement_prompt_config is not None, ( + "refinement_prompt_config is required when refinement is enabled. Please set refinement=False to disable refinement." + ) + self.refine_prompt = get_prompt(self.cfg.refinement_prompt_config) + + if self.cfg.nemotron_refinement: + assert self.cfg.nemotron_refinement_prompt_config is not None, ( + "nemotron_refinement_prompt_config is required when nemotron_refinement is enabled." + ) + self.nemotron_refine_prompt = get_prompt(self.cfg.nemotron_refinement_prompt_config) + + async def _generate_single_completion(self, prompt: str, **kwargs): + """Generate a single completion with semaphore-controlled concurrency.""" + if is_dataclass(self.cfg.inference): + inference_params = asdict(self.cfg.inference) + else: + # Already a dict from Hydra + inference_params = dict(self.cfg.inference) + generation_params = { + "prompt": prompt, + "stop_phrases": [self.cfg.stop_phrase] if self.cfg.stop_phrase else None, + **inference_params, + **self.extra_generate_params, + } + # Override endpoint_type to text since we already applied the chat template + generation_params["endpoint_type"] = EndpointType.text + for key, value in kwargs.items(): + generation_params[key] = value + + # Use semaphore for concurrency control (inherited from GenerationTask) + async with self.semaphore: + generation = await self.llm.generate_async(**generation_params) + if self.cfg.adaptive_reasoning: + assert generation_params["extra_body"].get("reasoning_effort", None) is not None, ( + "reasoning_effort is required when adaptive_reasoning is enabled" + ) + reasoning_effort_index = reasoning_effort_list.index( + generation_params["extra_body"].get("reasoning_effort", None) + ) + while len(generation["generation"]) == 0 and reasoning_effort_index > 0: + LOG.info( + "Reasoning effort is too high, reducing to %s", + reasoning_effort_list[reasoning_effort_index - 1], + ) + reasoning_effort_index = reasoning_effort_index - 1 + generation_params["extra_body"]["reasoning_effort"] = reasoning_effort_list[reasoning_effort_index] + generation = await self.llm.generate_async(**generation_params) + + if self.cfg.parse_generation: + parse_reasoning( + generation, + self.cfg.generation_key, + self.cfg.end_reasoning_string, + ) + return generation + + # factor out this part so it won't become a bottleneck. + async def _extract_and_replace_code(self, formal_statement, generation): + code = extract_code(generation) + full_code = replace_statement_in_proof(formal_statement, code) + return code, full_code + + def _transform_for_nemotron_refinement(self, proof_attempt: str, error_message: str) -> list[dict]: + """Transform multi-turn refinement into single-turn nemotron-style prompt.""" + return self.nemotron_refine_prompt.fill( + { + "proof_attempt": proof_attempt, + "error_message": error_message, + } + ) + + async def _single_data_point_generate(self, data_point, data): + formal_statement = ( + (data_point["header"].strip() + "\n") + + data_point["informal_prefix"].strip() + + ("\n" + data_point["formal_statement"].strip()) + ) + formal_statement = refine_by_sorry(formal_statement) + prompt_turn_list = self.prompt.fill({"problem": formal_statement.strip()}) + + full_prompt_turn_list = deepcopy( + prompt_turn_list + ) # We need to get a full copy of the prompt turn list for the final result in case remove_cot is enabled. This is only used to generate SFT data. + prompt_turn_list_list = [] # We need to store the prompt turn list for each turn for the final result in case delete_wrong_turns is enabled. This is only used to generate SFT data. + base_prompt_turn_list = deepcopy(prompt_turn_list) + + code_list = [] + results_dict_list = [] + assert isinstance(prompt_turn_list, list), "prompt_turn_list should be a list" + + success = False + turn_idx = 0 + last_proof_attempt = None # Track for nemotron refinement + last_error_message = None # Track for nemotron refinement + for turn_idx in range(self.cfg.refinement_max_turns): + results_dict = {} # everything will be stored in this dict + if turn_idx != 0 and self.cfg.nemotron_refinement and last_proof_attempt and last_error_message: + prepared_conversation = self._transform_for_nemotron_refinement(last_proof_attempt, last_error_message) + else: + prepared_conversation = prompt_turn_list + prefix_tokens = self.hf_tokenizer.apply_chat_template( + prepared_conversation, tokenize=True, add_generation_prompt=True + ) + num_tokens_prefix = len(prefix_tokens) + prefix = self.hf_tokenizer.apply_chat_template( + prepared_conversation, tokenize=False, add_generation_prompt=True + ) + # We need to check if the prefix is too long, if it is, we need to break the loop + if num_tokens_prefix > self.cfg.max_tokens: + break + + generation = await self._generate_single_completion( + prefix, + tokens_to_generate=min( + self.cfg.max_tokens - num_tokens_prefix, + self.cfg.inference.tokens_to_generate, + ), + ) + + new_prompt_turn_list = deepcopy(prompt_turn_list) + new_prompt_turn_list += [{"role": "assistant", "content": generation["generation"]}] + + prompt_turn_list_list.append( + new_prompt_turn_list + ) # This stores the latest turn list after each generation. + + code, full_code = await self._extract_and_replace_code(formal_statement, generation["generation"]) + last_proof_attempt = generation["generation"] # Track for nemotron refinement + code_list.append(full_code) + results_dict["code"] = code # We keep track of the uncleaned code. + if self.cfg.remove_cot and not ( + code == "None" or "**Error**" in full_code + ): # check if successfully parse the code. We do not want to delete the turn if there is a parsing error. + if self.cfg.delete_wrong_turns: + prompt_turn_list = deepcopy(base_prompt_turn_list) + [ + { + "role": "assistant", + "content": f"```lean4\n{full_code.strip()}\n```", + } + ] # only keep the latest turn + else: + prompt_turn_list += [ + { + "role": "assistant", + "content": f"```lean4\n{full_code.strip()}\n```", + } + ] + full_prompt_turn_list += [{"role": "assistant", "content": generation["generation"]}] + else: + prompt_turn_list += [{"role": "assistant", "content": generation["generation"]}] + full_prompt_turn_list += [{"role": "assistant", "content": generation["generation"]}] + + if code == "None" or "**Error**" in full_code: + if code == "None": + execution_result = { + "process_status": "failed", + "stderr": "", + "stdout": "Parsing error. Cannot parse the code from output. Please try again and write the code in the format of ```lean4\n\n```", + } + elif "**Error**" in full_code: + execution_result = { + "process_status": "failed", + "stderr": "", + "stdout": full_code, + } + else: + execution_result = { + "process_status": "failed", + "stderr": "", + "stdout": "Unknown error when parsing code.", + } + results_dict["execution_result"] = execution_result + results_dict["success"] = False + last_error_message = execution_result["stdout"] # Track for nemotron refinement + feedback = self.refine_prompt.fill({"error_message": last_error_message}) + results_dict["feedback"] = feedback[0]["content"] + else: + if self.sandbox is None: + raise RuntimeError( + "Sandbox is required for Lean4 code execution but was not configured. " + "Please provide sandbox configuration." + ) + # execute_code returns (result_dict, session_id) tuple + execution_result, _ = await self.sandbox.execute_code( + full_code, language="lean4", timeout=600.0, max_output_characters=1000000 + ) + results_dict["execution_result"] = execution_result + # Handle timeout (now indicated by process_status in the dict) + if execution_result.get("process_status") == "timeout": + results_dict["success"] = False + last_error_message = ( + "The compilation timed out. There might be a heavy computation in the code or an endless loop." + ) + feedback = self.refine_prompt.fill({"error_message": last_error_message}) + results_dict["feedback"] = feedback[0]["content"] + elif ( + execution_result["process_status"] == "completed" + and "sorry" not in execution_result["stdout"] + and "failed" not in execution_result["stdout"] + ): + results_dict["success"] = True + else: + error_list = parse_error(execution_result["stdout"]) + error_message = get_error_str(full_code, error_list, error_thres=True) + # checking for sorry + if execution_result["process_status"] == "completed": + stdout = execution_result["stdout"].lower() + stderr = execution_result["stderr"].lower() + combined = stdout + "\n" + stderr + if re.search(r"\bsorry\b", combined) is not None: + error_message += "\nThe code contains 'sorry', which means the proof is incomplete." + if error_message.strip() == "": # something in stderr indicating failure + error_message = execution_result["stderr"][:1000] + if len(execution_result["stderr"]) > 1000: + error_message += "... (truncated)" + + last_error_message = ( + "We use to signal the position of the error. \n" + error_message + ) + feedback = self.refine_prompt.fill({"error_message": last_error_message}) + results_dict["feedback"] = feedback[0]["content"] + results_dict["success"] = False + + results_dict_list.append(results_dict) + + if results_dict["success"]: + # This is the case when the code execution is successful. The theorem is proved. + break + else: + if self.cfg.refinement and turn_idx < self.cfg.refinement_max_turns - 1: + prompt_turn_list += feedback + full_prompt_turn_list += feedback + else: + # Proving attempt failed. + break + + if len(results_dict_list) > 0 and results_dict_list[-1]["success"]: + success = True + + # Usually only need prompt_turn_list for standard SFT, full_prompt_turn_list for SFT with remove_cot enabled, prompt_turn_list_list for SFT with delete_wrong_turns enabled. + return { + "code_list": code_list, + "results_dict_list": results_dict_list, + "prompt_turn_list": prompt_turn_list, + "turn_idx": turn_idx, + "success": success, + "full_prompt_turn_list": full_prompt_turn_list, + "prompt_turn_list_list": prompt_turn_list_list, + } + + async def pass_at_N(self, data_point, data, N=None): + if N is None: + N = self.cfg.n_pass + + new_results_dict = {"success": False} + for i in range(N): + results_dict = await self._single_data_point_generate(data_point, data) + + if results_dict["success"]: + new_results_dict["success"] = True + break + + new_results_dict["results_dict_list"] = results_dict + new_results_dict["n_pass"] = i + 1 + + return new_results_dict + + async def process_single_datapoint(self, data_point, all_data): + result = await self.pass_at_N(data_point, all_data) + result_dict = {"generation": result} + + return result_dict + + +GENERATION_TASK_CLASS = ProverTask + + +# Update the hydra main to use the class method +@hydra.main(version_base=None, config_name="base_prover_config") +def generate(cfg: ProverConfig): + cfg = ProverConfig(_init_nested=True, **cfg) + LOG.info("Config used: %s", cfg) + + task = ProverTask(cfg) + task.generate() + + +HELP_MESSAGE = get_help_message( + ProverConfig, + server_params=server_params(), + sandbox_params=sandbox_params(), +) + + +if __name__ == "__main__": + if "--help" in sys.argv or "-h" in sys.argv: + print(HELP_MESSAGE) + else: + setup_logging() + generate() diff --git a/nemo_skills/prompt/config/lean4/formal-proof-deepseek-prover-v2-nemotron.yaml b/nemo_skills/prompt/config/lean4/formal-proof-deepseek-prover-v2-nemotron.yaml new file mode 100644 index 0000000000..54da2461e8 --- /dev/null +++ b/nemo_skills/prompt/config/lean4/formal-proof-deepseek-prover-v2-nemotron.yaml @@ -0,0 +1,29 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Configuration for proving formal theorems in Lean 4. +# This file is tailored for tasks that involve constructing and verifying proofs +# of theorems within the Lean 4 formal system. + +user: |- + Complete the following Lean 4 code: + + ```lean4 + {header}{informal_prefix}{formal_statement} + sorry + ``` + + First, think through your solution step-by-step. Provide a detailed proof plan outlining the main proof steps and strategies. The plan should highlight key ideas, intermediate lemmas, and proof structures that will guide the construction of the final formal proof. + + Then provide your final answer. Your final answer must be a single, complete Lean 4 markdown code block containing the completed theorem. Do NOT include any text or explanation before or after the code block. Begin with ```lean4 and end with ```. diff --git a/nemo_skills/prompt/config/lean4/goedel-prover-v2-nemotron.yaml b/nemo_skills/prompt/config/lean4/goedel-prover-v2-nemotron.yaml new file mode 100644 index 0000000000..db6adac3a5 --- /dev/null +++ b/nemo_skills/prompt/config/lean4/goedel-prover-v2-nemotron.yaml @@ -0,0 +1,28 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Configuration for proving formal theorems in Lean 4. +# This file is tailored for tasks that involve constructing and verifying proofs +# of theorems within the Lean 4 formal system. + +user: |- + Complete the following Lean 4 code: + + ```lean4 + {problem} + ``` + + First, think through your solution step-by-step. Provide a detailed proof plan outlining the main proof steps and strategies. The plan should highlight key ideas, intermediate lemmas, and proof structures that will guide the construction of the final formal proof. + + Then provide your final answer. Your final answer must be a single, complete Lean 4 markdown code block containing the completed theorem. Do NOT include any text or explanation before or after the code block. Begin with ```lean4 and end with ```. diff --git a/nemo_skills/prompt/config/lean4/goedel-prover-v2-refinement-nemotron.yaml b/nemo_skills/prompt/config/lean4/goedel-prover-v2-refinement-nemotron.yaml new file mode 100644 index 0000000000..960e4dbc8a --- /dev/null +++ b/nemo_skills/prompt/config/lean4/goedel-prover-v2-refinement-nemotron.yaml @@ -0,0 +1,28 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Configuration for proving formal theorems in Lean 4. +# This file is tailored for tasks that involve constructing and verifying proofs +# of theorems within the Lean 4 formal system. + +user: |- + Here is a proof attempt for the following theorem in Lean4. + + {proof_attempt} + + The proof is not correct. Following is the compilation error message: + + {error_message} + + Your task is to fix this proof. Before producing the Lean 4 code to formally prove the given theorem, do a detailed analysis of the error message. Your final answer must be a single, complete Lean 4 markdown code block containing the completed theorem. Do NOT include any text or explanation before or after the code block. Begin with ```lean4 and end with ```. diff --git a/nemo_skills/prompt/config/lean4/goedel-prover-v2-refinement.yaml b/nemo_skills/prompt/config/lean4/goedel-prover-v2-refinement.yaml new file mode 100644 index 0000000000..b46fa0a3cc --- /dev/null +++ b/nemo_skills/prompt/config/lean4/goedel-prover-v2-refinement.yaml @@ -0,0 +1,24 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Configuration for proving formal theorems in Lean 4. +# This file is tailored for tasks that involve constructing and verifying proofs +# of theorems within the Lean 4 formal system. + +user: |- + The proof is not correct. Following is the compilation error message: + + {error_message} + + Before producing the Lean 4 code to formally prove the given theorem, provide a detailed analysis of the error message. diff --git a/nemo_skills/prompt/config/lean4/goedel-prover-v2.yaml b/nemo_skills/prompt/config/lean4/goedel-prover-v2.yaml new file mode 100644 index 0000000000..86d18a33f1 --- /dev/null +++ b/nemo_skills/prompt/config/lean4/goedel-prover-v2.yaml @@ -0,0 +1,27 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Configuration for proving formal theorems in Lean 4. +# This file is tailored for tasks that involve constructing and verifying proofs +# of theorems within the Lean 4 formal system. + +user: |- + Complete the following Lean 4 code: + + ```lean4 + {problem} + ``` + + Before producing the Lean 4 code to formally prove the given theorem, provide a detailed proof plan outlining the main proof steps and strategies. + The plan should highlight key ideas, intermediate lemmas, and proof structures that will guide the construction of the final formal proof. From 8cfb8d78cb7933e801500e18fef78de3a903d753 Mon Sep 17 00:00:00 2001 From: Jiacheng Xu Date: Fri, 12 Dec 2025 02:34:47 +0800 Subject: [PATCH 40/88] add stem dependencies in main python sandbox (#1099) Signed-off-by: Jiacheng Xu Signed-off-by: George Armstrong Co-authored-by: Jiacheng Xu Co-authored-by: George Armstrong Signed-off-by: Cheng-Ping Hsieh --- dockerfiles/Dockerfile.sandbox | 18 +++ requirements/stem.txt | 201 +++++++++++++++++++++++++++++++++ 2 files changed, 219 insertions(+) create mode 100644 requirements/stem.txt diff --git a/dockerfiles/Dockerfile.sandbox b/dockerfiles/Dockerfile.sandbox index 7c22cc3e29..3cccd8ee6c 100644 --- a/dockerfiles/Dockerfile.sandbox +++ b/dockerfiles/Dockerfile.sandbox @@ -63,6 +63,24 @@ ENV PATH="/lean4/my_project:$PATH" COPY requirements/code_execution.txt /app/requirements.txt RUN pip install --no-cache-dir -r /app/requirements.txt + +# Install STEM related libraries +COPY requirements/stem.txt /app/stem_requirements.txt + + +# Speed/size/env hygiene +ENV PIP_DISABLE_PIP_VERSION_CHECK=1 \ + UV_SYSTEM_PYTHON=1 \ + PATH="/root/.local/bin:${PATH}" + + +# Install uv (adds to ~/.local/bin), then install deps +RUN if [ "$GITHUB_CI" != "1" ]; then \ + curl -LsSf https://astral.sh/uv/install.sh | sh && \ + uv pip install --upgrade pip && \ + uv pip install -r /app/stem_requirements.txt --no-cache-dir --extra-index-url https://download.pytorch.org/whl/cpu; \ + fi + # For scicode eval - create data directory and download test data # Set GITHUB_CI=1 build arg to skip download (useful for CI when download fails) # If skipped, scicode evaluations will fail unless the file is manually mounted diff --git a/requirements/stem.txt b/requirements/stem.txt new file mode 100644 index 0000000000..c86cb3b4e5 --- /dev/null +++ b/requirements/stem.txt @@ -0,0 +1,201 @@ +arxiv +ascii_magic +astronomy +astroquery +atc +basc +bbn +beautifulsoup4 +bible +biopython +bioservices +bson +cactus +cantera +certifi +chardet +chemicals +chemics +chemlib +chempy +chemspipy +chess +cloudscraper +convertdate +conway +coxeter +Crypto +cssselect +cvxopt +cvxpy +data +datasets +ddc +depmap +diff +dill +dipy +dp_accounting +duckduckgo_search +easyocr +ecdsa +enchant +ephem +ete3 +feedparser +filetype +fiona +fishpy +fontTools +galois +gemmi +geocoder +geonamescache +geopandas +geopy +gita +gmpy2 +googletrans +grep +gutenbergpy +html5lib +HTMLParser +huggingface_hub +iapws +imageio +importlib_metadata +importlib_resources +IndianConstitution +indic_nlp_library +inflect +ipywidgets +isbnlib +jax +jdcal +language_tool_python +law +lie +LIEGenTools +lifelines +lingpy +lxml +matplotlib +mendeleev +mido +mimic +mingus +mip +molmass +molparse +molvs +music21 +mygene +myvariant +networkx +nibabel +nltk +nuclear +num2words +numba +numpy +numpy_financial +ocl +open_tamil +opencv-python +openmc_data +openmm +openpyxl +optopy +ortools +osmnx +packaging +paddleocr +pandapower +pandas +pandas_datareader +pdf2image +pdfminer +pdfplumber +pdfreader +piexif +pint +planarity +polyhedron +pretty_midi +pronouncing +ptable +py3Dmol +pybel +pyclipper +pycosat +pycountry +pydataset +pyequion +pyfiglet +pyfluids +pyhull +pylaw +pymatgen +pymcm +pymongo +PyMuPDF +PyPDF2 +pyromat +pysam +pysat +pyscf +pysmiles +pyswisseph +pytamil +pyteomics +pytesseract +python-igraph +python-snappy +qiskit +quote +quotes +qutip +ragas +rdflib +rdkit +requests +requests_cache +sanskrit +sanskrit_parser +scipy +seaborn +shakespeare +shapely +sieve +skimpy +skyfield +spacy +spherogram +statistic +statsmodels +steam +stim +sympy +tamil +tensorflow +tensorflow_datasets +thermo +thermochem +thermopy +thermostat +tinycss2 +torch +torchvision +transformers +trimesh +typ +utils +vedas +wbdata +webcolors +wikidata +wikipedia>=1.4.0 +wikipedia_api +wordfreq +wptools +yfinance From 843b8c603f77d31b9d05d4e9a8613b4bf2890117 Mon Sep 17 00:00:00 2001 From: George <37293288+Jorjeous@users.noreply.github.com> Date: Thu, 11 Dec 2025 23:35:56 +0400 Subject: [PATCH 41/88] Audiometrics unification (#1093) Signed-off-by: George Zelenfroind Signed-off-by: Nikolai Ludwig Signed-off-by: George Armstrong Signed-off-by: i-vainn Signed-off-by: Grigor Nalbandyan Co-authored-by: Nick Ludwig Co-authored-by: George Armstrong Co-authored-by: Ivan Co-authored-by: Wojciech Prazuch Co-authored-by: gnalbandyan <153070076+gnalbandyan@users.noreply.github.com> Signed-off-by: Cheng-Ping Hsieh --- README.md | 1 + nemo_skills/evaluation/evaluator/__init__.py | 2 + nemo_skills/evaluation/evaluator/audio.py | 387 ++++++++++++++++++ .../evaluation/metrics/audio_metrics.py | 375 +++++++++++++++++ nemo_skills/evaluation/metrics/map_metrics.py | 2 + 5 files changed, 767 insertions(+) create mode 100644 nemo_skills/evaluation/evaluator/audio.py create mode 100644 nemo_skills/evaluation/metrics/audio_metrics.py diff --git a/README.md b/README.md index 5306a317a7..e5231eb89b 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,7 @@ Nemo-Skills is a collection of pipelines to improve "skills" of large language models (LLMs). We support everything needed for LLM development, from synthetic data generation, to model training, to evaluation on a wide range of benchmarks. Start developing on a local workstation and move to a large-scale Slurm cluster with just a one-line change. + Here are some of the features we support: - [Flexible LLM inference](https://nvidia-nemo.github.io/Skills/pipelines/generation/): diff --git a/nemo_skills/evaluation/evaluator/__init__.py b/nemo_skills/evaluation/evaluator/__init__.py index a6cd083c71..0c2aa1e3b0 100644 --- a/nemo_skills/evaluation/evaluator/__init__.py +++ b/nemo_skills/evaluation/evaluator/__init__.py @@ -15,6 +15,7 @@ import asyncio from typing import Any, Callable, Dict +from nemo_skills.evaluation.evaluator.audio import AudioEvaluator from nemo_skills.evaluation.evaluator.base import BaseEvaluator from nemo_skills.evaluation.evaluator.bfcl import eval_bfcl from nemo_skills.evaluation.evaluator.code import ( @@ -66,6 +67,7 @@ "code_exec": CodeExecEvaluator, "ioi": IOIEvaluator, "icpc": ICPCEvaluator, + "audio": AudioEvaluator, } # Validation: Ensure no overlap between class and function maps diff --git a/nemo_skills/evaluation/evaluator/audio.py b/nemo_skills/evaluation/evaluator/audio.py new file mode 100644 index 0000000000..7a087831a8 --- /dev/null +++ b/nemo_skills/evaluation/evaluator/audio.py @@ -0,0 +1,387 @@ +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Audio evaluation framework supporting ASR, ASR-PC, Translation, CER, and more.""" + +import asyncio +import logging +import re +from typing import Any + +import numpy as np + +from nemo_skills.evaluation.evaluator.base import BaseEvaluator, BaseEvaluatorConfig +from nemo_skills.utils import get_logger_name, nested_dataclass + +LOG = logging.getLogger(get_logger_name(__file__)) + + +@nested_dataclass(kw_only=True) +class AudioEvaluatorConfig(BaseEvaluatorConfig): + """Configuration for audio evaluation.""" + + prompt_config: str = "eval/speechlm/audio" + apply_whisper_normalization: bool = True + normalize_asr_pc_standard_wer: bool = True + + +def normalize_whitespace(text: str) -> str: + """Normalize multiple spaces to single space.""" + return re.sub(r"\s+", " ", text).strip() + + +def split_tokens(text: str) -> list[str]: + """Split text into words and punctuation as separate tokens.""" + return re.findall(r"\w+|[^\w\s]", text) + + +def extract_punctuation(text: str) -> list[str]: + """Extract only punctuation characters from text.""" + return [c for c in text if not c.isalnum() and not c.isspace()] + + +def calculate_per(reference: str, hypothesis: str) -> float: + """Calculate Punctuation Error Rate (PER): (I+D+S) / (I+D+S+C)""" + ref_punct = extract_punctuation(reference) + hyp_punct = extract_punctuation(hypothesis) + + len_r, len_h = len(ref_punct), len(hyp_punct) + + if len_r == 0 and len_h == 0: + return 0.0 + + dp = np.zeros((len_r + 1, len_h + 1, 4), dtype=int) + + for i in range(1, len_r + 1): + dp[i, 0][2] = i + for j in range(1, len_h + 1): + dp[0, j][3] = j + + for i in range(1, len_r + 1): + for j in range(1, len_h + 1): + if ref_punct[i - 1] == hyp_punct[j - 1]: + dp[i, j] = dp[i - 1, j - 1].copy() + dp[i, j][0] += 1 + else: + sub = dp[i - 1, j - 1].copy() + sub[1] += 1 + delete = dp[i - 1, j].copy() + delete[2] += 1 + insert = dp[i, j - 1].copy() + insert[3] += 1 + dp[i, j] = min([sub, delete, insert], key=lambda x: x[1] + x[2] + x[3]) + + correct, substitution, deletion, insertion = dp[len_r, len_h] + total = correct + substitution + deletion + insertion + per = (substitution + deletion + insertion) / total if total > 0 else 0.0 + return per + + +def evaluate_asr_pc(reference: str, hypothesis: str, normalize_standard_wer: bool = True) -> dict[str, Any]: + """Evaluate ASR-PC: computes WER, WER_C, WER_PC, PER.""" + import jiwer + + ref_pc = normalize_whitespace(reference) + hyp_pc = normalize_whitespace(hypothesis) + + ref_tokens = split_tokens(ref_pc) + hyp_tokens = split_tokens(hyp_pc) + wer_pc = jiwer.wer(" ".join(ref_tokens), " ".join(hyp_tokens)) + + ref_c = normalize_whitespace(re.sub(r"[^\w\s]", "", reference)) + hyp_c = normalize_whitespace(re.sub(r"[^\w\s]", "", hypothesis)) + wer_c = jiwer.wer(ref_c, hyp_c) + + if normalize_standard_wer: + ref_std = preprocess_asr_text(reference) + hyp_std = preprocess_asr_text(hypothesis) + else: + ref_std = normalize_whitespace(re.sub(r"[^\w\s]", "", reference.lower())) + hyp_std = normalize_whitespace(re.sub(r"[^\w\s]", "", hypothesis.lower())) + + wer_std = jiwer.wer(ref_std, hyp_std) + per = calculate_per(reference, hypothesis) + + return { + "wer": wer_std, + "wer_c": wer_c, + "wer_pc": wer_pc, + "per": per, + "is_correct": wer_pc < 0.5, + } + + +def preprocess_asr_text(text: str) -> str: + """Apply Whisper-style normalization: lowercase, normalize, remove brackets.""" + from whisper.normalizers import EnglishTextNormalizer + + text = text.lower() + text = EnglishTextNormalizer()(text) + text = re.sub(r"(\[|\(|\{|\<)[^\(\)\\n\[\]]*(\]|\)|\}|\>)", "", text) + text = re.sub(r"\s+", " ", text).strip() + return text + + +def preprocess_hf_leaderboard(text: str) -> str: + """Apply HuggingFace leaderboard normalization: lowercase, remove punctuation, normalize unicode.""" + import unicodedata + + text = unicodedata.normalize("NFC", text) + text = text.lower() + text = re.sub(r"[^\w\s]", "", text) + text = re.sub(r"\s+", " ", text).strip() + return text + + +def evaluate_asr(reference: str, hypothesis: str, apply_normalization: bool = True) -> dict[str, Any]: + """Evaluate ASR: computes WER with optional Whisper normalization.""" + import jiwer + + if apply_normalization: + ref = preprocess_asr_text(reference) + hyp = preprocess_asr_text(hypothesis) + else: + ref = normalize_whitespace(reference) + hyp = normalize_whitespace(hypothesis) + + if not ref: + ref = "empty" + if not hyp: + hyp = "empty" + + wer_score = jiwer.wer(ref, hyp) + + return { + "wer": wer_score, + "is_correct": wer_score < 0.5, + } + + +def evaluate_asr_leaderboard(reference: str, hypothesis: str) -> dict[str, Any]: + """Evaluate ASR with HuggingFace leaderboard preprocessing for direct comparison.""" + import jiwer + + ref = preprocess_hf_leaderboard(reference) + hyp = preprocess_hf_leaderboard(hypothesis) + + if not ref: + ref = "empty" + if not hyp: + hyp = "empty" + + wer_score = jiwer.wer(ref, hyp) + + return { + "wer": wer_score, + "is_correct": wer_score < 0.5, + } + + +def evaluate_translation(reference: str, hypothesis: str) -> dict[str, Any]: + """Evaluate translation: computes sentence-level BLEU score.""" + try: + import sacrebleu + + ref = [reference.strip()] + hyp = hypothesis.strip() + bleu = sacrebleu.sentence_bleu(hyp, ref) + bleu_score = bleu.score / 100.0 + + return { + "bleu": bleu_score, + "is_correct": bleu_score > 0.3, + } + except Exception as e: + return { + "bleu": 0.0, + "is_correct": False, + "error": str(e), + } + + +def evaluate_cer(reference: str, hypothesis: str) -> dict[str, Any]: + """Evaluate CER: character-level edit distance.""" + import jiwer + + cer_score = jiwer.cer(reference, hypothesis) + return { + "cer": cer_score, + "is_correct": cer_score < 0.5, + } + + +def evaluate_hallucination(reference: str, hypothesis: str, audio_context: dict = None) -> dict[str, Any]: + """Detect potential hallucinations via speaking rate anomaly. + + Normal speech: ~10-15 chars/second. Higher rates suggest repetition/hallucination. + Requires audio_duration in audio_context. + """ + audio_duration = audio_context.get("audio_duration") if audio_context else None + + if not audio_duration or audio_duration <= 0: + return { + "hallucination_rate": 0.0, + "char_rate": 0.0, + "is_correct": True, + "error": "missing_audio_duration", + } + + char_count = len(hypothesis) + char_rate = char_count / audio_duration + + # Hallucination threshold: >25 chars/sec (too fast = likely repetition) + is_hallucinating = char_rate > 25.0 + + return { + "hallucination_rate": 1.0 if is_hallucinating else 0.0, + "char_rate": round(char_rate, 2), + "is_correct": not is_hallucinating, + } + + +def evaluate_pc_rate(reference: str, hypothesis: str) -> dict[str, Any]: + """Evaluate detailed Punctuation and Capitalization metrics.""" + # Extract punctuation with positions + ref_puncts = [(m.group(), m.start()) for m in re.finditer(r"[.,!?;:\-]", reference)] + hyp_puncts = [(m.group(), m.start()) for m in re.finditer(r"[.,!?;:\-]", hypothesis)] + + # Punctuation matching (within 2 char tolerance) + matched = 0 + for ref_p, ref_pos in ref_puncts: + for hyp_p, hyp_pos in hyp_puncts: + if ref_p == hyp_p and abs(ref_pos - hyp_pos) <= 2: + matched += 1 + break + + punct_precision = matched / len(hyp_puncts) if hyp_puncts else 0.0 + punct_recall = matched / len(ref_puncts) if ref_puncts else 0.0 + punct_f1 = ( + 2 * punct_precision * punct_recall / (punct_precision + punct_recall) + if (punct_precision + punct_recall) > 0 + else 0.0 + ) + + # Capitalization: check sentence starts and word capitals + ref_words = reference.split() + hyp_words = hypothesis.split() + + if len(ref_words) != len(hyp_words): + cap_accuracy = 0.0 + else: + cap_matches = sum( + 1 for r, h in zip(ref_words, hyp_words, strict=True) if r and h and r[0].isupper() == h[0].isupper() + ) + cap_accuracy = cap_matches / len(ref_words) if ref_words else 0.0 + + # Overall PC rate (average of punct F1 and cap accuracy) + pc_rate = (punct_f1 + cap_accuracy) / 2.0 + + return { + "pc_rate": round(pc_rate, 3), + "punct_precision": round(punct_precision, 3), + "punct_recall": round(punct_recall, 3), + "punct_f1": round(punct_f1, 3), + "cap_accuracy": round(cap_accuracy, 3), + "is_correct": pc_rate > 0.5, + } + + +class AudioEvaluator(BaseEvaluator): + """Audio evaluator supporting ASR, ASR-PC, Translation, CER, etc.""" + + def __init__(self, config: dict, num_parallel_requests=10): + super().__init__(config, num_parallel_requests) + self.eval_config = AudioEvaluatorConfig(**self.config) + + async def eval_single(self, data_point: dict[str, any]) -> dict[str, any]: + """Evaluate single audio sample - can be called during generation. + + Returns dict of updates to be merged into data_point by BaseEvaluator. + """ + return evaluate_sample(data_point, self.eval_config) + + +def eval_audio(cfg): + """Function wrapper for backward compatibility.""" + evaluator = AudioEvaluator(cfg) + asyncio.run(evaluator.eval_full()) + + +def evaluate_sample(sample: dict[str, Any], config: AudioEvaluatorConfig) -> dict[str, Any]: + """Evaluate single sample based on task_type. Returns dict of updates to merge.""" + updates = {} + task_type = sample.get("task_type", "unknown") + generation = sample.get("generation", "").strip() + expected_answer = sample.get("expected_answer", "").strip() + + if task_type in ["ASR", "ASR-PC", "AST", "CER", "ASR_LEADERBOARD"] and not generation: + return { + "is_correct": False, + "wer": 1.0, + "error": "missing_generation", + "predicted_answer": "", + } + + if task_type == "ASR-PC": + metrics = evaluate_asr_pc( + expected_answer, generation, normalize_standard_wer=config.normalize_asr_pc_standard_wer + ) + updates.update(metrics) + updates["predicted_answer"] = generation + + elif task_type == "ASR": + metrics = evaluate_asr(expected_answer, generation, apply_normalization=config.apply_whisper_normalization) + updates.update(metrics) + updates["predicted_answer"] = generation + + elif task_type == "ASR_LEADERBOARD": + metrics = evaluate_asr_leaderboard(expected_answer, generation) + updates.update(metrics) + updates["predicted_answer"] = generation + + elif task_type == "AST": + metrics = evaluate_translation(expected_answer, generation) + updates.update(metrics) + updates["predicted_answer"] = generation + + elif task_type == "CER": + metrics = evaluate_cer(expected_answer, generation) + updates.update(metrics) + updates["predicted_answer"] = generation + + elif task_type == "Hallucination": + audio_context = {"audio_duration": sample.get("audio_duration")} + metrics = evaluate_hallucination(expected_answer, generation, audio_context) + updates.update(metrics) + updates["predicted_answer"] = generation + + elif task_type == "PC-Rate": + metrics = evaluate_pc_rate(expected_answer, generation) + updates.update(metrics) + updates["predicted_answer"] = generation + + else: + if "requires_judge" not in sample: + updates["requires_judge"] = True + updates["predicted_answer"] = generation + if "is_correct" not in sample: + updates["is_correct"] = False + + audio_duration = sample.get("audio_duration", None) + if audio_duration and audio_duration > 0 and expected_answer and generation: + updates["ref_char_rate"] = len(expected_answer) / audio_duration + updates["hyp_char_rate"] = len(generation) / audio_duration + updates["char_rate_diff"] = abs(updates["hyp_char_rate"] - updates["ref_char_rate"]) + + return updates diff --git a/nemo_skills/evaluation/metrics/audio_metrics.py b/nemo_skills/evaluation/metrics/audio_metrics.py new file mode 100644 index 0000000000..a00a53f938 --- /dev/null +++ b/nemo_skills/evaluation/metrics/audio_metrics.py @@ -0,0 +1,375 @@ +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Audio metrics: Metrics aggregation for audio evaluation tasks. + +This module provides comprehensive metrics tracking and aggregation for various +audio-related evaluation tasks. It supports automatic metrics (WER, CER, BLEU, etc.) +as well as judge-based evaluation for open-ended audio tasks. + +Supported Metrics: +- WER: Word Error Rate (standard ASR metric) +- WER_C: WER with capitalization +- WER_PC: WER with punctuation and capitalization +- PER: Punctuation Error Rate +- BLEU: Translation quality metric +- CER: Character Error Rate (for character-level evaluation) +- Hallucination Rate: Detection of hallucinated content +- PC Rate: Punctuation/Capitalization recovery rate + +The metrics class is designed to be extensible, allowing easy addition of new +audio-specific metrics as the field evolves. +""" + +import logging + +from nemo_skills.evaluation.metrics.base import BaseMetrics, as_int, as_percentage +from nemo_skills.utils import get_logger_name + +LOG = logging.getLogger(get_logger_name(__file__)) + + +class AudioMetrics(BaseMetrics): + """Metrics class for audio evaluation tasks. + + This class tracks and aggregates various audio-specific metrics including + error rates (WER, CER, PER), quality scores (BLEU), and advanced metrics + like hallucination detection. It extends BaseMetrics to provide consistent + metric computation and reporting across different audio tasks. + """ + + def __init__(self, compute_no_answer: bool = True, max_k: int = 1): + """Initialize audio metrics with tracking lists for all supported metrics. + + Args: + compute_no_answer: Whether to compute no_answer statistics + max_k: Maximum k for pass@k and majority@k evaluation + """ + super().__init__(compute_no_answer=compute_no_answer) + self.max_k = max_k + + # Core audio metrics + self.wer_scores = [] + self.wer_c_scores = [] + self.wer_pc_scores = [] + self.per_scores = [] + self.bleu_scores = [] + + # Extended metrics + self.cer_scores = [] + self.hallucination_scores = [] + self.pc_rate_scores = [] + self.punct_f1_scores = [] + self.cap_accuracy_scores = [] + self.char_rate_scores = [] + + def _extract_judge_result(self, judgement_text: str) -> bool: + """Extract judge result from judgement text. + + Parses LLM judge output to determine if the response is correct. + + Args: + judgement_text: Text output from LLM judge + + Returns: + True if judge indicates correct, False otherwise + """ + import re + + if re.search(r"\byes\b", judgement_text, re.IGNORECASE): + return True + elif re.search(r"\bno\b", judgement_text, re.IGNORECASE): + return False + else: + return False + + def _get_score_dict(self, prediction: dict) -> dict[str, bool | int | float]: + """Extract correctness scores from prediction. + + Handles both automatic metrics and judge-based evaluation, + determining the overall correctness of a prediction. + + Args: + prediction: Prediction dictionary with metrics and/or judgement + + Returns: + Dictionary with correctness scores + """ + score_dict = {} + + category = prediction.get("category", "unknown") + + if "judgement" in prediction and category == "open": + judge_result = self._extract_judge_result(prediction["judgement"]) + score_dict["judge_correct"] = judge_result + + if category == "open" and "judge_correct" in score_dict: + score_dict["correct"] = score_dict["judge_correct"] + elif "is_correct" in prediction: + score_dict["correct"] = prediction["is_correct"] + else: + score_dict["correct"] = False + + return score_dict + + def get_incorrect_sample(self, prediction: dict) -> dict: + """Return a sample marked as incorrect for all metrics. + + Used for handling error cases or missing predictions. + + Args: + prediction: Prediction dictionary + + Returns: + Updated prediction marked as incorrect + """ + prediction = prediction.copy() + prediction["is_correct"] = False + prediction["judge_correct"] = False + if not prediction.get("generation", "").strip(): + prediction["generation"] = None + return prediction + + def update_common_metrics(self, agg_dict): + """Override to always include avg_tokens even if 0 since it's in metrics_to_print. + + Updates common metrics like number of entries, average tokens, and generation time. + + Args: + agg_dict: Dictionary to update with common metrics + """ + agg_dict["num_entries"] = self.total + agg_dict["avg_tokens"] = int(self.avg_tokens / self.total) if self.total > 0 else 0 + if self.max_end_time > float("-inf") and self.min_start_time < float("inf"): + agg_dict["gen_seconds"] = int(self.max_end_time - self.min_start_time) + + def update(self, predictions): + """Update metrics with new predictions. + + Collects all metric scores from predictions and updates internal tracking lists. + Supports both existing metrics (WER, BLEU) and new metrics (CER, hallucination, PC rate). + + Args: + predictions: List of prediction dictionaries with computed metrics + """ + super().update(predictions) + + predicted_answers = [pred.get("generation", "").strip() or None for pred in predictions] + + # Collect existing metrics: WER, PnC, and BLEU scores + for pred in predictions: + if "wer" in pred and pred["wer"] is not None: + self.wer_scores.append(pred["wer"]) + if "wer_c" in pred and pred["wer_c"] is not None: + self.wer_c_scores.append(pred["wer_c"]) + if "wer_pc" in pred and pred["wer_pc"] is not None: + self.wer_pc_scores.append(pred["wer_pc"]) + if "per" in pred and pred["per"] is not None: + self.per_scores.append(pred["per"]) + if "bleu" in pred and pred["bleu"] is not None: + self.bleu_scores.append(pred["bleu"]) + + # Collect extended metrics + if "cer" in pred and pred["cer"] is not None: + self.cer_scores.append(pred["cer"]) + if "hallucination_rate" in pred and pred["hallucination_rate"] is not None: + self.hallucination_scores.append(pred["hallucination_rate"]) + if "pc_rate" in pred and pred["pc_rate"] is not None: + self.pc_rate_scores.append(pred["pc_rate"]) + if "punct_f1" in pred and pred["punct_f1"] is not None: + self.punct_f1_scores.append(pred["punct_f1"]) + if "cap_accuracy" in pred and pred["cap_accuracy"] is not None: + self.cap_accuracy_scores.append(pred["cap_accuracy"]) + if "char_rate" in pred and pred["char_rate"] is not None: + self.char_rate_scores.append(pred["char_rate"]) + + self._compute_pass_at_k(predictions=predictions, predicted_answers=predicted_answers) + self._compute_majority_at_k(predictions=predictions, predicted_answers=predicted_answers) + + def get_metrics(self): + """Get computed metrics. + + Aggregates all collected metric scores and computes averages. + Converts error rates and scores to percentages for reporting. + + Returns: + Dictionary of aggregated metrics by evaluation mode + """ + metrics_dict = super().get_metrics() + + for _agg_mode, agg_metrics in metrics_dict.items(): + if "no_answer" in agg_metrics: + # Divide by 2.0 to compensate for double-counting in base metrics #MEL + agg_metrics["no_answer"] = agg_metrics["no_answer"] / 2.0 + + # Set success_rate based on correct field + if "correct" in agg_metrics: + agg_metrics["success_rate"] = agg_metrics["correct"] + elif "judge_correct" in agg_metrics: + agg_metrics["success_rate"] = agg_metrics["judge_correct"] + + # Add existing metrics: WER, PnC, and BLEU if available (convert to percentages and round to 2 decimals) + if self.wer_scores: + agg_metrics["wer"] = round(100.0 * sum(self.wer_scores) / len(self.wer_scores), 2) + if self.wer_c_scores: + agg_metrics["wer_c"] = round(100.0 * sum(self.wer_c_scores) / len(self.wer_c_scores), 2) + if self.wer_pc_scores: + agg_metrics["wer_pc"] = round(100.0 * sum(self.wer_pc_scores) / len(self.wer_pc_scores), 2) + if self.per_scores: + agg_metrics["per"] = round(100.0 * sum(self.per_scores) / len(self.per_scores), 2) + if self.bleu_scores: + agg_metrics["bleu"] = round(100.0 * sum(self.bleu_scores) / len(self.bleu_scores), 2) + + # Add extended metrics if available + if self.cer_scores: + agg_metrics["cer"] = round(100.0 * sum(self.cer_scores) / len(self.cer_scores), 2) + if self.hallucination_scores: + agg_metrics["hallucination_rate"] = round( + 100.0 * sum(self.hallucination_scores) / len(self.hallucination_scores), 2 + ) + if self.pc_rate_scores: + agg_metrics["pc_rate"] = round(100.0 * sum(self.pc_rate_scores) / len(self.pc_rate_scores), 2) + if self.punct_f1_scores: + agg_metrics["punct_f1"] = round(100.0 * sum(self.punct_f1_scores) / len(self.punct_f1_scores), 2) + if self.cap_accuracy_scores: + agg_metrics["cap_accuracy"] = round( + 100.0 * sum(self.cap_accuracy_scores) / len(self.cap_accuracy_scores), 2 + ) + if self.char_rate_scores: + agg_metrics["char_rate"] = round(sum(self.char_rate_scores) / len(self.char_rate_scores), 2) + + return metrics_dict + + def evaluations_to_print(self): + """Specify which evaluation modes to print. + + Returns: + List of evaluation mode names for display + """ + evals = [f"pass@{self.max_k}"] + if self.max_k > 1: + evals.extend([f"majority@{self.max_k}", f"pass@1[avg-of-{self.max_k}]"]) + return evals + + def metrics_to_print(self): + """Specify which metrics to print. + + Dynamically includes only the metrics that were actually computed + based on the task types in the evaluation. + + Returns: + Dictionary mapping metric names to formatting functions + """ + base_metrics = { + "avg_tokens": as_int, + "gen_seconds": as_int, + "success_rate": as_percentage, + } + + if self.compute_no_answer: + base_metrics["no_answer"] = as_percentage + + # Add existing metrics if they were computed + if self.wer_scores: + base_metrics["wer"] = as_percentage + if self.wer_c_scores: + base_metrics["wer_c"] = as_percentage + if self.wer_pc_scores: + base_metrics["wer_pc"] = as_percentage + if self.per_scores: + base_metrics["per"] = as_percentage + if self.bleu_scores: + base_metrics["bleu"] = as_percentage + + # Add extended metrics if they were computed + if self.cer_scores: + base_metrics["cer"] = as_percentage + if self.hallucination_scores: + base_metrics["hallucination_rate"] = as_percentage + if self.pc_rate_scores: + base_metrics["pc_rate"] = as_percentage + if self.punct_f1_scores: + base_metrics["punct_f1"] = as_percentage + if self.cap_accuracy_scores: + base_metrics["cap_accuracy"] = as_percentage + if self.char_rate_scores: + base_metrics["char_rate"] = as_int + + base_metrics["num_entries"] = as_int # Add at end for better display order + + return base_metrics + + +def compute_score(combined_metrics: dict) -> dict: + """ + Aggregate metrics from multiple sub-benchmarks into a single group score. + + This function is used for benchmark groups that contain multiple sub-benchmarks. + It computes weighted averages across all sub-benchmarks based on the number of entries. + + Args: + combined_metrics: Dictionary with benchmark names as keys. + Each benchmark has eval modes (e.g., 'pass@1') as keys, + which contain the actual metrics. + Format: {benchmark_name: {eval_mode: {metrics...}}} + + Returns: + Aggregated metrics dictionary in the same format, with weighted averages + computed across all sub-benchmarks. + """ + # Identify main benchmark categories (nonjudge, judge) + main_benchmark_names = ["nonjudge", "judge"] + benchmarks = {k: v for k, v in combined_metrics.items() if k.split(".")[-1] in main_benchmark_names} + + if not benchmarks: + return {} + + # Get all eval modes from first benchmark (they should all have the same modes) + first_benchmark = next(iter(benchmarks.values())) + eval_modes = list(first_benchmark.keys()) + + # Aggregate metrics for each evaluation mode + aggregated = {} + for eval_mode in eval_modes: + total_entries = 0 + weighted_success = 0.0 + total_gen_seconds = 0 + weighted_tokens = 0.0 + weighted_no_answer = 0.0 + + for benchmark_name, benchmark_data in benchmarks.items(): + if eval_mode not in benchmark_data: + continue + + metrics = benchmark_data[eval_mode] + num_entries = metrics.get("num_entries", 0) + total_entries += num_entries + + # Aggregate weighted by number of entries (metrics are already percentages) + if num_entries > 0: + weighted_success += metrics.get("success_rate", 0.0) * num_entries + total_gen_seconds += metrics.get("gen_seconds", 0) + weighted_tokens += metrics.get("avg_tokens", 0.0) * num_entries + weighted_no_answer += metrics.get("no_answer", 0.0) * num_entries + + # Compute aggregated metrics + aggregated[eval_mode] = { + "avg_tokens": int(weighted_tokens / total_entries) if total_entries > 0 else 0, + "gen_seconds": total_gen_seconds, + "success_rate": weighted_success / total_entries if total_entries > 0 else 0.0, + "no_answer": weighted_no_answer / total_entries if total_entries > 0 else 0.0, + "num_entries": total_entries, + } + + return aggregated diff --git a/nemo_skills/evaluation/metrics/map_metrics.py b/nemo_skills/evaluation/metrics/map_metrics.py index 83e2495b9f..94cf9b8c73 100644 --- a/nemo_skills/evaluation/metrics/map_metrics.py +++ b/nemo_skills/evaluation/metrics/map_metrics.py @@ -19,6 +19,7 @@ from nemo_skills.evaluation.metrics.aalcr_metrics import AALCRMetrics from nemo_skills.evaluation.metrics.answer_judgement_metrics import AnswerJudgementMetrics from nemo_skills.evaluation.metrics.arena_metrics import ArenaMetrics +from nemo_skills.evaluation.metrics.audio_metrics import AudioMetrics from nemo_skills.evaluation.metrics.bfcl_metrics import BFCLMetrics from nemo_skills.evaluation.metrics.code_metrics import ( BigCodeBenchMetrics, @@ -47,6 +48,7 @@ "lean4-statement": Lean4Metrics, "answer-judgement": AnswerJudgementMetrics, "arena": ArenaMetrics, + "audio": AudioMetrics, "bfcl": BFCLMetrics, "evalplus": EvalPlusMetrics, "if": IFMetrics, From 5738fee1ca97e6df332840750a9f509ff1bb1ac3 Mon Sep 17 00:00:00 2001 From: George Armstrong Date: Thu, 11 Dec 2025 12:50:09 -0800 Subject: [PATCH 42/88] FEAT Add Tavily Search (#1085) Signed-off-by: George Armstrong Co-authored-by: Sanyam Kapoor Signed-off-by: Cheng-Ping Hsieh --- nemo_skills/mcp/servers/tavily_search_tool.py | 142 ++++++++++++++++++ nemo_skills/mcp/tool_manager.py | 4 + nemo_skills/mcp/tool_providers.py | 3 + 3 files changed, 149 insertions(+) create mode 100644 nemo_skills/mcp/servers/tavily_search_tool.py diff --git a/nemo_skills/mcp/servers/tavily_search_tool.py b/nemo_skills/mcp/servers/tavily_search_tool.py new file mode 100644 index 0000000000..ef4ca54ff5 --- /dev/null +++ b/nemo_skills/mcp/servers/tavily_search_tool.py @@ -0,0 +1,142 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import json +import logging +import os +from dataclasses import dataclass +from typing import Annotated, Any + +import httpx +from mcp.server.fastmcp import FastMCP +from pydantic import Field + +from nemo_skills.mcp.tool_providers import MCPClientTool + +logger = logging.getLogger(__name__) + + +@dataclass +class ExecutionResult: + error: str | None = None + result: str | None = None + + +mcp = FastMCP(name="tavily") + +# Populated from CLI args in main() +TAVILY_API_KEY: str | None = None + +EXCLUDE_DOMAINS: list[str] | None = None + + +## See docs https://docs.tavily.com/documentation/api-reference/endpoint/search +## There is also a hosted MCP that can be used instead of this tool: https://github.com/tavily-ai/tavily-mcp?tab=readme-ov-file#remote-mcp-server +@mcp.tool(name="tavily-search") +async def answer( + query: Annotated[str, Field(description="Search query.")], + exclude_domains: Annotated[list[str], Field(description="Domains to exclude from the search.")] = [], +): + """Get a summary of search results from the web using Tavily.""" + + api_url = "https://api.tavily.com/search" + + headers = { + "Authorization": f"Bearer {TAVILY_API_KEY}", + "Content-Type": "application/json", + } + + payload = { + "query": query, + # "auto_parameters": False, + "search_depth": "basic", + "include_answer": "basic", ## or advanced. + # this should be statically set to the domains we want to exclude + "exclude_domains": exclude_domains, + } + + async with httpx.AsyncClient() as client: + response = await client.post(api_url, headers=headers, json=payload) + if response.status_code != 200: + return {"error": response.json()["error"]} + + result = response.json()["answer"] + + return result + + +def _parse_exclude_domains(exclude_config: dict) -> list[str]: + exclude_domains = [] + # this is pretty hard-coded so we ensure the file structure is correct + notices = exclude_config["notices"] + for notice in notices: + for prop in notice["properties"]: + if prop.get("type") == "domain": + exclude_domains.append(prop["value"]) + return exclude_domains + + +class TavilySearchTool(MCPClientTool): + def __init__(self) -> None: + super().__init__() + self.apply_config_updates( + { + "client": "nemo_skills.mcp.clients.MCPStdioClient", + "client_params": { + "command": "python", + "args": ["-m", "nemo_skills.mcp.servers.tavily_search_tool"], + }, + "hide_args": { + "tavily-search": ["exclude_domains"], + }, + "exclude_domains_config": None, + } + ) + + def post_configure(self) -> None: + # Required the exclude domains to be set--we do not want to accidentally include all domains + if (conf := self._config.get("exclude_domains_config")) is not None: + with open(conf, "r") as f: + exlude_config = json.load(f) + self.exclude_domains = _parse_exclude_domains(exlude_config) + else: + raise ValueError("exclude_domains_config is not set") + + async def execute(self, tool_name: str, arguments: dict[str, Any], extra_args: dict[str, Any] | None = None): + arguments = dict(arguments) + merged_extra = dict(extra_args or {}) + if not hasattr(self, "exclude_domains"): + raise ValueError("exclude_domains_config is not set") + merged_extra["exclude_domains"] = self.exclude_domains + result = await self._client.call_tool(tool=tool_name, args=arguments, extra_args=merged_extra) + return result + + +def main(): + parser = argparse.ArgumentParser(description="MCP server for Tavily web search tool") + parser.add_argument("--api-key", type=str, default=os.getenv("TAVILY_API_KEY"), help="Tavily API Key") + args = parser.parse_args() + + if not args.api_key: + raise ValueError("Missing Tavily API key.") + + global TAVILY_API_KEY + TAVILY_API_KEY = args.api_key + + mcp.run(transport="stdio") + + +if __name__ == "__main__": + main() diff --git a/nemo_skills/mcp/tool_manager.py b/nemo_skills/mcp/tool_manager.py index 8e192d5d83..2d98cdd772 100644 --- a/nemo_skills/mcp/tool_manager.py +++ b/nemo_skills/mcp/tool_manager.py @@ -60,6 +60,9 @@ async def execute( async def shutdown(self) -> None: # Optional hook return None + def post_configure(self) -> None: + return None + class ToolManager: """Registry/Router for module-based tools. @@ -98,6 +101,7 @@ def __init__( raise ValueError(f"Duplicate tool class registered: '{provider_key}'") tool.configure((overrides.get(provider_key) if overrides else None), context) + tool.post_configure() self._tools[provider_key] = tool async def shutdown(self) -> None: diff --git a/nemo_skills/mcp/tool_providers.py b/nemo_skills/mcp/tool_providers.py index bf03bb2728..a2c940b4c3 100644 --- a/nemo_skills/mcp/tool_providers.py +++ b/nemo_skills/mcp/tool_providers.py @@ -71,6 +71,9 @@ def _resolve_maybe_callable(self, value: Any): return value return value + def post_configure(self) -> None: + pass + def configure(self, overrides: Dict[str, Any] | None = None, context: Dict[str, Any] | None = None) -> None: cfg = dict(self._config) if overrides: From a833c4bc837c42bf07047b56fa12d2e81056d25a Mon Sep 17 00:00:00 2001 From: Wasi Ahmad Date: Thu, 11 Dec 2025 14:40:59 -0800 Subject: [PATCH 43/88] updating code extraction logic (#1086) Signed-off-by: wasiahmad Signed-off-by: George Armstrong Co-authored-by: George Armstrong Signed-off-by: Cheng-Ping Hsieh --- nemo_skills/evaluation/evaluator/code.py | 94 ++++++++++++------------ 1 file changed, 46 insertions(+), 48 deletions(-) diff --git a/nemo_skills/evaluation/evaluator/code.py b/nemo_skills/evaluation/evaluator/code.py index 4a0af0c6f8..3800495d28 100644 --- a/nemo_skills/evaluation/evaluator/code.py +++ b/nemo_skills/evaluation/evaluator/code.py @@ -15,7 +15,6 @@ import asyncio import json import logging -import re import shutil import subprocess import sys @@ -117,58 +116,57 @@ async def eval_full(self): # type: ignore[override] def preprocess_code(generation_dict: dict, language="python", strip_whitespace=True): - completion = generation_dict["generation"] - if strip_whitespace: - completion = completion.strip() + completion = generation_dict.get("generation", "") or "" completion = completion.replace("\r", "") - ##### To handle code generation by reasoning models - # check for and tags + # --------------------------------------------------------- + # 1. Handle reasoning traces: ... + # --------------------------------------------------------- if "" in completion: - if "" in completion: - # thinking trace completed, solution in after the trace - match = re.search(r"\s*(.*)", completion, re.DOTALL) - completion = match.group(1).strip() if match else None + # partition is faster than regex and avoids imports + _, separator, post_thought = completion.partition("") + if separator: + # Keep content after the closing tag + completion = post_thought else: - completion = None - - if completion is None: - generation_dict["completion"] = "" # no valid solution generated - return generation_dict - ##### - - start_with_lang_tag = f"```{language}" - generic_start_end_tag = "```" + # opened but never closed -> Invalid generation + generation_dict["completion"] = "" + return generation_dict + + # --------------------------------------------------------- + # 2. Extract fenced code block + # --------------------------------------------------------- + specific_fence = f"```{language}" + generic_fence = "```" + + # Find the *last* occurrence of the code block (handles CoT steps) + start_index = completion.rfind(specific_fence) + fence_len = len(specific_fence) + + # Fallback to generic fence if specific language tag is missing + if start_index == -1: + start_index = completion.rfind(generic_fence) + fence_len = len(generic_fence) + + if start_index != -1: + # Move past the opening fence + content_start = start_index + fence_len + completion = completion[content_start:] + + # Check for closing fence + end_index = completion.find(generic_fence) + if end_index != -1: + # Valid block found + completion = completion[:end_index] + else: + # STRICT MODE: Opening fence found, but no closing fence. + # The generation is truncated/incomplete. Discard it. + completion = "" - if start_with_lang_tag in completion: - def_line = completion.index(start_with_lang_tag) + len(start_with_lang_tag) - completion = completion[def_line:] - if strip_whitespace: - completion = completion.strip() - try: - next_line = completion.index(generic_start_end_tag) - completion = completion[:next_line] - if strip_whitespace: - completion = completion.strip() - except Exception: - print(completion) - print("================\n") - - elif generic_start_end_tag in completion: - def_line = completion.index(generic_start_end_tag) + len(generic_start_end_tag) - completion = completion[def_line:] - if strip_whitespace: - completion = completion.strip() - try: - next_line = completion.index(generic_start_end_tag) - completion = completion[:next_line] - if strip_whitespace: - completion = completion.strip() - except Exception: - print(completion) - print("================\n") - - if completion.startswith(" ") and strip_whitespace: + # --------------------------------------------------------- + # 3. Final Cleanup (The only strip that matters) + # --------------------------------------------------------- + if strip_whitespace: completion = completion.strip() generation_dict["completion"] = completion From 3e201c88bb57fd19cacedcda5a667a59098c2c56 Mon Sep 17 00:00:00 2001 From: Jiacheng Xu Date: Fri, 12 Dec 2025 08:57:43 +0800 Subject: [PATCH 44/88] Sandbox add stem (#1101) Signed-off-by: Jiacheng Xu Signed-off-by: George Armstrong Co-authored-by: Jiacheng Xu Co-authored-by: George Armstrong Signed-off-by: Cheng-Ping Hsieh --- requirements/stem.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/requirements/stem.txt b/requirements/stem.txt index c86cb3b4e5..e1ea1209a5 100644 --- a/requirements/stem.txt +++ b/requirements/stem.txt @@ -38,7 +38,6 @@ dipy dp_accounting duckduckgo_search easyocr -ecdsa enchant ephem ete3 @@ -103,7 +102,6 @@ ocl open_tamil opencv-python openmc_data -openmm openpyxl optopy ortools From 07986adcb769e93e518cf6b27b9e91571cae930d Mon Sep 17 00:00:00 2001 From: Mateusz Winiarek <72758259+Froxyy-dev@users.noreply.github.com> Date: Fri, 12 Dec 2025 18:21:39 +0100 Subject: [PATCH 45/88] Handle none output in wmtp24++ (#1091) Signed-off-by: George Armstrong Signed-off-by: Mateusz Winiarek Co-authored-by: George Armstrong Signed-off-by: Cheng-Ping Hsieh --- nemo_skills/evaluation/metrics/translation_metrics.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nemo_skills/evaluation/metrics/translation_metrics.py b/nemo_skills/evaluation/metrics/translation_metrics.py index dbeadbede8..23cd9cfc66 100644 --- a/nemo_skills/evaluation/metrics/translation_metrics.py +++ b/nemo_skills/evaluation/metrics/translation_metrics.py @@ -64,6 +64,9 @@ def update(self, predictions): generation = pred["generation"] ground_truth = pred["translation"] + if generation is None: + generation = "" + self.translation_dict[f"{src_lang}->{tgt_lang}"]["preds"].append(generation) self.translation_dict[f"{src_lang}->{tgt_lang}"]["gts"].append(ground_truth) From 8a3667a5331c0e1c945d52bb5448fb35ca7baa87 Mon Sep 17 00:00:00 2001 From: George Armstrong Date: Fri, 12 Dec 2025 15:20:22 -0800 Subject: [PATCH 46/88] ENH enable sandbox env overrides in generate (#1107) Signed-off-by: George Armstrong Signed-off-by: Cheng-Ping Hsieh --- dockerfiles/Dockerfile.sandbox | 2 +- nemo_skills/pipeline/generate.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/dockerfiles/Dockerfile.sandbox b/dockerfiles/Dockerfile.sandbox index 3cccd8ee6c..ea1d35acc9 100644 --- a/dockerfiles/Dockerfile.sandbox +++ b/dockerfiles/Dockerfile.sandbox @@ -75,7 +75,7 @@ ENV PIP_DISABLE_PIP_VERSION_CHECK=1 \ # Install uv (adds to ~/.local/bin), then install deps -RUN if [ "$GITHUB_CI" != "1" ]; then \ +RUN if [ "$GITHUB_CI" != "1" ] && [ "$TARGETARCH" != "arm64" ]; then \ curl -LsSf https://astral.sh/uv/install.sh | sh && \ uv pip install --upgrade pip && \ uv pip install -r /app/stem_requirements.txt --no-cache-dir --extra-index-url https://download.pytorch.org/whl/cpu; \ diff --git a/nemo_skills/pipeline/generate.py b/nemo_skills/pipeline/generate.py index 95e37abb59..f33796d05c 100644 --- a/nemo_skills/pipeline/generate.py +++ b/nemo_skills/pipeline/generate.py @@ -57,6 +57,7 @@ def _create_commandgroup_from_config( task_name: str, log_dir: str, sbatch_kwargs: Optional[Dict] = None, + sandbox_env_overrides: Optional[List[str]] = None, ) -> CommandGroup: """Create a CommandGroup from server_config. @@ -122,6 +123,14 @@ def _create_commandgroup_from_config( cmd, metadata = sandbox_command(cluster_config=cluster_config, port=sandbox_port) metadata["log_prefix"] = "sandbox" + # Apply user-specified environment overrides for the sandbox + if sandbox_env_overrides: + sandbox_env = metadata.get("environment", {}) + for override in sandbox_env_overrides: + key, value = override.split("=", 1) + sandbox_env[key] = value + metadata["environment"] = sandbox_env + sandbox_cmd = Command( command=cmd, container=cluster_config["containers"]["sandbox"], @@ -242,6 +251,11 @@ def generate( False, help="If True, will re-run jobs even if a corresponding '.done' file already exists" ), with_sandbox: bool = typer.Option(False, help="If True, will start a sandbox container alongside this job"), + sandbox_env_overrides: List[str] = typer.Option( + None, + help="Extra environment variables for the sandbox container in KEY=VALUE format. " + "E.g., --sandbox-env-overrides NEMO_SKILLS_SANDBOX_BLOCK_NETWORK=1 to enable network blocking.", + ), keep_mounts_for_sandbox: bool = typer.Option( False, help="If True, will keep the mounts for the sandbox container. Note that, it is risky given that sandbox executes LLM commands and could potentially lead to data loss. So, we advise not to use this unless absolutely necessary.", @@ -455,6 +469,7 @@ def generate( task_name=task_name, log_dir=log_dir, sbatch_kwargs=sbatch_kwargs, + sandbox_env_overrides=sandbox_env_overrides, ) # Use unique internal job name for dependency tracking, but same task_name From ed3377509ea11cb98c23d8d686dbdf384dad0903 Mon Sep 17 00:00:00 2001 From: George Armstrong Date: Mon, 15 Dec 2025 13:54:00 -0800 Subject: [PATCH 47/88] Search Tool Parameter updates (#1112) Signed-off-by: George Armstrong Signed-off-by: Cheng-Ping Hsieh --- nemo_skills/mcp/servers/tavily_search_tool.py | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/nemo_skills/mcp/servers/tavily_search_tool.py b/nemo_skills/mcp/servers/tavily_search_tool.py index ef4ca54ff5..f72f6761dd 100644 --- a/nemo_skills/mcp/servers/tavily_search_tool.py +++ b/nemo_skills/mcp/servers/tavily_search_tool.py @@ -40,18 +40,28 @@ class ExecutionResult: TAVILY_API_KEY: str | None = None EXCLUDE_DOMAINS: list[str] | None = None +MAX_NUM_RESULTS: int = 20 ## See docs https://docs.tavily.com/documentation/api-reference/endpoint/search ## There is also a hosted MCP that can be used instead of this tool: https://github.com/tavily-ai/tavily-mcp?tab=readme-ov-file#remote-mcp-server -@mcp.tool(name="tavily-search") +@mcp.tool(name="web-search") async def answer( query: Annotated[str, Field(description="Search query.")], exclude_domains: Annotated[list[str], Field(description="Domains to exclude from the search.")] = [], + num_results: Annotated[int, Field(description="Number of results to return.")] = 10, + answer_type: Annotated[ + str, + Field( + description='Type of results to return. Choose "answer" for a concise answer or "results" for a list of results.' + ), + ] = "answer", ): - """Get a summary of search results from the web using Tavily.""" + """Search the web for a query""" api_url = "https://api.tavily.com/search" + assert answer_type in ["answer", "results"], "Invalid answer type. Choose 'answer' or 'results'." + assert num_results <= MAX_NUM_RESULTS, f"Number of results must be less than or equal to {MAX_NUM_RESULTS}." headers = { "Authorization": f"Bearer {TAVILY_API_KEY}", @@ -63,6 +73,7 @@ async def answer( # "auto_parameters": False, "search_depth": "basic", "include_answer": "basic", ## or advanced. + "num_results": num_results, # this should be statically set to the domains we want to exclude "exclude_domains": exclude_domains, } @@ -72,7 +83,7 @@ async def answer( if response.status_code != 200: return {"error": response.json()["error"]} - result = response.json()["answer"] + result = response.json()[answer_type] return result @@ -99,7 +110,7 @@ def __init__(self) -> None: "args": ["-m", "nemo_skills.mcp.servers.tavily_search_tool"], }, "hide_args": { - "tavily-search": ["exclude_domains"], + "web-search": ["exclude_domains", "num_results", "answer_type"], }, "exclude_domains_config": None, } @@ -120,6 +131,9 @@ async def execute(self, tool_name: str, arguments: dict[str, Any], extra_args: d if not hasattr(self, "exclude_domains"): raise ValueError("exclude_domains_config is not set") merged_extra["exclude_domains"] = self.exclude_domains + for key in ["num_results", "answer_type"]: + if key in self._config: + merged_extra[key] = self._config[key] result = await self._client.call_tool(tool=tool_name, args=arguments, extra_args=merged_extra) return result From ebcab2ffcf4b63a4ebed8902e9ba5912de963712 Mon Sep 17 00:00:00 2001 From: Stephen Ge Date: Mon, 15 Dec 2025 17:12:22 -0500 Subject: [PATCH 48/88] autoformalize cleanup (#1098) Signed-off-by: Stephen Ge Co-authored-by: Claude Opus 4.5 Signed-off-by: Cheng-Ping Hsieh --- nemo_skills/inference/autoformalize.py | 339 ++++++++++++++++++ .../lean4/deepseek-R1-autoformalization.yaml | 24 ++ .../lean4/deepseek-R1-backtranslation.yaml | 26 ++ .../deepseek-R1-judge-backtranslation.yaml | 27 ++ .../config/lean4/refinement_code_error.yaml | 26 ++ .../lean4/refinement_consistent_error.yaml | 26 ++ .../lean4/refinement_parsing_error.yaml | 20 ++ 7 files changed, 488 insertions(+) create mode 100644 nemo_skills/inference/autoformalize.py create mode 100644 nemo_skills/prompt/config/lean4/deepseek-R1-autoformalization.yaml create mode 100644 nemo_skills/prompt/config/lean4/deepseek-R1-backtranslation.yaml create mode 100644 nemo_skills/prompt/config/lean4/deepseek-R1-judge-backtranslation.yaml create mode 100644 nemo_skills/prompt/config/lean4/refinement_code_error.yaml create mode 100644 nemo_skills/prompt/config/lean4/refinement_consistent_error.yaml create mode 100644 nemo_skills/prompt/config/lean4/refinement_parsing_error.yaml diff --git a/nemo_skills/inference/autoformalize.py b/nemo_skills/inference/autoformalize.py new file mode 100644 index 0000000000..b95737a1c3 --- /dev/null +++ b/nemo_skills/inference/autoformalize.py @@ -0,0 +1,339 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +import logging +import sys +from dataclasses import asdict, is_dataclass +from typing import List + +import hydra +from openai import BadRequestError + +from nemo_skills.code_execution.proof_utils import ( + extract_code, + move_imports_to_beginning, + refine_by_sorry, + remove_comments, +) +from nemo_skills.code_execution.sandbox import sandbox_params +from nemo_skills.inference.model import server_params +from nemo_skills.prompt.utils import get_prompt +from nemo_skills.utils import ( + get_help_message, + get_logger_name, + nested_dataclass, + parse_reasoning, + setup_logging, +) + +from .generate import GenerateSolutionsConfig, GenerationTask + +LOG = logging.getLogger(get_logger_name(__file__)) + +reasoning_effort_list = ["low", "medium", "high"] + + +@nested_dataclass(kw_only=True) +class AutoformalizeConfig(GenerateSolutionsConfig): + """LLM generation parameters.""" + + # Lean 4 specific parameters + refine_parsing_error_prompt_config: str | None = None # prompt for refining the code + refine_code_error_prompt_config: str | None = None # prompt for refining the code + refine_consistent_error_prompt_config: str | None = None # prompt for refining the code + refinement: bool = False # whether to refine the code + refinement_max_turns: int = 8 # maximum number of turns for refinement + judge_enabled: bool = False # whether to judge the code + backtranslation_prompt_config: str | None = None # prompt for backtranslation + judge_prompt_config: str | None = None # prompt for judging the code + judge_exact_match: bool = ( + True # recommend to set to true when using gpt-oss and should set to false if using deepseek + ) + adaptive_reasoning: bool = False # whether to adapt the reasoning effort + parse_generation: bool = False # whether to parse the generation + + +cs = hydra.core.config_store.ConfigStore.instance() +cs.store(name="base_generation_config", node=AutoformalizeConfig) + + +class AutoformalizeTask(GenerationTask): + def __init__(self, cfg: AutoformalizeConfig): + """ + Class that represents a generation task. It implements a template of steps to generate solutions using LLMs. + Individual functions can be overriden to customize the behavior of the generation task. + + Args: + cfg: AutoformalizeConfig object with the configuration parameters or subclass. + """ + super().__init__(cfg) + if self.cfg.refinement: + self.setup_refine_prompt() + if self.cfg.judge_enabled: + self.setup_judge_prompt() + + def setup_llm(self): + if self.cfg.code_execution: + raise ValueError( + "Code execution is not supported for autoformalization. Use sandbox config for Lean4 execution." + ) + llm = super().setup_llm() + # Validate sandbox is configured - fail early during setup rather than during generation + if self.sandbox is None: + raise ValueError( + "Sandbox is required for Lean4 code execution but was not configured. " + "Please provide sandbox configuration." + ) + return llm + + def setup_refine_prompt(self): + assert self.cfg.refine_parsing_error_prompt_config is not None, ( + "refine_parsing_error_prompt_config is required when refinement is enabled. Please set refinement=False to disable refinement." + ) + assert self.cfg.refine_code_error_prompt_config is not None, ( + "refine_code_error_prompt_config is required when refinement is enabled. Please set refinement=False to disable refinement." + ) + self.refine_parsing_error_prompt = get_prompt(self.cfg.refine_parsing_error_prompt_config) + self.refine_code_error_prompt = get_prompt(self.cfg.refine_code_error_prompt_config) + if self.cfg.judge_enabled: + assert self.cfg.refine_consistent_error_prompt_config is not None, ( + "refine_consistent_error_prompt_config is required when refinement is enabled and judge is enabled. Please set refinement=False to disable refinement." + ) + self.refine_consistent_error_prompt = get_prompt(self.cfg.refine_consistent_error_prompt_config) + + def setup_judge_prompt(self): + assert self.cfg.backtranslation_prompt_config is not None, ( + "backtranslation_prompt_config is required when judge is enabled. Please set judge_enabled=False to disable judge." + ) + assert self.cfg.judge_prompt_config is not None, ( + "judge_prompt_config is required when judge is enabled. Please set judge_enabled=False to disable judge." + ) + self.judge_prompt = get_prompt(self.cfg.judge_prompt_config) + self.backtranslation_prompt = get_prompt(self.cfg.backtranslation_prompt_config) + + def _extract_code_sync(self, completion: str): + try: + code = extract_code(completion) + if code == "None": + return None, None + clean_code = remove_comments(code) + clean_code = move_imports_to_beginning(clean_code) + clean_code = refine_by_sorry(clean_code) + except (ValueError, TypeError, AttributeError) as e: + LOG.debug("Code extraction failed: %s", e) + return None, None + else: + return code, clean_code + + async def _extract_code(self, completion: str): + # Offload the blocking work to another thread + return await asyncio.to_thread(self._extract_code_sync, completion) + + async def _backtranslate_code(self, code: str) -> str: + prompt = self.backtranslation_prompt.fill({"code": code}) + generation = await self._generate_single_completion(prompt) + return generation.get("generation") + + async def _judge_backtranslation(self, backtranslation_result: str, data_point) -> str: + prompt = self.judge_prompt.fill( + { + "backtranslation": backtranslation_result, + "problem": data_point["problem"], + } + ) + generation = await self._generate_single_completion(prompt) + return generation.get("generation") + + async def _judge_code(self, code: str | None, data_point) -> dict: + results_dict = {} + results_dict["code"] = code + results_dict["passed_compile"] = False + results_dict["backtranslation_result"] = None + results_dict["judge_result"] = None + results_dict["passed_compile_judge"] = False + results_dict["feedback"] = None + if code is None: + results_dict["parse_error"] = True + return results_dict + else: + results_dict["parse_error"] = False + + # execute_code returns (result_dict, session_id) tuple + code_execution_result, _ = await self.sandbox.execute_code( + remove_comments(code), language="lean4", timeout=600.0, max_output_characters=1000000 + ) + results_dict["code_execution_result"] = code_execution_result + + # Handle timeout (now indicated by process_status in the dict) + if code_execution_result.get("process_status") == "timeout": + results_dict["code_execution_result"] = { + "process_status": "failed", + "stdout": "Timeout error, please check for heavy computation, dead loop, etc.", + } + elif code_execution_result["process_status"] == "completed": + results_dict["passed_compile"] = True + if self.cfg.judge_enabled: + backtranslation_result = await self._backtranslate_code(code) + if backtranslation_result is not None: + results_dict["backtranslation_result"] = backtranslation_result + judge_result = await self._judge_backtranslation(backtranslation_result, data_point) + results_dict["judge_result"] = judge_result + if judge_result is not None: + if self.cfg.judge_exact_match: + if "true" == judge_result.lower().strip(): + results_dict["passed_compile_judge"] = True + else: + if "true" in judge_result.lower().strip(): + results_dict["passed_compile_judge"] = True + else: + LOG.warning("Judge failed but code compiled successfully") + results_dict["passed_compile_judge"] = False + results_dict["judge_result"] = "Backtranslation passed, but judge failed." + else: + LOG.warning("Backtranslation failed for compiled code") + results_dict["backtranslation_result"] = "Backtranslation failed." + results_dict["passed_compile_judge"] = False + else: + results_dict["passed_compile_judge"] = True + return results_dict + + def _construct_refine_prompt(self, results_dict): + if results_dict["parse_error"]: + # parse error + prompt = self.refine_parsing_error_prompt.fill({}) + elif results_dict["passed_compile"]: + # consistent error + prompt = self.refine_consistent_error_prompt.fill({"reason": results_dict["judge_result"]}) + else: + # code error + prompt = self.refine_code_error_prompt.fill( + {"error_message": results_dict["code_execution_result"]["stdout"]} + ) + return prompt + + async def _generate_single_completion(self, prompt: List[str]): + """Generate a single completion with semaphore-controlled concurrency.""" + if is_dataclass(self.cfg.inference): + inference_params = asdict(self.cfg.inference) + else: + # Already a dict from Hydra + inference_params = dict(self.cfg.inference) + generation_params = { + "prompt": prompt, + "stop_phrases": [self.cfg.stop_phrase] if self.cfg.stop_phrase else None, + **inference_params, + **self.extra_generate_params, + } + + # Use semaphore for concurrency control (inherited from GenerationTask) + async with self.semaphore: + generation = await self.llm.generate_async(**generation_params) + if self.cfg.adaptive_reasoning: + assert generation_params["extra_body"].get("reasoning_effort", None) is not None, ( + "reasoning_effort is required when adaptive_reasoning is enabled" + ) + reasoning_effort_index = reasoning_effort_list.index( + generation_params["extra_body"].get("reasoning_effort", None) + ) + while len(generation["generation"]) == 0 and reasoning_effort_index > 0: + LOG.info( + "Reasoning effort is too high, reducing to %s", + reasoning_effort_list[reasoning_effort_index - 1], + ) + reasoning_effort_index = reasoning_effort_index - 1 + generation_params["extra_body"]["reasoning_effort"] = reasoning_effort_list[reasoning_effort_index] + generation = await self.llm.generate_async(**generation_params) + + if self.cfg.parse_generation: + parse_reasoning( + generation, + self.cfg.generation_key, + self.cfg.end_reasoning_string, + ) + return generation + + async def _single_data_point_generate(self, data_point, data): + results_dict = {} + prompt_turn_list = self.fill_prompt(data_point, data) + code_list = [] + unrefined_code_list = [] + results_dict_list = [] + assert isinstance(prompt_turn_list, list), "prompt_turn_list should be a list" + results_dict["passed_compile_judge"] = False + turn_idx = 0 + + try: + for turn_idx in range(self.cfg.refinement_max_turns): + generation = await self._generate_single_completion(prompt_turn_list) + prompt_turn_list += [{"role": "assistant", "content": generation["generation"]}] + unrefined_code, code = await self._extract_code(generation["generation"]) + unrefined_code_list.append(unrefined_code) + code_list.append(code) + results_dict = await self._judge_code(code, data_point) + if "reasoning_content" in generation: + results_dict["reasoning_content_generation"] = generation["reasoning_content"] + results_dict_list.append(results_dict) + if results_dict["passed_compile_judge"]: + break + else: + if self.cfg.refinement and turn_idx < self.cfg.refinement_max_turns - 1: + prompt = self._construct_refine_prompt(results_dict) + results_dict["feedback"] = prompt + prompt_turn_list += prompt + else: + break + except BadRequestError as e: + LOG.warning("BadRequestError: %s", e) + return { + "code_list": code_list, + "unrefined_code_list": unrefined_code_list, + "results_dict_list": results_dict_list, + "prompt_turn_list": prompt_turn_list, + "turn_idx": turn_idx, + "success": results_dict["passed_compile_judge"], + } + + async def process_single_datapoint(self, data_point, all_data): + result = await self._single_data_point_generate(data_point, all_data) + result_dict = {"generation": result} + return result_dict + + +GENERATION_TASK_CLASS = AutoformalizeTask + + +# Update the hydra main to use the class method +@hydra.main(version_base=None, config_name="base_generation_config") +def generate(cfg: AutoformalizeConfig): + cfg = AutoformalizeConfig(_init_nested=True, **cfg) + LOG.info("Config used: %s", cfg) + + task = AutoformalizeTask(cfg) + task.generate() + + +HELP_MESSAGE = get_help_message( + AutoformalizeConfig, + server_params=server_params(), + sandbox_params=sandbox_params(), +) + + +if __name__ == "__main__": + if "--help" in sys.argv or "-h" in sys.argv: + print(HELP_MESSAGE) + else: + setup_logging() + generate() diff --git a/nemo_skills/prompt/config/lean4/deepseek-R1-autoformalization.yaml b/nemo_skills/prompt/config/lean4/deepseek-R1-autoformalization.yaml new file mode 100644 index 0000000000..a7e696f1da --- /dev/null +++ b/nemo_skills/prompt/config/lean4/deepseek-R1-autoformalization.yaml @@ -0,0 +1,24 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Configuration for proving formal theorems in Lean 4. +# This file is tailored for tasks that involve constructing and verifying proofs +# of theorems within the Lean 4 formal system. + +user: |- + Please formalize the following natural language problem statement in Lean 4. If the problem statement is not a theorem, please rewrite the problem as a theorem and formalize it. You do not need to prove the theorem. Please use the following theorem name: {problem_name}. Please include appropriate headers and wrap the code in ```lean4 and ```. + + The natural language statement is: + + {problem} diff --git a/nemo_skills/prompt/config/lean4/deepseek-R1-backtranslation.yaml b/nemo_skills/prompt/config/lean4/deepseek-R1-backtranslation.yaml new file mode 100644 index 0000000000..a1a8ab1dfa --- /dev/null +++ b/nemo_skills/prompt/config/lean4/deepseek-R1-backtranslation.yaml @@ -0,0 +1,26 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Configuration for proving formal theorems in Lean 4. +# This file is tailored for tasks that involve constructing and verifying proofs +# of theorems within the Lean 4 formal system. + +user: |- + Please translate the following Lean 4 code into natural language. + + ```lean4 + {code} + ``` + + Please only output the natural language translation without additional comments or explanations! diff --git a/nemo_skills/prompt/config/lean4/deepseek-R1-judge-backtranslation.yaml b/nemo_skills/prompt/config/lean4/deepseek-R1-judge-backtranslation.yaml new file mode 100644 index 0000000000..1d11ad8d2b --- /dev/null +++ b/nemo_skills/prompt/config/lean4/deepseek-R1-judge-backtranslation.yaml @@ -0,0 +1,27 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Configuration for proving formal theorems in Lean 4. +# This file is tailored for tasks that involve constructing and verifying proofs +# of theorems within the Lean 4 formal system. + +user: |- + Help me determine if the following two math problems are essentially the same. + + Original problem: {problem} + Formalized problem: {backtranslation} + + Disregard the names and minor changes in word order that appear within. If the original problem is a question, the formalized problem is regarded as the same problem if it addresses or gives a possible answer to the question. + If the two problems are essentially the same, respond with "True" only without any explanation. + If the two problems are different, explain why they are different. diff --git a/nemo_skills/prompt/config/lean4/refinement_code_error.yaml b/nemo_skills/prompt/config/lean4/refinement_code_error.yaml new file mode 100644 index 0000000000..f7ede310f7 --- /dev/null +++ b/nemo_skills/prompt/config/lean4/refinement_code_error.yaml @@ -0,0 +1,26 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Configuration for proving formal theorems in Lean 4. +# This file is tailored for tasks that involve constructing and verifying proofs +# of theorems within the Lean 4 formal system. + +user: |- + It seems that the compilation failed. Please fix the error and try again. The error message is: + + ``` + {error_message} + ``` + + Again, please include appropriate headers and wrap the code in ```lean4 and ```. diff --git a/nemo_skills/prompt/config/lean4/refinement_consistent_error.yaml b/nemo_skills/prompt/config/lean4/refinement_consistent_error.yaml new file mode 100644 index 0000000000..8b0449fffe --- /dev/null +++ b/nemo_skills/prompt/config/lean4/refinement_consistent_error.yaml @@ -0,0 +1,26 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Configuration for proving formal theorems in Lean 4. +# This file is tailored for tasks that involve constructing and verifying proofs +# of theorems within the Lean 4 formal system. + +user: |- + The code passed the compilation but it looks like the lean4 code is different from the natural language problem. + + {reason} + + Please try again to formalize the problem and make sure the lean4 code is consistent with the natural language problem. + + Again, please include appropriate headers and wrap the code in ```lean4 and ```. diff --git a/nemo_skills/prompt/config/lean4/refinement_parsing_error.yaml b/nemo_skills/prompt/config/lean4/refinement_parsing_error.yaml new file mode 100644 index 0000000000..fda631a526 --- /dev/null +++ b/nemo_skills/prompt/config/lean4/refinement_parsing_error.yaml @@ -0,0 +1,20 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Configuration for proving formal theorems in Lean 4. +# This file is tailored for tasks that involve constructing and verifying proofs +# of theorems within the Lean 4 formal system. + +user: |- + Please include appropriate headers and wrap the code in ```lean4 and ```. From a02b373eb56ce9604547a38412a95e0854a1a047 Mon Sep 17 00:00:00 2001 From: Meline Mkrtchyan <72409758+melllinia@users.noreply.github.com> Date: Tue, 16 Dec 2025 02:46:42 +0400 Subject: [PATCH 49/88] HF ASR Leaderboard Evaluation (#1104) Signed-off-by: mmkrtchyan Co-authored-by: George <37293288+Jorjeous@users.noreply.github.com> Co-authored-by: George Armstrong Signed-off-by: Cheng-Ping Hsieh --- README.md | 2 +- docs/evaluation/index.md | 2 +- docs/evaluation/speech-audio.md | 197 +++++++++++------ docs/index.md | 2 +- .../dataset/asr-leaderboard/__init__.py | 21 ++ .../dataset/asr-leaderboard/prepare.py | 200 ++++++++++++++++++ nemo_skills/pipeline/prepare_data.py | 2 +- tests/gpu-tests/test_eval.py | 1 + 8 files changed, 361 insertions(+), 66 deletions(-) create mode 100644 nemo_skills/dataset/asr-leaderboard/__init__.py create mode 100644 nemo_skills/dataset/asr-leaderboard/prepare.py diff --git a/README.md b/README.md index e5231eb89b..0153961b34 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ Here are some of the features we support: - [**Long-context**](https://nvidia-nemo.github.io/Skills/evaluation/long-context): e.g. [ruler](https://nvidia-nemo.github.io/Skills/evaluation/long-context/#ruler), [mrcr](https://nvidia-nemo.github.io/Skills/evaluation/long-context/#mrcr), [aalcr](https://nvidia-nemo.github.io/Skills/evaluation/long-context/#aalcr) - [**Tool-calling**](https://nvidia-nemo.github.io/Skills/evaluation/tool-calling): e.g. [bfcl_v3](https://nvidia-nemo.github.io/Skills/evaluation/tool-calling/#bfcl_v3) - [**Multilingual**](https://nvidia-nemo.github.io/Skills/evaluation/multilingual): e.g. [mmlu-prox](https://nvidia-nemo.github.io/Skills/evaluation/multilingual/#mmlu-prox), [FLORES-200](https://nvidia-nemo.github.io/Skills/evaluation/multilingual/#FLORES-200), [wmt24pp](https://nvidia-nemo.github.io/Skills/evaluation/multilingual/#wmt24pp) - - [**Speech & Audio**](https://nvidia-nemo.github.io/Skills/evaluation/speech-audio): e.g. [mmau-pro](https://nvidia-nemo.github.io/Skills/evaluation/speech-audio/#mmau-pro) + - [**Speech & Audio**](https://nvidia-nemo.github.io/Skills/evaluation/speech-audio): e.g. [asr-leaderboard](https://nvidia-nemo.github.io/Skills/evaluation/speech-audio/#asr-leaderboard), [mmau-pro](https://nvidia-nemo.github.io/Skills/evaluation/speech-audio/#mmau-pro) - Easily parallelize each evaluation across many slurm jobs, self-host LLM judges, bring your own prompts or change benchmark configuration in any other way. - [Model training](https://nvidia-nemo.github.io/Skills/pipelines/training): Train models using [NeMo-RL](https://github.com/NVIDIA-NeMo/RL/) or [verl](https://github.com/volcengine/verl). diff --git a/docs/evaluation/index.md b/docs/evaluation/index.md index 4153b7aabf..b63943e869 100644 --- a/docs/evaluation/index.md +++ b/docs/evaluation/index.md @@ -10,7 +10,7 @@ We support many popular benchmarks and it's easy to add new in the future. The f - [**Long-context**](./long-context.md): e.g. [ruler](./long-context.md#ruler), [mrcr](./long-context.md#mrcr) - [**Tool-calling**](./tool-calling.md): e.g. [bfcl_v3](./tool-calling.md#bfcl_v3) - [**Multilingual**](./multilingual.md): e.g. [mmlu-prox](./multilingual.md#mmlu-prox), [flores-200](./multilingual.md#FLORES-200), [wmt24pp](./multilingual.md#wmt24pp) -- [**Speech & Audio**](./speech-audio.md): e.g. [mmau-pro](./speech-audio.md#mmau-pro) +- [**Speech & Audio**](./speech-audio.md): e.g. [asr-leaderboard](./speech-audio.md#asr-leaderboard), [mmau-pro](./speech-audio.md#mmau-pro) See [nemo_skills/dataset](https://github.com/NVIDIA-NeMo/Skills/blob/main/nemo_skills/dataset) where each folder is a benchmark we support. diff --git a/docs/evaluation/speech-audio.md b/docs/evaluation/speech-audio.md index 1d7d439928..9a5f7c5251 100644 --- a/docs/evaluation/speech-audio.md +++ b/docs/evaluation/speech-audio.md @@ -2,8 +2,22 @@ This section details how to evaluate speech and audio benchmarks, including understanding tasks that test models' ability to reason about audio content (speech, music, environmental sounds) and ASR tasks for transcription. +!!! note + Currently supports only Megatron server type (`--server_type=megatron`). + ## Supported benchmarks +### ASR Leaderboard + +ASR benchmark based on the [HuggingFace Open ASR Leaderboard](https://huggingface.co/spaces/hf-audio/open_asr_leaderboard). Evaluates transcription quality using Word Error Rate (WER). + +**Datasets:** `librispeech_clean`, `librispeech_other`, `voxpopuli`, `tedlium`, `gigaspeech`, `spgispeech`, `earnings22`, `ami` + +#### Dataset Location + +- Benchmark is defined in [`nemo_skills/dataset/asr-leaderboard/__init__.py`](https://github.com/NVIDIA-NeMo/Skills/blob/main/nemo_skills/dataset/asr-leaderboard/__init__.py) +- Original datasets are hosted on HuggingFace (downloaded automatically during preparation) + ### MMAU-Pro MMAU-Pro (Multimodal Audio Understanding - Pro) is a comprehensive benchmark for evaluating audio understanding capabilities across three different task categories: @@ -17,108 +31,101 @@ MMAU-Pro (Multimodal Audio Understanding - Pro) is a comprehensive benchmark for - Benchmark is defined in [`nemo_skills/dataset/mmau-pro/__init__.py`](https://github.com/NVIDIA-NeMo/Skills/blob/main/nemo_skills/dataset/mmau-pro/__init__.py) - Original benchmark source is hosted on [HuggingFace](https://huggingface.co/datasets/gamma-lab-umd/MMAU-Pro) -## Preparing MMAU-Pro Data +## Preparing Data -MMAU-Pro requires audio files for meaningful evaluation. **Audio files are downloaded by default** to ensure proper evaluation. +These benchmarks require audio files for meaningful evaluation. **Audio files are downloaded by default** to ensure proper evaluation. !!! warning "Running without audio files" - If you want to evaluation without audio files (not recommended) use + If you want to evaluate without audio files (not recommended) use `--no-audio` flag. In this case you can also set `--skip_data_dir_check` as data is very lightweight when audio files aren't being used. -### Data Preparation - -To prepare the dataset with audio files: +### ASR Leaderboard ```bash -export HF_TOKEN=your_huggingface_token -ns prepare_data mmau-pro --data_dir=/path/to/data --cluster= +ns prepare_data asr-leaderboard --data_dir=/path/to/data --cluster= ``` -**What happens:** - -- Requires authentication (HuggingFace token via `HF_TOKEN` environment variable) -- Downloads audio archive from HuggingFace and extracts -- Prepares the dataset files for evaluation +Prepare specific datasets only: -### Text-Only Mode (Not Recommended) +```bash +ns prepare_data asr-leaderboard --datasets librispeech_clean ami +``` -If you need to prepare without audio files: +### MMAU-Pro ```bash -ns prepare_data mmau-pro --no-audio +ns prepare_data mmau-pro --data_dir=/path/to/data --cluster= ``` -Note: The git repository check is automatically skipped with `--no-audio`. - ## Running Evaluation -!!! note - Currently supports only Megatron server type (`--server_type=megatron`). - -### Evaluation Example +### ASR Leaderboard ```python -import os from nemo_skills.pipeline.cli import wrap_arguments, eval -os.environ["NVIDIA_API_KEY"] = "your_nvidia_api_key" # For LLM judge - eval( - ctx=wrap_arguments("++prompt_suffix='/no_think'"), + ctx=wrap_arguments(""), cluster="oci_iad", - output_dir="/workspace/mmau-pro-eval", - benchmarks="mmau-pro", + output_dir="/workspace/asr-leaderboard-eval", + benchmarks="asr-leaderboard", server_type="megatron", server_gpus=1, model="/workspace/checkpoint", server_entrypoint="/workspace/megatron-lm/server.py", server_container="/path/to/container.sqsh", data_dir="/dataset", - installation_command="pip install sacrebleu", + installation_command="pip install sacrebleu jiwer openai-whisper" server_args="--inference-max-requests 1 --model-config /workspace/checkpoint/config.yaml", ) ``` -??? note "Alternative: Command-line usage" +Evaluate a specific dataset: - If you prefer using the command-line interface, you can run: +```python +eval(benchmarks="asr-leaderboard", split="librispeech_clean", ...) +``` - ```bash - export HF_TOKEN=your_huggingface_token - export NVIDIA_API_KEY=your_nvidia_api_key - export MEGATRON_PATH=/workspace/path/to/megatron-lm +??? note "Alternative: Command-line usage" + ```bash ns eval \ --cluster=oci_iad \ - --output_dir=/workspace/path/to/mmau-pro-eval \ - --benchmarks=mmau-pro \ + --output_dir=/workspace/path/to/asr-leaderboard-eval \ + --benchmarks=asr-leaderboard \ --server_type=megatron \ --server_gpus=1 \ - --model=/workspace/path/to/checkpoint-tp1 \ - --server_entrypoint=$MEGATRON_PATH/path/to/server.py \ - --server_container=/path/to/server_container.sqsh \ - --data_dir=/dataset \ - --installation_command="pip install sacrebleu" \ - ++prompt_suffix='/no_think' \ - --server_args="--inference-max-requests 1 \ - --model-config /workspace/path/to/checkpoint-tp1/config.yaml \ - --num-tokens-to-generate 256 \ - --temperature 1.0 \ - --top_p 1.0" + --model=/workspace/path/to/checkpoint \ + --server_entrypoint=/workspace/megatron-lm/server.py \ + --server_container=/path/to/container.sqsh \ + --data_dir=/dataset + --installation_command="pip install sacrebleu jiwer openai-whisper" ``` -## How Evaluation Works +### MMAU-Pro -Each category uses a different evaluation strategy: +```python +import os +from nemo_skills.pipeline.cli import wrap_arguments, eval -| Category | Evaluation Method | How It Works | -|----------|-------------------|--------------| -| **Closed-Form** | NVEmbed similarity matching | Model generates short answer; compared to expected answer using embeddings | -| **Open-Ended** | LLM-as-a-judge (Qwen 2.5 7B) | Model generates detailed response; Qwen 2.5 judges quality and correctness | -| **Instruction Following** | Custom evaluation logic | Model follows instructions; evaluator checks adherence | +os.environ["NVIDIA_API_KEY"] = "your_nvidia_api_key" # For LLM judge -### Sub-benchmarks +eval( + ctx=wrap_arguments(""), + cluster="oci_iad", + output_dir="/workspace/mmau-pro-eval", + benchmarks="mmau-pro", + server_type="megatron", + server_gpus=1, + model="/workspace/checkpoint", + server_entrypoint="/workspace/megatron-lm/server.py", + server_container="/path/to/container.sqsh", + data_dir="/dataset", + installation_command="pip install sacrebleu", + server_args="--inference-max-requests 1 --model-config /workspace/checkpoint/config.yaml", +) +``` Evaluate individual categories: @@ -130,6 +137,24 @@ Evaluate individual categories: eval(benchmarks="mmau-pro.closed_form", ...) ``` +??? note "Alternative: Command-line usage" + + ```bash + export NVIDIA_API_KEY=your_nvidia_api_key + + ns eval \ + --cluster=oci_iad \ + --output_dir=/workspace/path/to/mmau-pro-eval \ + --benchmarks=mmau-pro \ + --server_type=megatron \ + --server_gpus=1 \ + --model=/workspace/path/to/checkpoint \ + --server_entrypoint=/workspace/megatron-lm/server.py \ + --server_container=/path/to/container.sqsh \ + --data_dir=/dataset \ + --installation_command="pip install sacrebleu" + ``` + ### Using Custom Judge Models The open-ended questions subset uses an LLM-as-a-judge (by default, Qwen 2.5 7B via NVIDIA API) to evaluate responses. You can customize the judge model for this subset: @@ -143,7 +168,7 @@ The open-ended questions subset uses an LLM-as-a-judge (by default, Qwen 2.5 7B os.environ["NVIDIA_API_KEY"] = "your_nvidia_api_key" eval( - ctx=wrap_arguments("++prompt_suffix='/no_think'"), + ctx=wrap_arguments(""), cluster="oci_iad", output_dir="/workspace/path/to/mmau-pro-eval", benchmarks="mmau-pro.open_ended", # Only open-ended uses LLM judge @@ -180,7 +205,58 @@ The open-ended questions subset uses an LLM-as-a-judge (by default, Qwen 2.5 7B ## Understanding Results -After evaluation completes, results are saved in your output directory under `eval-results/`: +After evaluation completes, results are saved in your output directory under `eval-results/`. + +### ASR Leaderboard Results + +``` +/ +└── eval-results/ + └── asr-leaderboard/ + └──metrics.json +``` + +Example output: + +``` +------------------------------------- asr-leaderboard -------------------------------------- +evaluation_mode | avg_tokens | gen_seconds | success_rate | no_answer | wer | num_entries +pass@1 | 736 | 233522 | 86.70% | 0.00% | 7.82% | 143597 + +----------------------------------- asr-leaderboard-ami ------------------------------------ +evaluation_mode | avg_tokens | gen_seconds | success_rate | no_answer | wer | num_entries +pass@1 | 732 | 3680 | 81.27% | 0.00% | 18.45% | 12620 + +-------------------------------- asr-leaderboard-earnings22 -------------------------------- +evaluation_mode | avg_tokens | gen_seconds | success_rate | no_answer | wer | num_entries +pass@1 | 736 | 3522 | 83.97% | 0.00% | 14.72% | 57390 + +-------------------------------- asr-leaderboard-gigaspeech -------------------------------- +evaluation_mode | avg_tokens | gen_seconds | success_rate | no_answer | wer | num_entries +pass@1 | 736 | 233469 | 71.86% | 0.00% | 12.34% | 25376 + +---------------------------- asr-leaderboard-librispeech_clean ---------------------------- +evaluation_mode | avg_tokens | gen_seconds | success_rate | no_answer | wer | num_entries +pass@1 | 735 | 3607 | 99.62% | 0.00% | 2.06% | 2620 + +---------------------------- asr-leaderboard-librispeech_other ---------------------------- +evaluation_mode | avg_tokens | gen_seconds | success_rate | no_answer | wer | num_entries +pass@1 | 733 | 3927 | 98.67% | 0.00% | 4.34% | 2939 + +-------------------------------- asr-leaderboard-spgispeech ------------------------------- +evaluation_mode | avg_tokens | gen_seconds | success_rate | no_answer | wer | num_entries +pass@1 | 740 | 4510 | 99.99% | 0.00% | 3.81% | 39341 + +--------------------------------- asr-leaderboard-tedlium ---------------------------------- +evaluation_mode | avg_tokens | gen_seconds | success_rate | no_answer | wer | num_entries +pass@1 | 732 | 3878 | 77.74% | 0.00% | 7.89% | 1469 + +-------------------------------- asr-leaderboard-voxpopuli -------------------------------- +evaluation_mode | avg_tokens | gen_seconds | success_rate | no_answer | wer | num_entries +pass@1 | 741 | 4007 | 99.51% | 0.00% | 6.47% | 1842 +``` + +### MMAU-Pro Results ``` / @@ -195,9 +271,7 @@ After evaluation completes, results are saved in your output directory under `ev │ └── metrics.json ``` -### Evaluation Output Format - -When evaluation completes, results are displayed in formatted tables in the logs: +Example output: **Open-Ended Questions:** @@ -213,7 +287,6 @@ pass@1 | 82 | 196 | 14.88% | 0.00% | 625 -------------------------- mmau-pro.instruction_following ------------------------- evaluation_mode | avg_tokens | gen_seconds | success_rate | no_answer | num_entries pass@1 | 0 | 102 | 21.84% | 0.00% | 87 - ``` **Closed-Form Questions (Main Category + Sub-categories):** diff --git a/docs/index.md b/docs/index.md index 01d63c1959..ce53e256c3 100644 --- a/docs/index.md +++ b/docs/index.md @@ -22,7 +22,7 @@ Here are some of the features we support: - [**Long-context**](./evaluation/long-context.md): e.g. [ruler](./evaluation/long-context.md#ruler), [mrcr](./evaluation/long-context.md#mrcr) - [**Tool-calling**](./evaluation/tool-calling.md): e.g. [bfcl_v3](./evaluation/tool-calling.md#bfcl_v3) - [**Multilingual capabilities**](./evaluation/multilingual.md): e.g. [mmlu-prox](./evaluation/multilingual.md#mmlu-prox), [flores-200](./evaluation/multilingual.md#FLORES-200), [wmt24pp](./evaluation/multilingual.md#wmt24pp) - - [**Speech & Audio**](./evaluation/speech-audio.md): e.g. [mmau-pro](./evaluation/speech-audio.md#mmau-pro) + - [**Speech & Audio**](./evaluation/speech-audio.md): e.g. [asr-leaderboard](./evaluation/speech-audio.md#asr-leaderboard), [mmau-pro](./evaluation/speech-audio.md#mmau-pro) - [**Robustness evaluation**](./evaluation/robustness.md): Evaluate model sensitvity against changes in prompt. - Easily parallelize each evaluation across many Slurm jobs, self-host LLM judges, bring your own prompts or change benchmark configuration in any other way. - [Model training](pipelines/training.md): Train models using [NeMo-RL](https://github.com/NVIDIA-NeMo/RL/) or [verl](https://github.com/volcengine/verl). diff --git a/nemo_skills/dataset/asr-leaderboard/__init__.py b/nemo_skills/dataset/asr-leaderboard/__init__.py new file mode 100644 index 0000000000..b81cace3bc --- /dev/null +++ b/nemo_skills/dataset/asr-leaderboard/__init__.py @@ -0,0 +1,21 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Settings that define how evaluation should be done by default (all can be changed from cmdline) +# Uses the audio evaluator which computes WER with HuggingFace leaderboard preprocessing +# Data samples should have task_type="ASR_LEADERBOARD" for proper WER calculation + +DATASET_GROUP = "speechlm" +METRICS_TYPE = "audio" +GENERATION_ARGS = "++prompt_format=openai ++eval_type=audio" diff --git a/nemo_skills/dataset/asr-leaderboard/prepare.py b/nemo_skills/dataset/asr-leaderboard/prepare.py new file mode 100644 index 0000000000..25bbafd986 --- /dev/null +++ b/nemo_skills/dataset/asr-leaderboard/prepare.py @@ -0,0 +1,200 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Prepare ASR Leaderboard datasets for evaluation. + +Downloads and formats datasets from the HuggingFace Open ASR Leaderboard. +Audio paths in JSONL: /dataset/asr-leaderboard/data/{dataset}/{sample_id}.flac + +Usage: + ns prepare_data asr-leaderboard + ns prepare_data asr-leaderboard --datasets librispeech_clean ami + ns prepare_data asr-leaderboard --no-audio # skip saving audio files +""" + +import argparse +import json +from pathlib import Path + +import soundfile as sf +from datasets import load_dataset +from tqdm import tqdm + +SYSTEM_MESSAGE = "You are a helpful assistant. /no_think" +MIN_AUDIO_DURATION = 0.1 # Skip audio shorter than this (causes mel spectrogram errors) + +# (hf_dataset, hf_config, hf_split, streaming) +DATASET_CONFIGS = { + "librispeech_clean": ("librispeech_asr", "clean", "test", False), + "librispeech_other": ("librispeech_asr", "other", "test", False), + "voxpopuli": ("facebook/voxpopuli", "en", "test", False), + "tedlium": ("LIUM/tedlium", "release3", "test", False), + "gigaspeech": ("speechcolab/gigaspeech", "xs", "test", False), + "spgispeech": ("kensho/spgispeech", "test", "test", True), # streaming to avoid timeout due to large metadata + "earnings22": ("distil-whisper/earnings22", "chunked", "test", False), + "ami": ("edinburghcstr/ami", "ihm", "test", False), +} + + +def save_audio_and_format_entry(entry, dataset_name, audio_dir, sample_idx, with_audio=True): + """Format a dataset entry and optionally save audio file.""" + # Different datasets use different field names for transcription + text = ( + entry.get("text", "") # ami, LS, gigaspeech, tedlium + or entry.get("normalized_text", "") # voxpopuli + or entry.get("transcript", "") # spgispeech + or entry.get("transcription", "") # earnings22 + ) + text = text.strip() if text else "" + + system_message = {"role": "system", "content": SYSTEM_MESSAGE} + user_message = {"role": "user", "content": "Transcribe the following audio."} + + audio_info = entry.get("audio", {}) + if isinstance(audio_info, dict) and "array" in audio_info and "sampling_rate" in audio_info: + audio_array = audio_info["array"] + sampling_rate = audio_info["sampling_rate"] + duration = len(audio_array) / sampling_rate + + if duration < MIN_AUDIO_DURATION: + return None + + sample_id = entry.get("id", str(sample_idx)) + audio_filename = f"{sample_id}.flac" + + if with_audio: + sf.write(str(audio_dir / audio_filename), audio_array, sampling_rate) + + user_message["audio"] = { + "path": f"/dataset/asr-leaderboard/data/{dataset_name}/{audio_filename}", + "duration": float(duration), + } + + formatted_entry = { + "task_type": "ASR_LEADERBOARD", + "expected_answer": text, + "messages": [system_message, user_message], + "subset_for_metrics": dataset_name, + } + + if "id" in entry: + formatted_entry["id"] = entry["id"] + if "speaker_id" in entry: + formatted_entry["speaker_id"] = entry["speaker_id"] + + return formatted_entry + + +def prepare_dataset(dataset_name, output_dir, with_audio=True): + """Prepare a single ASR dataset.""" + if dataset_name not in DATASET_CONFIGS: + raise ValueError(f"Unknown dataset: {dataset_name}. Available: {list(DATASET_CONFIGS.keys())}") + + hf_dataset, hf_config, hf_split, streaming = DATASET_CONFIGS[dataset_name] + + print(f"Loading {dataset_name} from {hf_dataset} (streaming={streaming})...") + try: + if hf_config: + dataset = load_dataset(hf_dataset, hf_config, split=hf_split, trust_remote_code=True, streaming=streaming) + else: + dataset = load_dataset(hf_dataset, split=hf_split, trust_remote_code=True, streaming=streaming) + except Exception as e: + print(f"Warning: Failed to load {dataset_name}: {e}") + return 0 + + output_file = output_dir / f"{dataset_name}.jsonl" + audio_dir = output_dir / "data" / dataset_name + + if with_audio: + audio_dir.mkdir(parents=True, exist_ok=True) + print(f"Saving audio files to {audio_dir}") + + if streaming: + print(f"Processing {dataset_name} (streaming)...") + else: + print(f"Processing {len(dataset)} samples from {dataset_name}...") + + count = 0 + skipped = 0 + with open(output_file, "w", encoding="utf-8") as fout: + for idx, entry in enumerate(tqdm(dataset, desc=dataset_name)): + formatted = save_audio_and_format_entry(entry, dataset_name, audio_dir, idx, with_audio=with_audio) + if formatted is None: + skipped += 1 + continue + if formatted["expected_answer"]: + fout.write(json.dumps(formatted) + "\n") + count += 1 + + if skipped > 0: + print(f"Skipped {skipped} samples with audio < {MIN_AUDIO_DURATION}s") + + print(f"Saved {count} samples to {output_file}") + return count + + +def main(): + parser = argparse.ArgumentParser(description="Prepare ASR Leaderboard datasets for evaluation") + parser.add_argument( + "--datasets", + nargs="+", + default=["all"], + choices=list(DATASET_CONFIGS.keys()) + ["all"], + help="Datasets to prepare (default: all)", + ) + parser.add_argument( + "--no-audio", + action="store_true", + help="Skip saving audio files (JSONL still includes audio paths)", + ) + args = parser.parse_args() + + output_dir = Path(__file__).parent + output_dir.mkdir(parents=True, exist_ok=True) + + with_audio = not args.no_audio + + if args.no_audio: + print("Running without saving audio files.") + else: + print("Running with audio. Saving to data/{dataset}/") + + datasets_to_prepare = list(DATASET_CONFIGS.keys()) if "all" in args.datasets else args.datasets + + total_samples = 0 + for dataset_name in datasets_to_prepare: + total_samples += prepare_dataset(dataset_name, output_dir, with_audio=with_audio) + + # Combine all dataset JSONLs into test.jsonl + combined_file = output_dir / "test.jsonl" + print(f"\nCreating combined file: {combined_file}") + + all_jsonl_files = sorted(output_dir.glob("*.jsonl")) + dataset_files = [f for f in all_jsonl_files if f.name != "test.jsonl"] + + combined_count = 0 + with open(combined_file, "w", encoding="utf-8") as fout: + for dataset_file in dataset_files: + with open(dataset_file, encoding="utf-8") as fin: + for line in fin: + fout.write(line) + combined_count += 1 + print(f" Added {dataset_file.name}") + + print(f"Combined {combined_count} samples from {len(dataset_files)} datasets into {combined_file}") + print(f"\nTotal: {total_samples} samples prepared") + + +if __name__ == "__main__": + main() diff --git a/nemo_skills/pipeline/prepare_data.py b/nemo_skills/pipeline/prepare_data.py index 49a401978d..36820c7337 100644 --- a/nemo_skills/pipeline/prepare_data.py +++ b/nemo_skills/pipeline/prepare_data.py @@ -31,7 +31,7 @@ # TODO: read this from init.py -DATASETS_REQUIRE_DATA_DIR = ["ruler", "ioi24", "mmau-pro"] +DATASETS_REQUIRE_DATA_DIR = ["ruler", "ioi24", "mmau-pro", "asr-leaderboard"] @app.command(context_settings={"allow_extra_args": True, "ignore_unknown_options": True}) diff --git a/tests/gpu-tests/test_eval.py b/tests/gpu-tests/test_eval.py index c69c75be71..aa5df51035 100644 --- a/tests/gpu-tests/test_eval.py +++ b/tests/gpu-tests/test_eval.py @@ -43,6 +43,7 @@ "human-eval-infilling", "mbpp", "mmau-pro", + "asr-leaderboard", "aalcr", # Has tokenization mismatch issues } From 218a0acd4df710bc3c70b0f717341a96c24bee66 Mon Sep 17 00:00:00 2001 From: Stephen Ge Date: Mon, 15 Dec 2025 21:10:18 -0500 Subject: [PATCH 50/88] Stepheng/nemotron math proofs docs (#1111) Signed-off-by: Cheng-Ping Hsieh --- docs/releases/nemotronmathproofs/index.md | 373 ++++++++++++++++++ mkdocs.yml | 1 + ...malization.yaml => autoformalization.yaml} | 0 ...ktranslation.yaml => backtranslation.yaml} | 0 ...lation.yaml => judge-backtranslation.yaml} | 0 5 files changed, 374 insertions(+) create mode 100644 docs/releases/nemotronmathproofs/index.md rename nemo_skills/prompt/config/lean4/{deepseek-R1-autoformalization.yaml => autoformalization.yaml} (100%) rename nemo_skills/prompt/config/lean4/{deepseek-R1-backtranslation.yaml => backtranslation.yaml} (100%) rename nemo_skills/prompt/config/lean4/{deepseek-R1-judge-backtranslation.yaml => judge-backtranslation.yaml} (100%) diff --git a/docs/releases/nemotronmathproofs/index.md b/docs/releases/nemotronmathproofs/index.md new file mode 100644 index 0000000000..e9de7e26aa --- /dev/null +++ b/docs/releases/nemotronmathproofs/index.md @@ -0,0 +1,373 @@ +--- +date: 2025-12-15 +--- + +# Nemotron-Math-Proofs + +## Dataset Overview + +Using our pipelines we created [Nemotron-Math-Proofs-v1](https://huggingface.co/datasets/nvidia/Nemotron-Math-Proofs-v1), +a large-scale mathematical reasoning dataset containing approximately 580k natural language proof problems, +550k formalizations into theorem statements in Lean 4, and 900k model-generated reasoning trajectories +culminating in Lean 4 proofs. + +The dataset integrates human-authored problems with systematically generated formalizations and solution traces: + +* **Natural Language Problems**: ~580k proof problems sourced from AoPS forums, Math StackExchange, and MathOverflow, + semantically deduplicated and decontaminated against popular benchmarks +* **Formal Statements**: ~550k Lean 4 theorem statements generated via autoformalization +* **Proof Trajectories**: ~900k verified reasoning traces and proofs + +Each natural language problem is formalized by [gpt-oss-120b](https://huggingface.co/openai/gpt-oss-120b) into +Lean 4 theorem statements. Reasoning traces and proofs are generated by +[Goedel-Prover-V2-32B](https://huggingface.co/Goedel-LM/Goedel-Prover-V2-32B) and verified by the Lean 4 compiler. + +This dataset was used as part of the SFT data for [Nemotron-Nano-v3](https://huggingface.co/collections/nvidia/nvidia-nemotron-v3). + +## Training Results + +We compare a Qwen3-8B model fine-tuned on this dataset against Goedel-Prover-V2-8B on miniF2F: + +| Model | pass@32 (no self-correction) | pass@32 (with self-correction) | +|-------|------------------------------|-------------------------------| +| Goedel-Prover-V2-8B | 84.6% | 86.7% | +| Qwen3-8B SFT on Nemotron-Math-Proofs-v1 | 85.3% | 90.2% | + +Nemotron-Nano-v3 (which includes this dataset in its training) achieves the following on miniF2F: + +| Model | pass@32 (no self-correction) | pass@32 (with self-correction) | +|-------|------------------------------|-------------------------------| +| Nemotron-Nano-v3 | 79.92% | 86.89% | +| gpt-oss-20b | 43.03% | 59.42% | +| Qwen3-30B-A3B-Thinking | 16.80% | 32.37% | + +## How to Reproduce + +Browse the sections below to see commands for autoformalization, theorem proving, training, and evaluation. + +!!! note + + These commands assume you have `/workspace` defined in your [cluster config](../../basics/cluster-configs.md). + Adjust paths and cluster settings according to your environment. + +### Autoformalization + +The autoformalization pipeline translates natural language theorems into Lean 4 formal statements using an iterative +refinement process with backtranslation verification. The input is natural language math problems—see +[OpenMathReasoning dataset construction](../openmathreasoning/dataset.md) for how to prepare these. + +=== "CLI" + + ```bash + ns generate \ + --cluster=slurm \ + --generation_module=nemo_skills.inference.autoformalize \ + --model=openai/gpt-oss-120b \ + --server_type=vllm \ + --server_gpus= \ + --input_file=/workspace/data/problems.jsonl \ + --output_dir=/workspace/data/autoformalize_output \ + --with_sandbox \ + --num_random_seeds=1 \ + ++prompt_config=lean4/autoformalization \ + ++inference.tokens_to_generate=120000 \ + ++inference.temperature=1.0 \ + ++inference.top_p=1.0 \ + ++adaptive_reasoning=True \ + ++refinement=True \ + ++judge_enabled=True \ + ++refinement_max_turns=8 \ + ++backtranslation_prompt_config=lean4/backtranslation \ + ++judge_prompt_config=lean4/judge-backtranslation \ + ++refine_consistent_error_prompt_config=lean4/refinement_consistent_error \ + ++refine_parsing_error_prompt_config=lean4/refinement_parsing_error \ + ++refine_code_error_prompt_config=lean4/refinement_code_error + ``` + +=== "Python" + + ```python + from nemo_skills.pipeline.cli import generate, wrap_arguments + + generate( + ctx=wrap_arguments( + "++inference.tokens_to_generate=120000 " + "++prompt_config=lean4/autoformalization " + "++inference.temperature=1.0 " + "++inference.top_p=1.0 " + "++adaptive_reasoning=True " + "++refinement=True " + "++judge_enabled=True " + "++refinement_max_turns=8 " + "++backtranslation_prompt_config=lean4/backtranslation " + "++judge_prompt_config=lean4/judge-backtranslation " + "++refine_consistent_error_prompt_config=lean4/refinement_consistent_error " + "++refine_parsing_error_prompt_config=lean4/refinement_parsing_error " + "++refine_code_error_prompt_config=lean4/refinement_code_error " + ), + generation_module="nemo_skills.inference.autoformalize", + cluster="slurm", + server_gpus="", + input_file="/workspace/data/problems.jsonl", + output_dir="/workspace/data/autoformalize_output", + server_type="vllm", + model="openai/gpt-oss-120b", + with_sandbox=True, + num_random_seeds=1, + ) + ``` + +The pipeline includes: + +* **Initial Formalization**: LLM translates natural language to Lean 4 code +* **Compilation Check**: Lean 4 sandbox verifies syntactic correctness +* **Backtranslation & Verification**: Formal code is backtranslated and compared to the original +* **Iterative Refinement**: Up to 8 iterations to fix parsing, compilation, or semantic errors +* **Adaptive Reasoning**: Automatically reduces reasoning effort when hitting context limits + +### Theorem Proving + +The prover pipeline generates proofs for formalized statements with iterative error correction. +Input: formal statements from the autoformalization step. + +=== "CLI" + + ```bash + ns generate \ + --cluster=slurm \ + --generation_module=nemo_skills.inference.prover \ + --model=Goedel-LM/Goedel-Prover-V2-32B \ + --server_type=vllm \ + --server_gpus= \ + --server_args="--max-model-len 40960" \ + --input_file=/workspace/data/formal_statements.jsonl \ + --output_dir=/workspace/data/proofs_output \ + --with_sandbox \ + --num_random_seeds=1 \ + ++prompt_config=lean4/goedel-prover-v2 \ + ++inference.tokens_to_generate=38912 \ + ++inference.temperature=1.0 \ + ++inference.top_p=0.95 \ + ++refinement=True \ + ++refinement_max_turns=8 \ + ++remove_cot=True \ + ++n_pass=4 \ + ++refinement_prompt_config=lean4/goedel-prover-v2-refinement \ + ++delete_wrong_turns=True \ + ++max_concurrent_requests=512 + ``` + +=== "Python" + + ```python + from nemo_skills.pipeline.cli import generate, wrap_arguments + + generate( + ctx=wrap_arguments( + "++inference.tokens_to_generate=38912 " + "++inference.temperature=1.0 " + "++inference.top_p=0.95 " + "++prompt_config=lean4/goedel-prover-v2 " + "++refinement=True " + "++refinement_max_turns=8 " + "++remove_cot=True " + "++n_pass=4 " + "++refinement_prompt_config=lean4/goedel-prover-v2-refinement " + "++delete_wrong_turns=True " + "++max_concurrent_requests=512 " + ), + generation_module="nemo_skills.inference.prover", + cluster="slurm", + input_file="/workspace/data/formal_statements.jsonl", + output_dir="/workspace/data/proofs_output", + model="Goedel-LM/Goedel-Prover-V2-32B", + server_type="vllm", + server_gpus="", + server_args="--max-model-len 40960", + num_random_seeds=1, + with_sandbox=True, + ) + ``` + +The proving strategy includes: + +* **Chain-of-thought removal**: Strips reasoning, keeping only formal proof code +* **Wrong turn deletion**: Discards failed attempts to prevent context pollution +* **Structured error feedback**: Compiler errors annotated with `` tags +* **Pass@N with refinement**: Multiple independent attempts, each with iterative refinement + +### Model Training + +To fine-tune a model on the Nemotron-Math-Proofs dataset. +Input: processed SFT data from the theorem proving step. + +=== "CLI" + + ```bash + ns sft_nemo_rl \ + --cluster=slurm \ + --expname=qwen3-8b-lean-sft \ + --output_dir=/workspace/training/qwen3-8b-lean-sft \ + --hf_model=Qwen/Qwen3-8B \ + --training_data=/workspace/data/sft_data.jsonl \ + --num_nodes= \ + --num_gpus= \ + --backend=megatron \ + ++checkpointing.save_period=250 \ + ++sft.max_num_epochs=2000 \ + ++sft.max_num_steps=1000 \ + ++policy.megatron_cfg.tensor_model_parallel_size=4 \ + ++policy.megatron_cfg.pipeline_model_parallel_size=1 \ + ++policy.megatron_cfg.context_parallel_size=4 \ + ++policy.train_global_batch_size=2048 \ + ++policy.max_total_sequence_length=49152 \ + ++policy.megatron_cfg.optimizer.lr=1e-4 \ + ++policy.megatron_cfg.optimizer.min_lr=1e-4 \ + ++policy.megatron_cfg.scheduler.lr_warmup_iters=0 + ``` + +=== "Python" + + ```python + from nemo_skills.pipeline.cli import sft_nemo_rl, wrap_arguments + + sft_nemo_rl( + ctx=wrap_arguments( + "++checkpointing.save_period=250 " + "++sft.max_num_epochs=2000 " + "++sft.max_num_steps=1000 " + "++policy.megatron_cfg.tensor_model_parallel_size=4 " + "++policy.megatron_cfg.pipeline_model_parallel_size=1 " + "++policy.megatron_cfg.context_parallel_size=4 " + "++policy.train_global_batch_size=2048 " + "++policy.max_total_sequence_length=49152 " + "++policy.megatron_cfg.optimizer.lr=1e-4 " + "++policy.megatron_cfg.optimizer.min_lr=1e-4 " + "++policy.megatron_cfg.scheduler.lr_warmup_iters=0 " + ), + cluster="slurm", + expname="qwen3-8b-lean-sft", + backend="megatron", + output_dir="/workspace/training/qwen3-8b-lean-sft", + hf_model="Qwen/Qwen3-8B", + training_data="/workspace/data/sft_data.jsonl", + num_gpus="", + num_nodes="", + ) + ``` + +### Model Evaluation on miniF2F + +To evaluate a model on the miniF2F benchmark with 32 samples per problem (without self-correction): + +=== "CLI" + + ```bash + ns eval \ + --cluster=slurm \ + --model=nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16 \ + --server_type=vllm \ + --server_gpus= \ + --output_dir=/workspace/evals/nemotron-nano-3-minif2f \ + --benchmarks=minif2f:32 \ + --with_sandbox \ + ++prompt_config=lean4/goedel-prover-v2-nemotron \ + ++inference.tokens_to_generate=120000 \ + ++inference.temperature=1.0 \ + ++inference.top_p=1.0 \ + --extra_eval_args="++eval_config.timeout=600" + ``` + +=== "Python" + + ```python + from nemo_skills.pipeline.cli import eval, wrap_arguments + + eval( + ctx=wrap_arguments( + "++prompt_config=lean4/goedel-prover-v2-nemotron " + "++inference.tokens_to_generate=120000 " + "++inference.temperature=1.0 " + "++inference.top_p=1.0 " + ), + cluster="slurm", + model="nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16", + server_type="vllm", + server_gpus="", + benchmarks="minif2f:32", + output_dir="/workspace/evals/nemotron-nano-3-minif2f", + extra_eval_args="++eval_config.timeout=600", + with_sandbox=True, + ) + ``` + +To evaluate with self-correction (iterative refinement based on compiler feedback): + +=== "CLI" + + ```bash + ns generate \ + --cluster=slurm \ + --generation_module=nemo_skills.inference.prover \ + --model=nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16 \ + --server_type=vllm \ + --server_gpus= \ + --input_file=/nemo_run/code/nemo_skills/dataset/minif2f/test.jsonl \ + --output_dir=/workspace/evals/nemotron-nano-3-minif2f-self-correction \ + --with_sandbox \ + --num_random_seeds=32 \ + ++prompt_config=lean4/goedel-prover-v2-nemotron \ + ++inference.tokens_to_generate=120000 \ + ++inference.temperature=1.0 \ + ++inference.top_p=1.0 \ + ++refinement=True \ + ++refinement_max_turns=8 \ + ++remove_cot=True \ + ++n_pass=1 \ + ++refinement_prompt_config=lean4/goedel-prover-v2-refinement \ + ++delete_wrong_turns=True \ + ++max_concurrent_requests=512 + ``` + +=== "Python" + + ```python + from nemo_skills.pipeline.cli import generate, wrap_arguments + + generate( + ctx=wrap_arguments( + "++inference.tokens_to_generate=120000 " + "++inference.temperature=1.0 " + "++inference.top_p=1.0 " + "++prompt_config=lean4/goedel-prover-v2-nemotron " + "++refinement=True " + "++refinement_max_turns=8 " + "++remove_cot=True " + "++n_pass=1 " + "++refinement_prompt_config=lean4/goedel-prover-v2-refinement " + "++delete_wrong_turns=True " + "++max_concurrent_requests=512 " + ), + generation_module="nemo_skills.inference.prover", + cluster="slurm", + input_file="/nemo_run/code/nemo_skills/dataset/minif2f/test.jsonl", + output_dir="/workspace/evals/nemotron-nano-3-minif2f-self-correction", + model="nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16", + server_type="vllm", + server_gpus="", + num_random_seeds=32, + with_sandbox=True, + ) + ``` + +To summarize evaluation results: + +```bash +ns summarize_results /workspace/evals/nemotron-nano-3-minif2f/eval-results/minif2f --cluster slurm +``` + +## Known Limitations + +* **Difficulty balance**: No explicit normalization of problem difficulty; implicit selection biases from pipeline stages +* **Token/length normalization**: Some solutions contain warnings (e.g., unused hypotheses) since verification only checks for compilation errors +* **Placeholder definitions**: Some formalizations use trivial placeholder definitions instead of mathlib (e.g., `def MyContinuous... Prop := True`) diff --git a/mkdocs.yml b/mkdocs.yml index 285329cdf8..899e8f3826 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -91,6 +91,7 @@ nav: - tutorials/index.md - Papers & Releases: - releases/index.md + - Nemotron-Math-Proofs: releases/nemotronmathproofs/index.md - OpenReasoning: - releases/openreasoning/index.md - Model Evaluation: releases/openreasoning/evaluation.md diff --git a/nemo_skills/prompt/config/lean4/deepseek-R1-autoformalization.yaml b/nemo_skills/prompt/config/lean4/autoformalization.yaml similarity index 100% rename from nemo_skills/prompt/config/lean4/deepseek-R1-autoformalization.yaml rename to nemo_skills/prompt/config/lean4/autoformalization.yaml diff --git a/nemo_skills/prompt/config/lean4/deepseek-R1-backtranslation.yaml b/nemo_skills/prompt/config/lean4/backtranslation.yaml similarity index 100% rename from nemo_skills/prompt/config/lean4/deepseek-R1-backtranslation.yaml rename to nemo_skills/prompt/config/lean4/backtranslation.yaml diff --git a/nemo_skills/prompt/config/lean4/deepseek-R1-judge-backtranslation.yaml b/nemo_skills/prompt/config/lean4/judge-backtranslation.yaml similarity index 100% rename from nemo_skills/prompt/config/lean4/deepseek-R1-judge-backtranslation.yaml rename to nemo_skills/prompt/config/lean4/judge-backtranslation.yaml From 30457b894e5e0cc04806fc5ff733def586dd4521 Mon Sep 17 00:00:00 2001 From: Stephen Ge Date: Mon, 15 Dec 2025 21:32:05 -0500 Subject: [PATCH 51/88] Stepheng/prover gpt oss fix (#1114) Signed-off-by: Cheng-Ping Hsieh --- nemo_skills/inference/prover.py | 80 +++++++++++++++++++++++++++------ 1 file changed, 66 insertions(+), 14 deletions(-) diff --git a/nemo_skills/inference/prover.py b/nemo_skills/inference/prover.py index 848c7c70dd..711b5fe9aa 100644 --- a/nemo_skills/inference/prover.py +++ b/nemo_skills/inference/prover.py @@ -195,6 +195,62 @@ def _transform_for_nemotron_refinement(self, proof_attempt: str, error_message: } ) + def _parse_gpt_oss_output(self, content: str) -> tuple[str, str | None]: + """Parse gpt-oss model output to extract thinking and final content. + + gpt-oss models output in the format: + <|channel|>analysis<|message|>...thinking...<|end|><|start|>assistant<|channel|>final<|message|>...final...<|return|> + + The chat template expects analysis content in 'thinking' field and final content in 'content' field. + + Returns: + tuple of (final_content, thinking_content or None) + """ + import re + + # Check if the content contains gpt-oss channel tags + if "<|channel|>" not in content: + return content, None + + thinking = None + final_content = content + + # Extract analysis/thinking content: between <|channel|>analysis<|message|> and <|end|> + analysis_pattern = r"<\|channel\|>analysis[^<]*<\|message\|>(.*?)<\|end\|>" + analysis_match = re.search(analysis_pattern, content, re.DOTALL) + if analysis_match: + thinking = analysis_match.group(1).strip() + + # Extract final content: after <|channel|>final<|message|> until <|return|> or end + final_pattern = r"<\|channel\|>final<\|message\|>(.*?)(?:<\|return\|>|$)" + final_match = re.search(final_pattern, content, re.DOTALL) + if final_match: + final_content = final_match.group(1).strip() + else: + # If no final channel found, try to strip all channel tags and use what remains + # This handles cases where the format might be slightly different + final_content = re.sub(r"<\|[^|]+\|>", "", content).strip() + + return final_content, thinking + + def _make_assistant_message(self, content: str, reasoning_content: str | None = None) -> dict: + """Create an assistant message dict, optionally with thinking/reasoning content. + + Some models (e.g., gpt-oss) output <|channel|> tags that need to be in a separate + 'thinking' field rather than in 'content' for the chat template to work correctly. + + If reasoning_content is not provided, attempts to parse it from content if the content + contains gpt-oss channel tags. + """ + # If reasoning_content not provided, try to parse from content + if reasoning_content is None: + content, reasoning_content = self._parse_gpt_oss_output(content) + + message = {"role": "assistant", "content": content} + if reasoning_content: + message["thinking"] = reasoning_content + return message + async def _single_data_point_generate(self, data_point, data): formal_statement = ( (data_point["header"].strip() + "\n") @@ -243,8 +299,11 @@ async def _single_data_point_generate(self, data_point, data): ), ) + # Get reasoning_content if available (e.g., from gpt-oss models) + reasoning_content = generation.get("reasoning_content") + new_prompt_turn_list = deepcopy(prompt_turn_list) - new_prompt_turn_list += [{"role": "assistant", "content": generation["generation"]}] + new_prompt_turn_list.append(self._make_assistant_message(generation["generation"], reasoning_content)) prompt_turn_list_list.append( new_prompt_turn_list @@ -259,22 +318,15 @@ async def _single_data_point_generate(self, data_point, data): ): # check if successfully parse the code. We do not want to delete the turn if there is a parsing error. if self.cfg.delete_wrong_turns: prompt_turn_list = deepcopy(base_prompt_turn_list) + [ - { - "role": "assistant", - "content": f"```lean4\n{full_code.strip()}\n```", - } + self._make_assistant_message(f"```lean4\n{full_code.strip()}\n```") ] # only keep the latest turn else: - prompt_turn_list += [ - { - "role": "assistant", - "content": f"```lean4\n{full_code.strip()}\n```", - } - ] - full_prompt_turn_list += [{"role": "assistant", "content": generation["generation"]}] + prompt_turn_list.append(self._make_assistant_message(f"```lean4\n{full_code.strip()}\n```")) + full_prompt_turn_list.append(self._make_assistant_message(generation["generation"], reasoning_content)) else: - prompt_turn_list += [{"role": "assistant", "content": generation["generation"]}] - full_prompt_turn_list += [{"role": "assistant", "content": generation["generation"]}] + assistant_msg = self._make_assistant_message(generation["generation"], reasoning_content) + prompt_turn_list.append(assistant_msg) + full_prompt_turn_list.append(assistant_msg) if code == "None" or "**Error**" in full_code: if code == "None": From 7c3957d8b215ae31079198d20bdc719d113eabf6 Mon Sep 17 00:00:00 2001 From: Wei Du Date: Tue, 16 Dec 2025 00:06:35 -0600 Subject: [PATCH 52/88] add Nemotron-Math-V2.pdf (#1113) Signed-off-by: Wei Du Signed-off-by: Igor Gitman Co-authored-by: Igor Gitman Signed-off-by: Cheng-Ping Hsieh --- README.md | 1 + docs/releases/index.md | 6 + docs/releases/nemotron-math-v2/dataset.md | 228 +++++++++++++++++++ docs/releases/nemotron-math-v2/evaluation.md | 157 +++++++++++++ docs/releases/nemotron-math-v2/index.md | 35 +++ docs/releases/nemotron-math-v2/paper.pdf | Bin 0 -> 469388 bytes docs/releases/nemotron-math-v2/training.md | 101 ++++++++ mkdocs.yml | 13 +- 8 files changed, 537 insertions(+), 4 deletions(-) create mode 100644 docs/releases/nemotron-math-v2/dataset.md create mode 100644 docs/releases/nemotron-math-v2/evaluation.md create mode 100644 docs/releases/nemotron-math-v2/index.md create mode 100644 docs/releases/nemotron-math-v2/paper.pdf create mode 100644 docs/releases/nemotron-math-v2/training.md diff --git a/README.md b/README.md index 0153961b34..f396558675 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,7 @@ Here are some of the features we support: - [Model training](https://nvidia-nemo.github.io/Skills/pipelines/training): Train models using [NeMo-RL](https://github.com/NVIDIA-NeMo/RL/) or [verl](https://github.com/volcengine/verl). ## News +* [12/15/2025]: Released the recipe for reproducing [Nemotron-Math-v2](https://huggingface.co/datasets/nvidia/Nemotron-Math-v2) and [Nemotron-Math-Proofs-v1](https://huggingface.co/datasets/nvidia/Nemotron-Math-Proofs-v1) datasets that were used as part of the training data for [NVIDIA-Nemotron-3-Nano-30B-A3B-BF16](https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16). * [11/25/2025]: Added the recipe for reproducing the main [experimental results](https://github.com/NVIDIA-NeMo/Skills/tree/main/recipes/proof-gen-verification) for [Scaling Generative Verifiers For Natural Language Mathematical Proof Verification And Selection](https://arxiv.org/abs/2511.13027). * [08/22/2025]: Added details for [reproducing evals](https://nvidia-nemo.github.io/Skills/tutorials/2025/08/22/reproducing-nvidia-nemotron-nano-9b-v2-evals/) for the [NVIDIA-Nemotron-Nano-9B-v2](https://huggingface.co/nvidia/NVIDIA-Nemotron-Nano-9B-v2) model by NVIDIA. * [08/15/2025]: Added details for [reproducing evals](https://nvidia-nemo.github.io/Skills/tutorials/2025/08/15/reproducing-llama-nemotron-super-49b-v15-evals/) for the [Llama-3_3-Nemotron-Super-49B-v1_5](https://huggingface.co/nvidia/Llama-3_3-Nemotron-Super-49B-v1_5) model by NVIDIA. diff --git a/docs/releases/index.md b/docs/releases/index.md index 60e103769e..ae997f352f 100644 --- a/docs/releases/index.md +++ b/docs/releases/index.md @@ -8,6 +8,10 @@ On this page you can find a list of papers, model and dataset releases that were ## Releases +* [Nemotron-Math-v2](nemotron-math-v2/index.md) dataset + +* [Nemotron-Math-Proofs](nemotronmathproofs/index.md) dataset + * [OpenReasoning](openreasoning/index.md) models * [OpenCodeReasoning](opencodereasoning/index.md) dataset and models @@ -18,6 +22,8 @@ On this page you can find a list of papers, model and dataset releases that were ## Papers +* [Nemotron-Math: Efficient Long-Context Distillation of Mathematical Reasoning from Multi-Mode Supervision](./nemotron-math-v2/paper.pdf){:target="_blank"} (2025) + * [Scaling Generative Verifiers For Natural Language Mathematical Proof Verification And Selection](https://arxiv.org/abs/2511.13027){:target="_blank"} (2025) * [GenSelect: A Generative Approach to Best-of-N](https://openreview.net/pdf?id=8LhnmNmUDb){:target="_blank"} (2025) diff --git a/docs/releases/nemotron-math-v2/dataset.md b/docs/releases/nemotron-math-v2/dataset.md new file mode 100644 index 0000000000..1e99df2a45 --- /dev/null +++ b/docs/releases/nemotron-math-v2/dataset.md @@ -0,0 +1,228 @@ +# Dataset construction + +Nemotron-Math-v2 dataset consists of mathematical problems collected from [AoPS forums](https://artofproblemsolving.com/community) and [Math Stack Exchange](https://math.stackexchange.com/) and [MathOverflow](https://mathoverflow.net/). + + +## Data Overview + +This dataset is constructed from AoPS and StackExchange-Math forums. Because forum threads contain discussion, commentary, and sometimes multiple or incomplete questions, we first use an LLM to perform problem extraction, isolating explicit mathematical problem statements from the original threads. Each extracted problem is then passed through a series of LLM-based classifiers to determine whether it is a proof-style question, a multiple-choice question, a binary yes/no question, or an invalid or context-dependent prompt; all such items are removed. We then attempt to extract the final answer to each problem if it's been identified in the forum discussion. We further perform benchmark decontamination by removing problems that overlap with public math datasets. + +### AoPS Problems +We directly a subset of [nvidia/OpenMathReasoning](https://huggingface.co/datasets/nvidia/OpenMathReasoning) dataset, removing all converted proofs (we found them to be low quality) as well as doing further difficulty filtering as described below. + + +### StackExchange-Math Problems + +We collect all math problems from StackExchange, including content from [Math Stack Exchange](https://math.stackexchange.com/) and [MathOverflow](https://mathoverflow.net/). We first preprocess the raw crawled XML files and extract the problem description as the key `forum_post` and the associated discussions as the key `forum_discussions`, matching the exact data format produced by [`prepare_raw_data.py`](https://github.com/NVIDIA-NeMo/Skills/blob/main/recipes/openmathreasoning/scripts/prepare_raw_data.py). This allows us to reuse the full [problem generation pipeline](https://github.com/NVIDIA-NeMo/Skills/blob/main/docs/releases/openmathreasoning/dataset.md?plain=1#L54), using only the ‘extract_problems’, ‘classify_problems’, ‘extract_answers’, and ‘decontaminate’ stages, while excluding 'convert_proofs' stage. We further remove all binary, multiple-choice, and invalid problems. + +!!! note + All StackExchange data used in this dataset comes from official data dumps released prior to the July 2024 policy change, when the content was licensed under CC BY-SA without additional usage restrictions. We do not include any content released after this change. + + +## Solution generation pipeline +We use [gpt-oss-120b](https://huggingface.co/openai/gpt-oss-120b) to generate solutions in three modes (‘high’, ‘medium’, and ‘low’), both with and without Python Tool Integrated Reasoning (TIR). + +## Data generation with Python TIR +```python +from nemo_skills.pipeline.cli import generate, wrap_arguments + +cluster = "slurm" # change this to match your cluster config name + +# with python +generate( + ctx=wrap_arguments( + "++inference.tokens_to_generate=120000 " + "++inference.temperature=1.0 " + "++inference.top_p=1.0 " + "++prompt_config=gpt-oss/math " + "++inference.endpoint_type=text " + "++code_tags=gpt-oss " + "++code_execution=true " + "++skip_filled=true " + "++server.code_execution.max_code_executions=100 " + # Change reasoning_effort to high / medium / low to control the reasoning depth + "++chat_template_kwargs.reasoning_effort=high " + "++chat_template_kwargs.builtin_tools=[python] " + ), + cluster=cluster, + expname="gpt-oss-generation-with-python", + model="openai/gpt-oss-120b", + server_type='vllm', + server_gpus=8, + # We generate 8 solutions with Python TIR for each problem + num_random_seeds=8, + # Change the filepath to StackExchange-Math Problems to generate the corresponding solutions + input_file="/workspace/aops_problems.jsonl", + output_dir="/workspace/with-python", + # any vllm arguments can be used here + server_args="--async-scheduling", + with_sandbox=True, + num_jobs=1, +) +``` + +## Data generation without Python TIR +```python +from nemo_skills.pipeline.cli import generate, wrap_arguments + +cluster = "slurm" # change this to match your cluster config name + +# without python +generate( + ctx=wrap_arguments( + "++inference.tokens_to_generate=120000 " + "++inference.temperature=1.0 " + "++inference.top_p=1.0 " + "++prompt_config=gpt-oss/math " + "++skip_filled=true " + # Change reasoning_effort to high / medium / low to control the reasoning depth + "++chat_template_kwargs.reasoning_effort=high " + ), + cluster=cluster, + expname="gpt-oss-generation-no-python", + model="openai/gpt-oss-120b", + server_type='vllm', + server_gpus=8, + # We generate 8 solutions without python TIR for each problem + num_random_seeds=8, + # Change the filepath to StackExchange-Math Problems to generate the corresponding solutions + input_file="/workspace/aops_problems.jsonl", + output_dir="/workspace/no-python", + # any vllm arguments can be used here + server_args="--async-scheduling", + num_jobs=1, +) +``` + + + +## Prepare SFT Data + +After generating all data from the previous steps for both AoPS and +StackExchange-Math problems, follow the steps below to construct reliable +expected answers and filter solutions for supervised fine-tuning (SFT). + +### Prepare Expected Answers + +**1. Aggregate candidate solutions** + +For each problem, we aggregate previously generated solutions into a single +candidate set: +- 8 solutions with **Python TIR** +- 8 solutions without **Python TIR** + +**2. Initialize expected answers** + +Each problem starts with an initial expected answer: +- the forum-extracted answer (if available), otherwise **missing / unknown**. + +**3. Answer-level judgment** + +We judge **answer agreement only** by comparing the **final answer** of each of +the 16 model-generated solutions against the current expected answer. + +- Only the **final answer** is used for this judgment. + +This step is performed using the +[`judge_answers`](../../pipelines/llm-as-a-judge.md) stage. + +**4. Majority vote and expected-answer repair** + +Based on the judgments in Step 3, we finalize (or repair) the expected answer: + +- **If the forum-extracted expected answer is missing**: + Set the expected answer to the **majority vote** over the 16 model-generated + final answers. + +- **If the forum-extracted expected answer exists**: + - If **at least one** model-generated final answer is judged to agree with it, + keep the extracted expected answer. + - If **all** model-generated final answers are judged to disagree, replace the + extracted expected answer with the **majority-vote** answer over the 16 + model-generated final answers. + +The **majority-vote computation and expected-answer replacement logic** are +implemented in +[`aggregate_answers.py`](https://github.com/NVIDIA-NeMo/Skills/tree/main/nemo_skills/evaluation/aggregate_answers.py) +via the `fill_majority_answer` stage. + +**5. Re-judge against the finalized expected answer (filtering for SFT)** + +After the expected answer is finalized in Step 4, we run [judge_answers](../../pipelines/llm-as-a-judge.md) again +to judge each model-generated solution’s **final answer** against the finalized +expected answer. The resulting labels are used to filter out incorrect solutions before preparing the SFT dataset. + + + +## Prepare Data for SFT + +Use the following script to prepare **6 types of SFT data**: + +- **Reasoning effort** (`EFFORT`) + - `high` + - `medium` + - `low` + +- **Execution mode** (`USE_TOOL`) + - `True`: with **Python TIR** + - `False`: without **Python TIR** + +### Configuration + +You can control the behavior **directly in the script below** by setting the following variables: + +- `EFFORT`: one of `high | medium | low` +- `USE_TOOL`: `True` (with Python TIR) or `False` (without Python TIR) + + +```python +import os +from nemo_skills.pipeline.cli import wrap_arguments, run_cmd + + + +CLUSTER = "slurm" + + +INPUT_PATH = "/path/to/input.jsonl" +OUTPUT_PATH = "/path/to/output.jsonl" +LOG_DIR = "/path/to/logs" + +EFFORT = "low" # high | medium | low +USE_TOOL = False #True| False + +EXPNAME = "prepare-sft-data" + +EXTRA_ARGS_CODE = ( + " ++chat_template_kwargs.builtin_tools=[python] " + " ++assistant_end=\"'<|return|>'\" " +) + +extra_args = EXTRA_ARGS_CODE if USE_TOOL else "" + +cmd = ( + f"python -m nemo_skills.training.prepare_data " + f" ++input_files='{INPUT_PATH}' " + f" ++output_path={OUTPUT_PATH} " + f" ++filters.drop_multi_boxed=false " + f" ++filters.trim_prefix=false " + f" ++filters.remove_no_think_tags=false " + f" ++filters.remove_contaminated=false " + f" ++filters.remove_len_outlier_solutions=false " + f" ++filters.remove_len_outlier_problems=false " + f" ++use_judgement=true " + f" ++prompt_config=gpt-oss/math " + f" ++tokenizer=openai/gpt-oss-120b " + f" ++exclude_optional_keys=False " + f" ++chat_template_kwargs.reasoning_effort={EFFORT} " + f" {extra_args} " +) + +run_cmd( + ctx=wrap_arguments(cmd), + cluster=CLUSTER, + log_dir=LOG_DIR, + expname=EXPNAME, +) + +``` \ No newline at end of file diff --git a/docs/releases/nemotron-math-v2/evaluation.md b/docs/releases/nemotron-math-v2/evaluation.md new file mode 100644 index 0000000000..d184b61b3e --- /dev/null +++ b/docs/releases/nemotron-math-v2/evaluation.md @@ -0,0 +1,157 @@ +# Model evaluation + +Here are the commands you can run to reproduce our evaluation numbers. + + +## Prepare evaluation data + +```bash +ns prepare_data comp-math-24-25 hle +``` + +## For comp-math-24-25 +Below are the evaluation commands for three reasoning modes (high, medium, and low), with and without Python TIR. + +```python +from nemo_skills.pipeline.cli import eval, wrap_arguments + +cluster = 'slurm' +modes = [ + 'low', + 'medium', + 'high' +] +max_length = 120000 +model_path = '/workspace/final_sft_model' +for mode in modes: + output_python = f"/workspace/final_sft_model/{mode}/with-python/" + output_no_python = f"/workspace/final_sft_model/{mode}/no-python/" + + #with Python TIR evaluation + eval( + ctx=wrap_arguments( + f"++inference.tokens_to_generate=120000 " + f"++inference.temperature=1.0 " + f"++inference.top_p=1.0 " + "++max_concurrent_requests=1024 " + "++prompt_config=gpt-oss/math " + "++code_tags=gpt-oss " + "++code_execution=true " + "++server.code_execution.max_code_executions=100 " + "++server.enable_soft_fail=True " + "++inference.endpoint_type=text " + f"++chat_template_kwargs.reasoning_effort={mode} " + "++chat_template_kwargs.builtin_tools=[python] " + ), + cluster=cluster, + expname=f"with-python-comp-math", + model=model_path, + server_type='vllm', + server_gpus=8, + benchmarks="comp-math-24-25:16", + output_dir=output_python, + server_args="--async-scheduling", + with_sandbox=True, + ) + + #without Python TIR evaluation + eval( + ctx=wrap_arguments( + f"++inference.tokens_to_generate=120000 " + f"++inference.temperature=1.0 " + f"++inference.top_p=1.0 " + "++max_concurrent_requests=1024 " + "++prompt_config=gpt-oss/math " + "++inference.endpoint_type=text " + "++server.enable_soft_fail=True " + f"++chat_template_kwargs.reasoning_effort={mode} " + ), + cluster=cluster, + expname=f"no-python-comp-math", + model=model_path, + server_type='vllm', + server_gpus=8, + benchmarks="comp-math-24-25:16", + output_dir=output_no_python, + server_args="--async-scheduling", + ) +``` + +## For hle-math +For hle-math it's necessary to run LLM-as-a-judge step to get accurate evaluation results. We use the [Qwen2.5-32B-Instruct](https://huggingface.co/Qwen/Qwen2.5-32B-Instruct) model as the judge which can be specified as follows. + +```python +from nemo_skills.pipeline.cli import eval, wrap_arguments + +cluster = 'slurm' +num_gpus = 8 +modes = ['low', 'medium', 'high'] +num_chunks = { + 'high': 4, + 'medium': 0, + 'low': 0 +} + +model_path = '/workspace/final_sft_model' +for mode in modes: + output_python = f"/workspace/final_sft_model/{mode}/with-python/" + output_no_python = f"/workspace/final_sft_model/{mode}/no-python/" + eval( + ctx=wrap_arguments( + f"++inference.tokens_to_generate=120000 " + "++inference.temperature=1.0 " + "++inference.top_p=1.0 " + "++max_concurrent_requests=1024 " + "++prompt_config=gpt-oss/math " + "++code_tags=gpt-oss " + "++code_execution=true " + "++server.code_execution.max_code_executions=100 " + "++server.enable_soft_fail=True " + "++inference.endpoint_type=text " + f"++chat_template_kwargs.reasoning_effort={mode} " + "++chat_template_kwargs.builtin_tools=[python] " + ), + cluster=cluster, + expname=f"with-python-hle-math", + model=model_path, + server_type='vllm', + server_gpus=8, + benchmarks="hle:4", + num_chunks=num_chunks[mode], + judge_model='/workspace/Qwen2.5-32B-Instruct', + judge_server_type="sglang", + judge_server_gpus=8, + extra_judge_args="++inference.tokens_to_generate=4096 ++server.enable_soft_fail=True", + split='math', + output_dir=output_python, + server_args="--async-scheduling", + with_sandbox=True, + ) + + eval( + ctx=wrap_arguments( + f"++inference.tokens_to_generate=120000 " + "++inference.temperature=1.0 " + "++inference.top_p=1.0 " + "++max_concurrent_requests=1024 " + "++prompt_config=gpt-oss/math " + "++inference.endpoint_type=text " + "++server.enable_soft_fail=True " + f"++chat_template_kwargs.reasoning_effort={mode} " + ), + cluster=cluster, + expname=f"no-python-hle-math", + model=model_path, + server_type='vllm', + server_gpus=8, + benchmarks="hle:4", + num_chunks=num_chunks[mode], + judge_model='/workspace/Qwen2.5-32B-Instruct', + judge_server_type="sglang", + judge_server_gpus=8, + extra_judge_args="++inference.tokens_to_generate=4096 ++server.enable_soft_fail=True", + split='math', + output_dir=output_no_python, + server_args="--async-scheduling", + ) +``` \ No newline at end of file diff --git a/docs/releases/nemotron-math-v2/index.md b/docs/releases/nemotron-math-v2/index.md new file mode 100644 index 0000000000..20515fff3e --- /dev/null +++ b/docs/releases/nemotron-math-v2/index.md @@ -0,0 +1,35 @@ +--- +date: 2025-12-15 +--- + +# Nemotron-Math-v2 + +## Nemotron-Math-v2 Dataset + +Using our pipelines we created [Nemotron-Math-v2 Dataset](https://huggingface.co/datasets/nvidia/Nemotron-Math-v2). +This dataset contains + +* 350K unique mathematical problems sourced from [AoPS forums](https://artofproblemsolving.com/community), [Math Stack Exchange](https://math.stackexchange.com/) and [MathOverflow](https://mathoverflow.net/) + * 7.5M natural language solutions generated by gpt-oss-120b + * with and without Python tool use + * 3 reasoning regimes, high, medium, and low + + +We used [Qwen2.5-32B-Instruct](https://huggingface.co/Qwen/Qwen2.5-32B-Instruct) to preprocess problems, and +[gpt-oss-120b](https://huggingface.co/openai/gpt-oss-120b) generate solutions. + + + +See our [paper](paper.pdf) to learn more details! + + +## How to reproduce our results + +Browse the sections below to see all commands needed to fully reproduce our results. + +Please note that unless you have an access to a large GPU cluster, it might take a very long time +for some of the commands to complete! + +- [Model evaluation](./evaluation.md) +- [Dataset construction](./dataset.md) +- [Model training](./training.md) diff --git a/docs/releases/nemotron-math-v2/paper.pdf b/docs/releases/nemotron-math-v2/paper.pdf new file mode 100644 index 0000000000000000000000000000000000000000..539889acda6e65f7db3f7ab77bf7e6fdca6b267b GIT binary patch literal 469388 zcmeFYWmJ{X);6rPq%=rMN^ZJCy1QFq6C&NnrbGlmN;(y!ySqV=?vxOuyBprc@tpHK z&-;Dfzi+%_yki`O4#tMfz3(;GTytLYx~`j6RZ^OTos}Dvw)H&o4wZwGmy(Ur!R$4v zkPszA(Z(KX>TKf;wV-4NuPC{A_&NFbC?T?x29z8;9GsM#Ts)LUln@n44leKpDN0Uu zehwa9N{BKgJ0~SX6Z`=$4?j1hhzKgw-s0h^l>djmz(V-`+d?$G9HEpDU3I7>SO(PI z%GDa&AXtdB%^O#!Gx+|-)D* zb5{pvN_GygDoThBhy*(&C-@J-AZKp@^?V2o2qFhB|KA^3I#60hgs3|>xPo8u z|9zza;!4TQ&hw9}y8jIo&;K4Ch@2#)feLOc#?NkH#?QenV)U>HF0Rf{Q#(}8 zRC6OH=g#aC=urFQ4CBveZ04ymY+6{e?iOvI9B(t(mkNiTPTh+pAOx8;=k%SK?JD~9{I z{%W%~e?>|wBgYsMF64sz3!^j2LWupqo{07X-IGTnN(2cHY4OjS`ahf?|9hbS^ZGv# z_)i4>6M_Fk;6D-gza|0z-Y}A%9}Hh+!PhH8}GlP zXVgP0jxqO@wX3`xt`HF9Ut4@$Eg}9xwA{{TF*98wBPuH|?&guCkY;>>>nu`i8QL#P z(e|oVOH%g0v1QI@Lz^ARSkDdM9t7u~`CClGo-)c>`$V)ph9t}Xyua`AHfV@J4nIsf_n zFSz95{kv0KynpwVi}wM@tAqDH;9&5b?_r}JUjNMwF8;r-0c@3Uuy=*pgR66J{kWUQ=<02Wb27N8J3}tQ_ED__)|v`Tl`&0agYV_y^|w3v{6_4sOoo z56SqiH2a%C4_Wm;vgctj|4LtR8&?-qsI!EFouh;O!}UPc0POAHtl?;C{tsO4ZetEr zml6N}!0ob78!KyY7wiD2Ybcqz*n$Q96M1X6nYsRB1dy|PxC6*P=K&S7*HCj;fZ}Z| z9wq`d?*B?qPF^-H{{NApJ86innp0=@<}^W1zvw#U-h9D=;GyE#1!PKv#n+s~otFjG zJ$@4Z)KJ{Xz+ zc+%>$lt;2Z3h~M{Qeg+q-yLiJF6BjiyW_ZYvFKm8JT!%R-sy`S;>iMwra}p%1pkVWu01$EqQqq7o{@?M@uP7Y8?S3hnQWhIJydA#%@KwjJs_H!K-QIu2EdYK4 z?QJN%UM{sO=`&)CYUwb+Ha_crhbD@S3J2{n5ab0EQS{txhaA49Kty3+JpXh4=kH<( z3Apa&wm$_YywfG(9KFN6+Znb5ztg`-%x-`pU9?QT_)b;jV z8O8wxZ(u^IFK_#(`2~BVsmw2Aa`$aCIEz*n!lw-1QzMX=X@?1CF>&f2`^n=Wmv4`Z zif~5cVEyZ^WXUr1pQC&nkMU@q@eo#5sldf*3b`AqZhOBD%gs2yJ^OY(HJNTg=0f4t z1?yP}F#)&hztK|~{$8z1iE#z0Gl{2pbi5s`nL?v*FOGgV<;+OMArHTFt|8yl;gWG> zWq2yff5_oY9}=6=t}xV+VzD=>kda+i+EVutHd27}&lHS0Lw^ytb5u4h{^QF|KZJsJJoa+zN)Jq^GUc8PR;_%ZsoIq5s)^a+B90s+)Yd$3 zsqB@A37cO*C6%by_sU7Pq)l!PzuUhp!_a|S%Q$%E(pH}m@vei_;OB5mu$n1)oY23c zu#4)r+2_Uo{@1tEql0NU!TpotQge3`G=1w`;Vb_>!X=6=0WS5E{JTf=CO_0NAxDgv z)*KZvCQ(M%1Znr*g;%?d^ZZOO3>`uf+*{IVT%5lq_B9;E~BKfMf_Us*4-}iYV1R%3I%`E(%iQEsg%7G zO)(Oh?LvT_hwhUc4+(9Gb5hfOwFyLqu8!VXXWoCE;wn=~GgDuv zqU4^-Io>vJx&C>FZ~fBwJbUDGkQQ%uX|ZJ_FLPj+124$?5?d}%TFNrZ7@yZq3-eu< zxE%dn9}z4;zO`88N005c-;{ibYkE6JK=H4FGE20q??fvqTFRlHoC6zT)!whTCv{0k zm$b=&ah!j(W`Lg}BapWiWu`OmQ4{BVEvG>?Zvb|LosaL#HbL zqOY24ytN`Pm&Z0-TrvLZDO(-Bt~(xtly7eW<5bJD3f>y}K8P&Mo%)V7HGaP@6Y2z?Q3Zbz5g zSO`o0<7^2jgz?-DsoP=ta;Zc`Do|4`GmC{nosmWr&rj^Dt~{|wp{5d!4Ii2|!#;jQ z6%-U?Vk(4J$E?vuSz%#eaad^DTv}?gtRw?p-}w0X-vdP7!| zEs5Trg^vxxc6Q$I^70xO7|2`&K+9`u&kqh{m6e0f%|U&T^GCHT*F`VFB8AhYKEZjk~|BWE#Lm{-@m=Qyl5%> zNr?BB85x2yp7bgx??Ij0T+PhQjSLN!B*z-k9lbhNQB_hVPrbfk^7T3XXva-Yl3C3S z2*VxJ@$vCdC`Ii+fHoB!{K~Cy5||1kNIxJqU;CwH@C-S-TT{LAW;;yawdAj+-o!fb z__tg62G5x=>s&iAVXjzSV}zHNmpM5(D?cJBn4z+T@7a7W4t}qE|CAyEyYg6s@!rC7 zTK4Ofy=w1`W6~+sul71#gI%8`B_*W^c^wU=2@Yk5BCBw4a#nT=|f|#@zw)zUs zYb$+*Lrp#*K&o3+Nf(-_vca9tQAg{4X1`A1dtuuB5x-ER%(N2;27{rYg>`JFY5?C=l z%$;h-j2+S}^Uhp*a3ZPbVjCYqtc(~!X=i8mJsvuk%1?QoY`t)KyncPYKNN2I<)s`l+Uoi`NJlI( zfmmXV&9yK{3lZC^wl8AuzDJN<-<~f*3~JCde}f&uA{VkYH$UDOp?>@Jt%HMuD(OM& z=X=VZ8yg!&jqWzq)@)_2@W_}!Ka6Z$L#3!j=4u_*hcbrR7`fB36%wv4FRMSiXmJm& z(^ri*A;wLG-K$*?C`5jh-oZp-mJ*iH-=f}>6P(0$k`x&A)&Nagv}rG;^|^uGVtE#} z%&pVKWUSecv#~odq_XOYLIzQ^wjc)usWu0NPMx|4=i>#H6WWgcM_Vb!js7D_<(T?U ziymvN!Wc6JL1rDh=%^2eH#IetOwiqRceXk^W0=K16Aubq#UvBR%g+yJJSz3OwLRUM z`2PKSt^GXQ5Dg1U@wab5L+hL_tj;bjN{WhYP%~%jmsa)l_3WHtqy8=^Ez2>q3W<8S ze29)1T(_JTV3omG6ocV*VCD3?-`m2KGem^FPpmB!p*VHXaj~&W#@cFXYFb*m)8*zF zA}~6|#4?{tXUAEPrB9|*)YWlb6jfAkT%N#-4Fv}7%~aCE_i;P_=|RRKuabtT85zNL zrrEv5dZ?wt%c`oFSy}BjSH!mK|P+~YqA+^?EzFC-7L^LbkH+kPY2puMjY>CTb=m2u}`?R`@#{F*TOoHa?I zIbp69g~L*^8F4I(zNSqiyQ{E_(fAh*Bm9e1=(o5F&+N4KTJ!rwa^pL8PPI(~==|e^ z_m@L{ZPoUh^i@s6^)Bnaw^spJJZ@X#T7}x+lsCh!d4z@2T!-I|1x($B5kgw-@7~g_ z&oz4N+fJ1Pj7gpESOoN~rt>+$rBYc=TI~I*A4(Ut^*-rY^t->E&@o{*pKHCpbpO<+=yIsB42yut`V?0m58OFySa; zzCZW7KR@{0Ihh9;9UYx4HbBOS3D9~i@Cr*8v8MXuh!U@jFg6$Ot#@vkAL-$o(~#QI zqx~4|mzg~2GAa#nk5S=B#V>B>-DB>(hO@5$j>v8LVf-Y~gV%L;CBzPr<-t!!)R>#@ zlVFba*d_9rKooguBeehekVmcOIU*YwaWYn=kbW5VWa<55KNR&#!s`z8hG| zKJS=p4Y`YS8O6>xQ82BKTtVa4TYbKIV@Riy$n+wDp=^bQypFsIh545)1lwMqF8cW~ zz>f_?e2z=Y-C@;LRf8F#`^(EQX=zG}^-n`3^YioPYV6FN13}@Kot+&S895nHaF>?{ z9eEM6g^JU*!fku9r>7?;_s^FWYZlj?X>7yo)xLyx^nQoSQgvCY^p_~7-Byp^?e6T* zsigO};czF&eWrWJ{F<3h!;foo_)4h+UTkAtoFN-|PvXBD&7#8lD4R7L9V`@3fk;0`E&ZFRwS!yuklvBCppG8>utA#43RHZpid+}8LFKf2G5QT;0gbr!!()E$@JPWghdg4d|;&FI`c-&6UTMIZNVq|rCG^0;hC zyFQnU3%JT88?Ce&0>>|1z^(Vk4|)hBvgIk7i;K%aAW=#}LBUqXdo)l1ZIUB)o+j<1 z5iq~96a(EkJUl!(&i~z2CN~_Ga9tK?}+fsQ>7H}3gd=X zC!6L_Xk1d#%xG9pltiiy_DZz(mh|wlIeDiNUs~t^4cSMIy3?&}ou}B^Urbo9h8`_0 zP@lSb?j8TZ8EWdyqq0Y}Mm;#NmCW!a<)4UDg{JrsIxsK=4N@`b1}m5qhWx6d9IrnL8& z{Mb2F6eNqj7HZ{)&xtElQTCc{JVEmipCWfAd8vqT+#&}?OFRF^@JP%iEjMie@XQ2Z zeXHtkr40=YCyVqt{}3Q~>LDQ^#i|4`s%vPR?=NEYMg(O0L+>|pGe8?ihCbg4n^XoR z#_!@cG7NSLI@Q=S_)9=6_#_`49f6v3x;OuTJOeT_%@$i=5Tl0i+)J=y-@}O~CnxXj?)sdnzo#-?IY>7jo*Q)c*Eh!fC?gRIBg0i(MkVtKbe5lNRRi-Fq9~-M?YGSz zcbBP!pnkE?krQxqS;W4DfmZipem_zY)H*Dtb3jqe1G4@k z3VN^#HLo6veS8{wSRw9A-V{FLHR2E-`uV$k<2eZiM}zwTnj{$we9f|4DLYmNuS-n7 zgfqEi?3IqHQCg@7OwI2pe#x2ayGC@~k23SfCQl4s5$3LEc(D`;1P6|y z))xkZDW;B_YMqY=jqz}TO%raT)vdG~_)M>o_i+&u9X*xPDgq2U<{FRAr+a(lF-a)i z75c4w$N28ztRS7t%xHW1G;s zjKPt3M9UMY+pp(eS2NgUPGiqojV15txz5#;r97+sOO(6B%PAS}2R`>K7Bx$6@-xbz zPMhV14}rPe4o6E%OBI2Khi7whvmzt5s*3A?>*{Q`FYy+%mK0ux_|B_c$!;Rct3HqI zNsJ~B94nA>DF{B@#Rj#Y^Y9cV64|Jl+~86k1_lNau;_QS_4Z&+6XW8((<5?zFU$O* zNd82ww*J{gCt#bdn4x4J&h?PK!RpUhnrQtPQv_@bk1_`2*}?{DnTnOypD=Y*a~VHk z`2rEao1$HA3Fm(Xc4ZOGfFc%F`%S<|QbpK6&W*wTqse!Cl2+p2H(J8aymM;HL7Ek3 zW@d7L?0JWbwYJvVK&-E=?S6lE3%VM+Wm9J6^2z2{XJ;oveO{h8GeN7@vE}cM_r1Nn z=*%6l3>x+eO?t))I~VF8BKS{hT-FD{q`(Sq@HP)}_hh*NF@;!{c!rq!_4w{wt%AI~ zzV}SjIoKcsi+N9?md&84R&-s)7*CR<<2u?ARlt&Aj`t2MWz1_R(-80zI(-hJq7#r| zD}`(Y(^O09W1|I!YOHha(+Q(5E@yZZ2PsVb_*szLaUp(Im?#Vwxsbi*EPLPkb^!#a zA?A6L`2PL-@J1r>uWVpw0^^FJKIZbq>GqURx}jk_LK!BQp`Ho2zTjjkbKlih)ulPI z2Ki|!;JR@zz`Hm-J#7JnMjM>b1zs&@ghWP~R9b!t4n~4!q89Apv9_?VP=Jt7IVou< zG#P!4Ogl}2izxrLmH&pF$9t6PVBQ^v_d4&ojOj5N>GG##R6U>PS+(kDTVYocGRe+x z>f1#_79Tv#bw6;B8)5xyD$rg3(@E%)OHw`|HoMcQoBtgNoMJ3lbOnXJwMojW5D@VB zMi0)(@@E_KjpxzSGA}~+Gf{OBKxekRLB+?90~2HPbW@%bYy>k=z-{nAj zHIV*&e2IyPnI#ix`AYA_<8&G@iTd4Mxha$yHcG!(OVGBufEqSGyIAo}u=i$`BAdyy z4e5{{HBn`E*>i4pNOl?h-Csf3@<>$n#(cjgU2p6CWHJ89BKpsnVcNU(tZDNXx0H zi27VOp6|^Q>k<^GYG??0pS-A##1GjyKv>208cLV67yQjV>pw3R_R;?Z6~~Whxm)KY zxX5vhpWWnNO!Qv|I%h&czS6td(sI|4;zf9&H&9x4??iha?fWB2lWK|U^=t^eE1E{@4IJAH+&`IOe@k^_tcj-2S1(<1A~jFRc(PT2 zr|cbB{b?)|RDkudBkr7#lpR2m#oRMfyUVZ?F=;sf1S_hT%7-mvF^U8Ki-?nxlkQrs z9qPE;^`zjLSC6c=c6@#PWz=&CqC`(H*+r1@i#IrNZiSt}qZpHrq;zzMJLfLpf~(-^ z?qX9*vyI&ai!TK@uJsd-6DoG#^=@# zYA$m#^NBI=XQ$zO_%SqA`KCw18slJ-mvR>Bu@6H!1@gNq=Nh_GX5IAWUj+ysdHb>U0}D zBU$+E*`L)_wYQ(5sW~{DK}z*&Hthp&`a*0pC$q&`M=^zGccJ-h>^a{{*dexAd0kx{ zk+BLoGs@}8>~kzt3%W;dM|85I|ltgg_{Mj2$1b+J{w8pupT*eX;t-D`7CtgR8% zSzaix9Qu+p?uT4cvM}l=RRw^OYcx{tVs$_)yY5{%v*Wb#o!#c3K&$YNg+T_OQN{-Q zJ^DaeU0hs5@qqMi%#`?m2@0#5qa!7Z>KWo`w~S-GAwd|o-<=N z_wzYE{F|GQCc+15bKIQo_nW>vW>?aMzR#Z*R^FJOcN0vHO-b!lqIBPI7;76tHTCly z3*9YJaG#V{nr0Z#E{DYfnDLjQS-bc zjrHa;RKZ1gRMU3FP(zzun33HkM>6P2Xofeex8lZ7bh7Qw2H2}HJ|{&(nq)Rvo$f7R zVCl{5-X|Mi$c=H40)3~(Zf>&LR_A-s1JwoCFGfcF3keKQk*N|RKys15YfKIg>o&Tx zczS#*1_x;Y9QL^isDi5M(@oo)=KupxiHe%&=+v6D`hx8wLnmTeT55mC1>L6ff2nDt z8@G|>ewTT#*zq-Pq!~e}T;(e5{)<9n-q0sdne$&lhNl48a-R z=(gR;LI-@yQ(?ZG5pN?zmPHQyU3jV8ku%ZiYnm||S3Xn|qn_bcCkn2bA@i@;}LpB%+c;H^ig|e}@Or_UKXd_rHuAuf1 zHeb@eGXCwQ;(%1j<}x{M_XFpl+9LQM0RpnCt7|QtELs-t=~K41XK%hMYN@O1H+edR zd9;_+)*^Ez=BFWw_$MSJFh4iVgUr%%S`GT#T(E0BTmr_(#MCbS-P?8)A!OK~*8Znl zcX*AYF*Mxc>T+8QU{bCZn?IDgB#E8IOORR5uv?!Ax&`8o*(Cg9!IDx^ z(4#*;9Tr#bf^~uc)X$u^N|N`Q+@(e#V2!b7x7PZTK1ch#V!~y} zI`!+9Pn-Ya@Fdg-VZMGAA}YkY%KUr=zDMYaHC99EA27*c)D*$uYyC;xac!vQV$7r#GH&?%?N+QtxWgzdN9-~Ied&(fS~hmfAf!F>L+I0_%| zGaCY`QHjh9?lewo_$uD_eGETA6Npg$p~#Gbrr?!Sj(AptFb1+k#q&Pny{!XoM6ifL z^u}WCe&@G4h9`j{u;=9}%tZvT@rM<-VTRM8$l$!}0(vJbOF>P`K+ZetBp0*J=MYqe zSTl#OA!ZE7$=PL$0dcR~e;qSr#%MLh5P{{hh~8bSY)_S5`uJ8VKdo!2si{$jXErU0 z<~2kYbl*WiL(AHXxn>Rx4FzL`<*S^7vkZ{wW4S8On}(39%y~GE)gryh9y=vpgsm_E z1h~g}zWSZiI;F;T=H^-1*?=dNy^4SX-0sBmbX|QU?A!D135LD=Ny6I}PIvVL;km%4 zga?AbZ>oXBd}V3Kx|TC+37OQTCE zg2^JFy*5xsx6U`8_ETc!OEX!Ck+WF_B0-pwpas-oL4h1Y%MYy(CoKk3&nsfw*MyShISL7^?0VVm zAi=>ro&$E43&>M|$hOF_^8lC@+<*JZPj7kvTx7w3GFfT@$SB`~=1(c%fGiFOXpbNh z1n99+67|IZKneb$LXJx>=^O$qAA}77KE9r5ZlqI0F({O_SSLR-lj_$6AaFjIudc1x z&sK%*75^?b{{g7sxERU?ca*w1z&qK0xOuZRo+PXCMbgl})8_ZtYGI;L5;Q;@`?OJE zX-Vtocmx5Fl-sE7(7+N$5OqX!KYHiT+1Q!FA!l?gt;EST!Nk(#Kkx= z7}+qqf_F!@Egb>pXZdxEaEFMPmXez1-k7ae*ohuvg@=?4J}A;}E&udH1J;6WWPDE3lz=^Fuk2@O`vt5!J;o|9(}H=u7#Wq{dC zfqG-uEB*7w&W4R=L1N07(m^Hmsaa zKo%U<$C8w8aNDi{r9LFYa;oL{_}Jj@{Ar#Om=+x!-Jc-rHaW8H~Xu zGh-(W8;Jr)x3RwdXG=}>DAw#NFTicf%A#zv1Nt9mSq?K5h;&qrDZydJ78baZrba{EjcE#9ZE7n;1PjAg-}gTc{0KV(e%qiWX>GnT6?Tg3@! zd1U4bZ?a8Z7xRcKy^NoOavu~5IPFds~RvkARqEsaZ@^Fx)wE!&lgA zO{yOJDV3lh&^$O^Mw=-$KhuBvDLzPCPQw2ML!+i&Rb9)YdlSkcWoG9F!kWQf)wPcm zyfWTsHJ#}0kIfAjy%h<)V9DywIBKXC+GPDc(MEBqS70Ydm?gZ8c2H7XUA?lha#_{B za#&DU2z2(bu`yvGp%UX})Y3OkpS?rE7#$yvMg9uD9`9MZxZE!73-a?Tr3)po3xZOF zZib;9lY>^OhJ%X>J_-S1%6zN_93IFto;nM6EUK>4xkr5>N(fK*xMXEuKy*OMUF6=l_@qetZ zuYYRds6@lf?RI&z8b0%c-WE(Iw6wIYsmXqDTzE~lphe4`@4U3>*vV_aQ#0WTvWKZ5 z7a{$6271+ewe1v8)`%1Jv)b^vU$yyzQ}{U@WeaTy#0Bg^=Dx)6@K*^IW@eFDnMZ1z z4hus16oLoS6aWqZ*-rnpsR7z$TzR@pF6-3*oRU~9QOx`%m3k<~3|T2sKh}U(7jCQ? zv{;r|7IJzlFX$oY8kgi!D~w00cQdpdudj@!HxUz&=0MwRBxZuArM3vySP$|ouwqYM zW4|@3sz07ZTV5wQ!n9+=^Jxo1c2}6QZ6P9b)F_~tNeL?-JPU`0 z<0fcEn0v&Wzi;n@Dk_eEpA&SFiC-+?;D~}sH8P8_MZGqZfhB_VnSsbL(E5~hYhxtq zLuja+n;SQ+WmbzdiuqM$L|IuGm}glG>&miw72A(7d~`B$ACYq^o#99hrEFPD_#dO z$-onw#oTDW#KyDnxO!+!I#*_Y(AcK&bVHT8N$!q(4;&ahJ-ze$*K_qPl!Nm}t9_f- zO^?X$N{kwxxid1bu)uCE4!c5eZ3OJez$oqR;gL`t;gk}D!tPCey5DJk8v;4ggc-*N zNjV7sA(kTgx+*y{4XunPyQ_~?`TV5uP&$=0sJ%k4(3^tv4Vg_r$P<~{*r}-m__cWq zqg$Gz+FUh1p-(rrY8N$cziwwpzX2*M`d~|f;f|VSoF?_MItRo#q`RZ(qcw07#I5TD z-BvKeeMKHf6>R7+m4iYz&lGnoKw`=URUf=bwuB+4`18=4bnhi=`$2$_iWbL`hS}E{!Bd zN0)<+c7>_L2EZ^Vhu^7!4A8hU3x-oa)JpjAcx-0xz7)fr6;O#o32VUN6J=hPj z>Gjm}P14y9DiQcInPA<;d&7ECQiERZ`b#gcFW-3&Q z)h4@T6Y9H}?F&RJx$eEvOb@-cx7$to{esU$ML*q)i7d2n?xGi&z7F-EHOXO|`4;W2 zBkI**%jJXwRU?)Z|B$b-zCPl&su`qV&)CyH{7c5~+OKp-eJm%Fq5cK{XqFNP3LG39 z7#grF)k8@tEC-UCnwr*T4ULTKY;1n&daMA5SfX3rvACEKae9w(z`D2CD(bSPh_r@? z=6Q4Bq$hvt14f15s@N9FN3z~aD?=owoF5s6L>+ZA~lB@`9tcl&Fmg93N3@= zVHmfguj5(sGi=1KWj>}*d#NA*sh^ax@7qfJeDyV5^Uj~zu1#HNV{~kG`@RJ6?n1!Q z2jxs`Z;*U7G-A>ApT-GX+AA z;H`ZTyTRlJHP!SE;2W+zZQleJ+Aalp_@?env|&X=!9y(MiQ=9LDihvzweExERx9XKMO*Ko+%TyOp6|~WtV^3aSm(}B-^)%u zN;BIUzl<^vjbh||m*Dj+y`K>)A6w(I`LrYr@#0KvDIzO}(-uviNGuJ$)a=HuUKb)J zQ$K9pBxen#%ga<67lp{^u25?y8~-bD=|4&)A(B)oDnq~#!OnIbnaE{>(=^!In@om@ ziZrI*dPv>ix(O_Foo#bZ4J$yS1B}azylJIt zrkKP5oMKVdq+{u)Qig`V09LgbNCx^)Y;p0fC~OovtQ%M|1U>e?U1N_I8}Mpqj0FWD z1ncNxG87LwDp!=9T_xwmNKWut2;1BRvgeM)wIQ0;s}vyO(To#1;{v1d4H?^9NLGtM z4bSotzpbF`>zBk%nvko(sz~1Op10{b#I$(aEBToBA5{q`2J{sg^2_J6#WIR5cDb+Y zH}r?9H27D?KM=tJUyIpj-lYJL4wN(jjy$%G-4AkP=cJqQZ}fEIWb5-mSKdLWq~3$4 zb+3668gM1eQQaJ(6PumLI9FcB-z!KC0SR(-0zxFIOJ^jJ-2ar9Rc`oR40Ah&3 zPO_dlBIYWk06Wyx`M!g_eR{m~YCO~5sO81^;sX^SP1H{WzM(n-pj&-CJ>XH`o|-)? zf=s_T`}6bjrfsH@k5)+djb@+!*4FE~=uKbzf`WpMcY6R8hZh%r=hsSv^LwvsWQS<3!3j%% z-6T4;Y-fKMa6W(jEPIkfW}vs^T)(s%PRbt>K>Ue9^bW{}G3ilyHeh$&>gwvcDB0Cs1v>87%v(NVH|rY7k`YZgTHDy*GwncVn6O?SOap3} z*bPTz(^0T$mkPrnLhfG3wg0&w;rOMFz-U#7Ik^NFQM8^|43xo(@@G#Ut95C+Op$v> zaCA$`p)$!?dADV6e_Emq(5nrrW5`ghxIoC5AarZyTvE}XA5ggCY*rtN*BrY`E5i)P z6JXn(obmXKKEo)$5yoL8lihl|F8QbV3(a`3$g;kgrPuInnTcW=_i3Je&EU;T7%&6F z30VR2Al-Hg+}73>lTHabCMKP!qpRx)825=d@qeiu49j-C9c1I=07m?=Zu+k%i1O;PAhqp zL4yT8N@7nE`?(qn@)gAzf*9Pc>g8V_L#(3xlKIa^YI2J-HRm6#Z}}e+rqUc!l@vj- z3K6rHh8Lo_BDd)_f3Qd(5PDwgj|xaxEhxRBO$6Lt4^j209!zcVcWe!VrqCw$cUEOc zEr74<8?Fp#m$~BjuS4zq^^{w;q;?6T`#PwGI@;CUdE2X@snX zJum0>Kv8*zCR5ThL=VKrrnlALIq>*P~`zCgtPYJvN8^J>|TXPQa3 z&c}wZG{Xd+sqwMWx2P0P(H%l_Wto{;q`PbB$G$&URz0;apzkTR0ibUrX$YCTiXZet}V1QA_&g}xC-U6g*pDFA}0HC|d zXLh%?`9m-QNB6Ya!sBVu{z5mF_gZ{Rh763T)s3dMoPB{7Rb`oULy+TH{)~J9OvVbD z^mmLi^h$wcj<-iPf6ju$BX%w+5e6m*y^GGbT0$pS3vyaWH6%O(D|(s;b>RKD zqV-JtZ{NPb$82!*P^e*C=N7Eq9gEerY6=|zXiMj8&7Vc&@3KK(IXyW^yzG#B6cLn_ zMg2qr5iTGhVd^736F3#db-up7T)S&%@5L1q6f7C{>}M)y+1R3v3~~N0{jnA49(N)i zjN<2ZR!)wkg~j-<0+d}&kIN@0Jr&H!?YlCtHl@)@9Cn@-;Fdajtv6be)jP(?+OCaf!$*eY4 zgipk5q<>wWb+_mhW@VgnQv{C^|yUuKB>)qD={vz{!IoTFk9VM`eE83nl1C*DY4IN9dajrM?oK#BY z3KjW`f`m@=Xz0_;P5frfuRxS;ZgxcXsdogMs)P}6t>*n|%@i#n^@)nIj{;YYF>JL#)aG?8LREVbI&dC7rT@ZFQ(q;}$#mK;Cs!*eHBRX2f0c;|ymv||JnWPDnII$; zK})mK(AuSIBCRv?2FPQIQb9xqJkx8vht1j&dM? z^J$MuV>5+=cli+Z;NAl^iGkN&W-w0?6}<+MxvRbxW03Id00Kja_ldHeg~cBrSZ;8J zzSPjbUE5nENne+p9`EG zy#oEs=0z1Xk@f3Oerp&lWicJu)Rei9t;t^h?3>#AUtkuAU6)6b;;zS!-syS=o@{!} z-!r?MTBfZt3Uvswnk9}->=iK-GQDg(3A2K=JdYS%9`!FEd3-9Rpwi%^uCCrcFhIQh z9noO|)VmnWb~jho2bQx~N@wd4KBxJ2bP5WJ*Mv>l&*G76MF45sko%~@3DD@q#>TNK zUkOg7%zIK=o}wVMw6qjz7n{A>1C$7GYIz?lJtnQ%B&gwqJ;w>%%*PzFN zUbV4-Gl#@e0s4_2D9-LcssV!l|8H*nglt*4kl2!wzNVx~x}NxF*gp#!>I6>R`WJw@ zZvAsuE>)K6QSMdhe(X}}wIO4}HMb-IOl+r^sBC=yL-7zZTBf!c4W58-*v(4WTR*#e zFOH9B7wcbMV%52NDAmwRz4(*yXoS${;B4Q2rXhlnqsA7=@TEtfWuzo0)6uCf!dLw> z5P5MQ^&})DEIDA(5u{!K$XRN+**jcOv9Rp+L{nGU8~b~f*VND*VjvqM07Fggbv;4? zm0{G(%#15@Y3gS!@*6O&AQm|#{f(@NJzfN1H9z|=8+i;4W9xVS3{pHiE+DTW?Euz^jZZUHCx`Z-aRTFd+r!fd zz|+=t(&)C0k$_OLX=ca#|X#cEc@)Z7#zsS#e5PqgmUuyyn5xUq&TGy7CRkA`kqa1VlCQzSi zIZrg=f*VM;F;dh|L+Tvk9X^zByejOe^LOS0-J`O_U*`tMf=N#VV%SAcQJy@x1a^-k zxptt7FP57l*9ZIGORA}jfa!%OkEit8H!im=OL=xoP67U=Q-RA3Tw1c@A@NQVPWtk#GDze(ViIL^E#j zku$Aw^zjkm#MA=j=ewI@m8<2a4}C)9RO;o+M}|*;ml3__(XW&N=cT#0mY$)Z4@`P` zdPyz(5zn}{0OvA*s#f@JgHgn8mg}mY+TUXtkX5UfX|fbPrf~{K>Sj48>?(I^JN%hIki%)zf;2_0#BJshNOu4h!3zYFiCcH8=Rv z_d=oyrrhA!6-nD(Q!>!YfkjhUnT|p?F(JVNkRO&p22xVpEwF2S0|Sk^S>Qesy*Yq~ zw`-(VOne2neGNxVmaVYwQ7W+L_;@5idU!Pf5n)7nf{G3!6^b@#p01hA(b6+uAt zwg+0GB5H6$!ykz%>5wuSRHe-+iqudztG6BofX`Y9{TXgR?Wb z-FQEO2jrvzd;IJe7_9ogy#h{hA$Ioi%{#m65yGVg;5s=6vP;X8g8Y0vUEL_O2r7S% zcEDGEkEBq^h(>^2odisL$-1h&p~jo?%t?EUtR)l7GjsZ(ra2H>=G6V*V;7Z`Gu^6A zchjc{o>f^9r_XFl63=@6_!p4~1*EQdOkrehrk7-G0ac@-HwF8K*=Y)GSBylz?re7b zr5;Qtl${xoDKdIcjsV}MSHE){rJll_RlufCl~nN#JiYEUNw@@_$f##AZb}T8c%ZLp z?TyAr`gjlw2|WYu0L6o@t561Lv${Gy`S|3a^>?Lnpr0Z2PmYcA``ucj)X{7Nn!I6F_EV+vlZVEEjourWpT98MsuivCD17=&r(P zRJFB{@Fg0iUfa@_3?Dk&&jc4?#$M!&GNX6DV{HZ`MNDO`bQi`?U&k)kqxuEXhEdbC zU#B*I_BG=yU!!PU1rjyx;`JMO*X5EziU+NGM+#q~f@haau{hOY9D zGK^$?`N&bvqBMu>^!#W2^juE&dyy@)%SplmWuTku)x05wFMNDuHc_DUhQ~nqT_h?0 zXs!wc5;_uaRv+G6y3EYuc<%rB@ne6c^1~i#6d)$RQ)=LW!pv|azh{4QK#j>R>*vb_ zRyM#N)-|U1R?LDUin@qF#OB{J1DGKUS21hryzrbO&!*e!b52!^2W({D5B%ry+I4Q= zVen-dJ>9yqNvdj-VMqTTXe)Am9B3yt|ERwMk2{nivvB-z9k=sM5N^gIMgac)oP?`U zJSwT;mg)7pS3TO5hyRPFua4?!ZQ7O;0YQ+E4h19x1u5z74rv6XQ@T@18tIk>2}x-X zK|-XFPC>f6zu`RZ_vcwl&vO6vz2}~}YEp=jPK0$Xh@`Nxx>^3N&t2OeYD%jT3BDu< z5<}ALlzyHKFw#9;t1fk#Z&ai8Fg0?I5dww;B+c75?1K6*I{)nMmJ(0-z5uETtdgV{ zN1rllb8`mwtSZH)9?laR}{IYd0POjkHp@*nFlu-k%lkWs(?{l zxbBc>@8|%zw=Z(0jT5*dTVQN3pz21uJfE(8|F%6C$Kga4Tx`AhMX3wQ(m5cEr33m7$Jp-s^Addl=PS_ho(sK)P4kXZUO4?(=`i>^3%%obpm7 zj9%BiDi|d^7nNq0Syt;URfwI9`7_pIC&lkH=}vlSgg!Ndb6*1Mm9sZaQ}$PekRr9B z=-9042K)-d5tS*d&>~RPENMc;|Lf$mJ>A`Eu9(Y0O-=2zHJMvl+UN7z7B%6SHY%>H zc#13UEZ`(WlFfHGe*)UMJ$_5FM{9fH5E0dx4pO5Yp282vMJhkML0J9Jb8mlHMuSL; zli8#F&!xLqmB z9l&`|)!_zJ($sW4+c(mhn*Xx`yo1-tc3K!mLTv0MGy#~%s zA3AJLyLLv9Rpo_rKcg(8o5;dZ2&@z~9X zzz{I~H@z%$8kl4Fy68{PQ!#g*B@>YvEqAp4y>uP@;EqLTR9<`~R$asJ`LPg2&X=6@ zfPQz}=uj5y9*O^ef_+`F; zglb7&-_*=(kj(arU}3Y*w`1UqOz)eOiW~=LLth5!zZZ(9I6BiH`*?2K1j4ak-i^>( z)3R@H!orUr3U0e6;I0k{tOhPlPtR+)Ck~VZgRj`R>R>d=G%8>v>H7sJ(ZYMVFB9m> zLT>9L^qS`DHRj6FjoSew?283TiyZztMSn9P0KoRmKRLT&kP*(PM5J(=ZPScS`${5l zZ*xySoA^kVQ+tFz&u2KV6MmeHX% zz@SUb;dFT1x`%0{{mPFQ^46QMU#;6hZwdKV%>syDVU0)fdbjB9<~SjE)Vg6O|@o66%H;fo%R zw~j#`*=bw}#YgDdZ}cJat@+gHddJYJYAPx!;6aTu4?4ikXlC{0HHfYPA=>6D^c)Qt zk%grUMEOIU^fWZNAY8Bj@053?YYRc=5?OnHT&71ePLyf=pI3(o zMX!Ik&J{-7%afN10=i}>ej8D7`OAi4)bD{GmOR zrRV==Bz4zx6~s<5|lc)(mAqLL5F@nLDt07LT}2O<^~ zYUak4KaNp&J6VNzCFw7@CC6}%#8jZxMBPf@=9!hhfb4>ys;cTymln+_JW54H*Gsx* z%*?2S-@s+)ppBMr=!Ubr+-TeGTt5zT0(42*c`72ASxkeM@2C1KtO^37)WBOub6<|@H27UaLk>r=0*D_YChc zXwNF%kIGk?q6x^6oDlgioO+bBncyl$X>QJ$svz>MIn!fRiqG4u?Ltz$?K<_Nr1T#l z9mc*=GUjVKF!Y2-CJCV_SSU9%+$P|X-jV_z&sZQMbs`B;M#S0eZh@?HQxRs6y- zf1ezrcT+YkFH~}!fRCY5NVjyex_On(@A(QKh5E<$!HZM{n%yo*b8^EBlp`^bC3XGw z&dytvrf(X5?+&exgo=|Bhiv#CqVLaIprOnwbF%lP*U;27GdCZ4h2aOx&ky&cm}{RB zoaV2%I`1*Q&BiX_ZAdvk=uLe&qg-6e{@cr7RC;O)#b~DdkW{?%DuX=KbUV+D#lebS zEM&GsUjM;89FUrEhuqESuqmvo5jy9Kn06Y-4SMGsQvW3Srr%CNsx>aI6U}MSclzTl zu>pQqW6>iJG~Wtj4V@_u15%e_vYQ1xg#=5Q3*rJLzmr7y{xt9j4Ezb$DQF7@nJ^Z@ zUE$6I)9rwj9*8*0bn1i+?q4c^gPO|j3ybZIu&<^kh_OIZS82N-tYz{NZeKX#YU(j& zd4KRksiXbBjpHPVX2$`nivi7AF8 z`ITU(wbo`ueW1Gp4TKR*V*Gyc$&h8NdJfBB4tt@XLE6FDPrlLmOa}SmwJ~WG$E<_7 zS!`{af0roHwn9Nc3b9LkYkc3i&upP6dm{CmapTmnK(Z45h4n`q3H*POYBOxN6wC-A za_k1iJkSl8E<4D#_e_Y{?ARkkL6G4YXU1HD3~t<#;b4vbRutA%(hSyvj9bdG>F)v& zqFc+FnY5LfeygsHq2cZ1p=DA9lD*(yH1Pb)pFj`#v$c<~7{BppH%)Ez{aSV!1MWzm zSE$o8u5RjTl~;y)3#Wdq-o)-ns$WVz+q<5+x$rzD>++;a$U)^|AUz?)N%c(7k=Z-1 zBJf18s2}zi9I*k^Uev7^^!Q3mIYH#2`Cwo0_`lr0{;%>iatCe3uj}f32Ch z@q7WL$*=>x{=UlV*Pv5ouZ_lI-u$nDsv9K~5)y(zn;nVqx!2_>38ytsR_udcYl>I^ zoLyzM8iQ*HE@YLd<18u|Pr*+Hc@>$C%*cSUW@SH6XUwX|o0r=pXIm+)SLL#7$7)-1 z{_9_{V~g@my2B2I#=5(+gs;=5U$(R+F5cgdN_!b>Ij7V$VNFHa0nGz(;siurW6SPd z;iaG85h%vfkHPqLEXn6M>->>lJC4@mT7)5J9~kq&GaG>!a-0!EFC$XZ2dBTlS-)P1 z=V&&iaJC5%Z|8nKShwJZv4O#hTbMZfE?|J}aJ=(^ydk$|7G!>4Rd68r=|Q6NzqWZ{ zV}l1Ad8q1dK!*c<^G=I(`)0XyjKu5Pybsw#!8x7SxVU`TB=~y|>sP38LmHc!Zppkk zo%L1Frtm%^3f+xW027K$5YyQED2M2%nWLD~`b~MD2lL|2Vpt_o^@%Q6&s%6y`*OZL zpQiwN(gXi$4!2ZRFY0mw@n2~33UZf^T$BdY7B=(ntN#%-hA?0OJLguoAoC?pi+ zy_U58bdH#Q4ADPTs1lCb(`S(a*JxN+dG5^BLp-Lx;-4M<&W(^ydj**%kC~Xle5Jad z%}=7d(-ru~?~>t0 z0Mg@-d(G{#CNUiZgHoC%B>fPSaZGvf!h4SjC{EqA~!28B?_oG5mFPia-tF^eRaN0_TN`_vYE~MTPi*F86mt}^z@T72W@_a= z2zsnq1Q0Q=rDNPNIe@Vm2^BN*0s%y!yl@plF=2g9?@HUkaBP(t5uiTW7huM zH1?G3cy$@SjG98dO-ESI|GolRPbFpL3X3sjW8y8xtx5bpMInK{WgUV#(vXe-y4Ym$ zSYfbo-9A0y2&WEtFYmT-fBOAdM^FP%N26KauNR<)*a%KvU*BF;`+)3LZkWuTAsMxf zj*QG}8CBfPBBjPV82)uxhVpF0Q8o7Tpn}`%L%vDx+BvVe`{0c}R>4}VYW~E}b*);g zNr{>((!p=rF*hf0C3jCyW}N#4YqC17doa3)fy|1PoV_E<;5(p8db=ssZEK>#eF4o? zU2)_xIP$lyZ@Zb<4VA}5eGJ%GJ@psQFJIe_>1kobY?7d|0=;~D>ZSB;>!!h%&-!n> zjRazJ4lC7t*AWzuQoXgpeG~Y(93hOypw{75)&t1+?%v)+jTH$}hnX+11b*-CP&9

gmLER z{tn&S8JySdAs=QAQLIq;x=*?t?dGQw8!O4w#em_4dzKQSLTGo^(eCW^He&MkJPjCg zp6CdHJfOYETt>J>oImp1S;se9QdWqD^K&h2B~x&k%Y5z1Q>&npH1xvi)Kvzo_!RDw zM*IruCoBAQ{5v0w`y43w3 z*{w99dQ44yygf6OD?_LiehcwrLl2|Ukn_KeMn%8T>xvlfDO=wHD8)lVQx;mQ_&nc( z5f$c0uqX_2MngoGUV~E!eFoRlB~bZ;ST@&2+428`i_oFV_Q~H$oW6#RAqAzz*j0-< zWu_G=GtQan)*R#p0&^a|0&FK9r5npzi;MQGWG&cU6`aj|0ua0t>d3MIuQ23 zF$S=Z`cu*j1cmwo?`bldk$Ub z<4Q!wmKD(b_UABkZX~}{0fm+^nXcDi2Y=x=%k9YvrS`HHXEn?3rb0bLaX2T~JJ!Js z&9<_Av1w*zCnU*xIy`Cdl5S5>U93P+cE4j15At(x(Yb9GBiyg%__O3t$?_p8bsdBL za1Aswx8NXHl?M1wP|TWQ9BFd z3xGwon(e@eHXHbTLB! ze+bN=fQj5Tb+twmf>YpTXl`CIPx*B#Bn95sM+_;_8GvTYUNQ(^35||csyrvGqVZJH z(eYRt%2mo1ALK-VaR*u;ey_{KJbDkrsBmZ%{^XH69&XQEfKFV_hHWIUaDIimorKJ3 zgYoQ`vGR4^MLz2~CvnGaCa7kuSatIE zk&s%RBUaMXb$heD7HSZR<$8aVw&KCZmHbsgE1?!sj1n&pEVRI!bUXCTpx+Ut<>kFp z+t64zQB1l8sq>m?-oMPG2q7DP;_S_rNAI!$gz zYAQJ{yy_Ve$ry|%D2AulWSFooCTHjH_XJ%)T}hMg&VgygISiU zkWDJ$XB+BnHxGFa-Y;&u^!-Twsa(!e@{Ij5MkAt0q%9&F;cr)}gF~_B?9R2vr!Swr zvo*2(4t3p&Rn@~~*s!BUx8`aW`5_P(Xd#&zhJA&2<@Z6o?z*o|gCe)in-8 zEj|_>lah#L7{OoYD9mBwnmv z;)I={)bYjAF7VeE>vbajob0&7jd#w?icglY%KloKNJtoEzmMr2LH?!Fi&HCIv#M{6 zbyt;}@D?Lk!&dtE{hIZ6W9qbNVaTzt6$f zj)a87Q!EY?PP-PUpfVegY@( zD)DJ!PwZSzL!;V@jMIY-;jmBxj2P+EQ^dU%Jh8^w{O)z}S&dkSjOEgJ;ynMPVO$^4UOq3*UD;)df_OR2%!|tMv#-Ql zQ+pN7D`G?~1RE^>CDks%bRya_wFTAD46KiEj_9@!uyHTwQO{A%QSQxnAt)6zG+f*o zU98b2;-LqvnR|AZbOml5Ov|NOFm;29$2mDdnCkW&d#XPa- zXgrk&x#8PnOZC8|>Yi~ytD}0(|4*Qg%D_bRv#x=otcAImdVMtQsF%<`Po#XGJNVbb z?Iz>h^JdO(z3+F`^!-D{5R2VH)YZ&L)7VEQc5HVZ=jZ&$X?iur)b-M-)GjdHpCyRJ zXhpZ^UuYf9i?=?_-=p~!FFx1QV*eXp8DNoIJq|TC%s7U*U~uLEw_#fyC=|uTf6B0Z zc=+()LNEl2CjLTTn1=IQmc-?UCmOf*@jHTdZ7o_P?2 zRs*(#nmjP$!v^YaE63_yxveOI2SOe1N#z4&Ei^@_-iJvWHw>Sw+H+dl6L2a%!F8Xz zgYz^{rl_Q&95eUs+ApG>R=zKf?1>&uEs> zLikK~h1O=B=hC3fn~l-qOb*jV_` zf65tg=912eq7A+-1#PPFiStfG3tU-pkOmm)r6SjKAW@*Xx2xGrOzQh6+$u+a0&FAJzT-9TGiK%>VEz&#RdY4O}BoI`FjnL zYwm`E<{knp*`jhXpY6T}VIS_}{ADm*FmFjEEc=?`?46eCpA~84^5maGKB=Dwg;z-27%aF{6ggQf1J{Y%js@^p z|7tDm%NWSq+)H{t;vH6gGLJ;wRE73;e0e-Zoy1SpF7Kgcm-mkyA)2luv~%*NFi%v9 z0}6MBrO;mnbF(2ct4Z&lL>&02q&zXtvrAzO-jXqodCQGWPP0|9OHm^AQfTvN$A;Hq z>G|^`wI-Y<+TwXFtG`ouq6q}}juT2^-)zKhR-jJDw}%}6_gy9Kg@SYVnRd+~2vxTK zfgOAU94IzCQib*PlLG@$+x-D9ypm^e5!#9B1wy~z z$JA&Mz|)`)W~b>U_t$cA;lAh23XqS<#KdI#aYP31U@-*KxrbPlw}UdCsw?Fz;;`rW z3P&UN=!}KWBbMBeE&1M`KW6EYh7tUJnTTob?ZD3Q9b~5+LH;(pvg>H-twhd+{TM z@Z}t3fi>PArP?mU2WV%|O%W~gE5saYp~~OMrgBEg`&%Ax9P0eXamST+_hxzsi$GNj zU6i$Hj##xUm~bW9+JoA(h`F4zZ0X`z_J4=#FJVVV2c3NC8_Fx>-5KY(YUb~t2DRi! z@vWMHk{rl&oa?HId3&np&rvW@gGZfrX5~)WGO&`K8?=Ev>hqVm>dCYj7Qt2EbU-TH zDtOgEhmMX8I1x4`X1nMkE1VEySG5J~hfcz#B~;dd4K-q)5=Y)GaQ=M!L&L74s|@`_ zQos`bG-Qdf*)iC0b)FD=@ur~Qm*2?>hRgIhue_YHo|=YRowIU?yO7OC0i#GhanLEfHSGPyL**zt!>h$|IsCU&LwM0ZEvsxLI! zM*N&OCh_y>zmB`h>2-M5;msx#T$zS+??hoRfohz%he654X8+%8CrrWTR+W^b>E$K! zXAmq$V3n)4p@dQ6@@P!oaGEiX0In#>4i<=o-kE9m2C;hJlt!H#rvf4@@Ke#FxSbKN z!GQr=U@t*|$EtX?jk!AYGn}M@j?Qni|L^raI>5jRdi(C( zOL!Ue+Q@_aIu~7XlLk6M8>`pt(1CpaPMwh`i388~e3S4wDlRTA6aoI5OM7}+{YNiN zA8`4(Xwx|BwLCx%C@(L+#YQv#b)|CK^YZfM@H|%KD6}aGW2oTiyp~QF6$$-V$h9O& zZ!f?e>G0ONN%UcEa>#B)BQsn2!2rK?LwSEDL5s`DnsUV&p=0=R2u%e1Yty|rezlt< zMl|r&^(E)QevY?hXno(uriU=?YQHPqHFN$!^0YX@PL9tup0BgZ{ka2{ntc|u7%(rk zCF(;GE}z$>%XdE%hi{TfN;rnE?HK9c4ge`*y})~rJPi06c224K!q-iEMg92k51^rE z>K#EBAp397q8|cGtmm44uY^{XemyuB0|g}v@`*&RyMTrT)AnxZ#6rV8_V8RBl$w=s z{q@=wiEdGPy9J+sISD-?i6`D4g(!QLD?BSBbVDz{x37rYH#EurHupn4?^2l3oT(rp zjH$|AZnW~=&ChZ72T4cTZE{h1`d#sOo_^*e2>qUi!Zf|D?0)27k(52vuZ$QI+DJt@N8tR*441OvU6clCstzdXQ z)XS{mVy5TMcLA(}(IJ`Jo_5AeeKznmZG@iwUh{jym-+CFZf>u-$dFP-sO;JUkX~{; zdGaCV?by*gr&{F!bB2+3VW(SUjP~l}_L69!5`U6J-Tm07;=ddUd(pi+-<5a^=cSxt z4j-%*E}!#$>evtyA-K$v($Jq#M0;V+)Ol^NDE}oh#I2b8LeJ9s$pYf>SnMCifxvVm z1!M)ITq1(KcYN7=k^NGPCl2zXG! zCL-VSO>xeeBhSOVVyZZEd(%2u;-#@0X|jw#w2O(E$hJB0li$Buzqs~uf6MLnsAVwD zQmWs=@4Ej^OJxQD0MnWBw?UsjFC^%DNo^P^tTv%w5Wq%>7sKwrcxCa;69Pq!=foP> z2{EH?z%v15$q8`^AX|i>)`>ck(#*`~K)Omql84N+!UID#C=hVHn?B)DKHY6`s$ryz zGeN{&QZBw*6Y(L+d{25a<>EctKe*brM5d`6uh>mI;8Rvpv-V^b-SnD>cY3h_uZr!E zSnyw)f8^8{GilsM@XKcHPln~-(%o2Y+`Z?mgRX77oiCktko{z(Yvb27q2@uvX+$G_ z6aSMF9UPYU@|sXz!I4CLe5|9!hYf>m-k1D_EUf3p|HN7-Z&9HiUO=ohSkJtwr!ctf zV7`evD4Sf61yf|p0KhXO0>wec#-+Et;{^jxcl^IRGlA^X+Mt`AHZz*O3 zL*w6gw*lHqi-#H5S~a6tH~b!7hNCsv!W!>p81%oFVZT%PP%#-fN-O!p5luDypqMl1 z;tdzUbNQ(>zc9L0LYsTHGsAHxIBg$GL_kVZ#vgHd1_mq=jxgU5B&c9xW%@zEvH$l~ ztU)lO>Jf~Z5U%vtmhC81w;{g42wb?EOe`!g;-4Hp zUC;4CKt!`L$?TKVo+ot&`Bf_Qk=rSvp2qj;KOxF75^hT>m`MFz2|rpQo;FTUX4-8C zf0i5=P}Ln(O#E@*U`5v}_H?dtlRKzU&0*So^($T%y6J<^?>n7V^o3GcyV+8rzl96@ z1xGZL5|Ub7H=~U$2`jH1x(M@{A7eJn3d|3FlC+k;;HB6iJgZ&ECzlyywY|)4*QGbw zh8vAK+Gk%~#jikfWI#1e=bw4T*Wavf8k;4&m6=U-*CV_J;byLZn+sy3LGp$v`&Lm= zG1HxZV7G%%UsPON+S7;=en$5A&}bbs1vQ{e8ZrrDDOPa(#CoU8y!`tOZ-bK!;rV43vxG1jDVbe zd+WvBnQpPjrur!{TFlzGqnEcpy@c|_ns0t^iq&@UsEeYJ?HRi3Vn0_dNwlPfUP`p` z`oAOtCn|>pgvj(~THm`~VL_dZ+FW?3;nt}U>48LN8{tL}(-TknMppLb$JwziyXzaj2sZx@1NrZ7;#hm09-~`xTW3k;Z&v|TFMQfmq z<&`}B!S}oM@sD=4AMM%M!q?<_2hSH23Z7q%&Ix@AZ@Nq3)?~BFKA)K%(=5TF8}D}< zN~CzPD%*4=z$aBaPy4!Jqued|)Z@;md%SmH6-OLPLG*8rX+)*Bx^Kmd12R^Rm~l>}5D@Kv7Al9A+M%t`)a21^I~F1(^Uhom=+txkHph%kX!ET|F3=<&(Z)oOa{jF^p zZjenKwLzBpXMmSMfXzKj%vcXRN9Nnl`4(~FdQUy028pZ{AYDjHOY=0D2NoC5woG@D zaCKBHQnWeK+F%*}%lf-nsSL;d9|W*2!k+vvlcv2f#r@8Wyo+^@$B-Mlhy;Dj8la_g zf^y6cjb|+xwf6GHtL~nIZZ9@Ml1v_!G2|o{@_ceSPFI^>N*TP_>*h`VaUX}{*DCsU zAg!L4nv3e7)#@dNK`6k%$6Vh?oizv04>7n8LvP^x-l5C_4_883F~ZJAz=$Bk7~~qBhIIB!8>pV#*XBneefVky$^X4qnv6`^+XA0 zsrEI)Ud<1xOz8@(M+AE9;R5{={qa^P2#QF#v{%)YmcI^g2h8BVpxn}bN;?*Z50cJy z^1~-OHM*+4WkSeBx)lq#uM_|?2{67)t!=siNc)?CDEKM=)wb}J_jTjaC;=Ms(b173 zGH|!HxlxyVMh~MURn-$v3Eq$$t{teMs(x@<;-zrm#Aqq60Rxv^zv_t5;K+v>ufdRy z98`qK?dYZ(^Yjtj7t|PAc6FcG!)abR#(wbGf2V%xYMeMOeubRfF`%Lu>at1wxbpKP z;eSwl;VH`e&Q1&Lk2-azCy~4h-w9;5NlRvORu44zc@6Su*4Y`oc|(b892=G;zeFe} zCkJ=1AIsyCIAmAfa&r?C38@oSabsgz9pxZQB4LY_Dnt`INYcKSk*TJt3b8G7Kw{AP zHZ^%dK5nFhk=+(JM=;6)7!{Ev_#?|dE@9o4Ni z9|c7KKb4R7<>9abbS1UWi%3cae;~>3WQaL~`2^>JRtcA{rZi;WWYa#7 zuaj~3WeQ2}U&4n6-UUKGulzOr!IH+^ba;KK00WS?}$eKE_XOb6NV0=ukN%f3Z;V*18=rut}t9LBMur zILXTPcAstJf8Q8NmyH~!UNHv{&NEx_`UH>m(B z6zet|!ym9+&nzsEFqvcJ)jhkDDL&RjhE)S_tXP zQ05pb1#VuToQ)$E{~oZo^Ky<}lP7Q0i;AVb{v!w=K#gY(@beLE=vN?fWkAd23~DE$=!# zrFofERAdg)Fd!+j8e(j51KHG6K5hQn^2CQz3p1x}Q+7kBB>J-Dox5Lk~`(Z ze&~E|Oy*B(Z;mTQt*JApGNuu&ON6Io+rh1NA>DpLfbjM+RYRJVvB--SHJ0;$5BWle zq3SuSgb+B}a&?>nR<0SV9D-Uj@PhWt8;7kD&ICcnaJy)ZjqTGU)yTBeC1TbU>^c(* zB^cBd*-fx%KBJ|f3E1~CMnxN0@aPz4*n!Yde%LYb?6vID%F5LExW`@xp)&u zNRhMo8NVH*xT7@ot8zH%0`;l!LbCf|1pPxGBoWPdwvZ6i7#n#G`LgMV#tX?s89M7) zGnCob6c!}Eer;5+GF(EPo{gBXefg%CS;n`m<9e0wmQr5Z_qw%#fN==*xHLZ=Hg?t5 z)=H*#h1o$dMgo%@*u7yM4kY!Iyq77jcuW?by3&nMADksESMpi6kGh;Oy+9Zjg6kn7TA zKTz(o$C-8WH^C}SjcEL-%Szd2)ypjZaNC)TGL099n>J*J-B-0Gn~gFPaY9DbYrOmt z;=3h2qKOoHZe1p!8WpW2C1igJT6vRm2+n^&|N1K)Y#*(kas*wH)nTC`b7-#_~KkX9Dfd~mu@b6+u0Qu=tL4#JX9{90ZJ3jU4( zC#98Xe!rxs2%Rp9`4a6fN>*dh^ZeK=I=_6bfHcV>H?mqxpG zs1zd^8b7J6>G#SMm_IGBS^OwUMjS=zpxXe4x{LDt)n&FH5qj=x^n2`RvK;LIz zraSRF%>GWSR@8P%2UoeE)mx$;K`YOEXTMlKWmxPmRzD3`)_0UV{R`y9p$4DUr$iau zy8|pwk*~6z4IN?B4v=!&A(s+5aDKvV_5atG<_@9`A`h3rFt5Q}BDh5>jDAt77k~MZ z?KIx_Ve>-F)s;K-4mVcFqF==CHZ+!ohE_k8($e@P+d$f(PyM8oTG4?pagLM^K?t*`(7 zz!v!6OGr%I)5@n!bb^5<+SJ&n4%hC{@j08%8|5#MkVi!`pr6 zXZ0|His0FoN`WT4=0oGddVtlw4uy;CGm8_>NHIJ9F2!UnlqyZ)k29Z$I8reZi_b2?Dyhx~O@tBqJdE z3HfJ5ym}e@=FHuoGy)IlTSLPFvS($&$5)FG2n*#n=ORKB>^Dw3vt%SBGWxICkTUv% z2FQdRv>G;YOQ!%s`a;kF49~62C!r>sgcDq&91l5@wXP7W5HAX)Scobl+_9xlmB4B&T_IbDVc*!;L_&b}q*Degz%zEo!{EG}@y9Kom*I z$Z|)OY@h&yB1qUncLP}nvfKJket99-A-w(wMq$?7T6-``Lr<8?;rzL(tGnBPb&&kl zT+Wd)vp^4EgDwmSFqK0~rTqFeT|RfTmr!3MF}pdQOD6 z0EDIfl|lagA}hMiEaWit-_Llc<}k&vE&*ubp+?hjgFwa)1&PhcSJ5G(f)c*mJ_;U*Bp|TqS}12eOZCtzJ|nB*BIq>bU>lL z<MPHwP7J7a5RSY=Uk5yLNr%afI^L2R_ul99`+nxnGfB5AGgDOnt0~7iZ|}PB zTqRwlvD?-p9Je93TD--cLPxF;ZJnH85!=Twg=pgEZ|)$wgQ8e@v*5P5WcPW9hgkzI zAPsG8lXvex)nVS*y%BBH;`17y@DHh4B8;}o=lj1{AZ_KHeeP%jA{C#`eqvo?OaXKg z5OpLV(Bf$Xm?wDSUN~)Pz_kG(Hmo$9rvYVUJFt{>DR>WJPLK&{xecY)VJ|{@Vxr_W z0ZKfA4LCS z)VH#}X`?DbSo*26-q-&vDbB4R((Cp{z?tb}f#rK+#hSz56{sg+yk{!}iA zdBS*ZIsO%Ln6x!CT3;(OFlrc0+>(qCag-UlS~?E!xAXS2R4M&7qTu0e@Y!?PpW6!{ z8ZiOd66E(=jE^6n!RZ7q%AM`Z&!0h+3bBl5C?gLdSaeP)ANhDgHo`vSnUuueMY>Zu zW1D-Y1y+pgqkn=tNhkW}yvUK}b=+Z)@qN;|9Uae5S2jVXEZl4F7JCyTxJ2}GLMwRZL0EWe zmC#@UQ+YJot?E?^Rrlh1aK)xHNW3C_5g_y1FF7R(%JX2Myp63qQ+EYbE z3|bAzFDUsHWMw}vGR<85Az){(hWr^PjSRRMSglU51ug}pbvDj1Y|&4t9}yF`sYyk2 zZFmBN`v)f`Hns{ZOx%VY&Hw^`)AAmq5>`@?vXXw&N1qaGBj+ix>i#r3<@Zq29{n2O z|H$Q>nY%CHm*9A*=C7_gVq&C@jJ%TW8C$y{JWl;~$M@Ct&RPYY!(=aY_H&^2SXK$4 zm9ChFmT}g#20R<)`H6kOc0u5L3?OT6O&*H(^ zP}--*m0k#Z_8`B8rs}z3Ii!`np!zI>A`h4bxRhnLmA_mMKg#Lu| zx3`?+dsm8~*Q5hhmy5saCV3ub87skwcB>n!oIU`QKsWqyTeS;xFn_V!vH!enlGyIS zNE<-0v&z<2yyeG1e3Tr5AN-(oct)cf+?@Tloog}~p%DVS(>6%|hLqW_o)=D_)d8PN z#l+^}p#u?-F6#y%@Wi&^Zz#7=QgPW|zI=BkmFvRA#T7=v8L>TUfl=U_o&EL#MHGMo z&;vrG7d$#JR(-v_AN0>JEQMvzEq82G+t6`r`Q_=e)D* z96q=rZtkU9FO_7ioqOEXl^ZK@3l%l9!4_}t z-~ezd!?G4GXi9dPFRY=a&l>>*5A2lB8|*4!)P#d4w6xok)uBl?i@`XLp|&Zhs2p(M zH-P&fNjuyc4w(J+Gg$}LI_0lH=klB6SzuiYcp;rv~$^HEbn+FOBL7gk2gfotk zf2eU!>Gjz*B}f}$pb8)3W{ohXv82=T9Jy%TU3KnT4grZnX!O7x1oOdWArxrAl6`Afz8ZU~9wB5H9n?((rv3h39e}E_-gL ze#t%OsnZNjzc^INNnki1eS2iuAS>9WSf_FkP@viVm(WE0Jm}w4ZO-~^upSb%6R&Md zT}EQ+FG$=cRdNp9}ZpaezJ z$*eo<>g0!6)1&gyR&AfGr;3`Bk)p;md;j5MN@A1oO6&_vDwB%>nW&Qz`!ker(HAVt zm2E6}r(S}`_rAUK2MJl3*p)5Al0BSI(8aEX>qhh^sAeiU@U{3H+)GuPFV>T4%?$_S z=ky*EsCH37KjT+XwHuF~>yHzXx5@y9;s0l|)QsP0e z%89KnKd{nR6UW+{VhQrL!si+sd3`^1F)E=)NP&hg-f$}SX}?Ow-(g@wkcA!~3)K_o zFmp06^h{2=fsp470gQPcSst+&_d>%A^B(BsVV=&_9WF%@{Tp{w=c;C~m)hCC&;0+m zd&eN#f~H-tZQHhOyHDG;Z5yX;^R#W-wr$(?^n1U1zqk`IGe0L{W=BP>+!3qRkIbEG zS3Q~eWbHiMR^)~!qxIZfFQk|*6$1V>*)g<5ClN|&wVv`4dhf}xFM- zqB9=Zbth&MT82H-D>L-ZoY?k_Fs~LFALrKwLe@vHZ@_Z)baKBzIb9cb={w$?%7d7(j26RX9Vo4LxzUpmV6xvZSo$jrl4aJFj9ChkbzNRZgp{89BjA1`suU3dswLoRLE}7M1mkLZo+3Nf*dm1|BNoVPQmGz%NNoa{C#4?J0d!9i> z*i2tDB~9l7JI#9$Z~MsMkZQA7&OBqP1s27A@4M4mP*dHJ@EgJFYNhqqoD~zaNv!Q| z`VH;ml}fHM8I<{0SUQBV(T4!}Z4>MYmf7v6ESda9NrJ>l9B_p1r$3Bwog|k!uv^C1 z&PN_D-+qCW^jB~FL*CQ475|`rKCnFRIJfTb`5N-!KEbMud}e(rsHf<^YWy8xm4pba z_pezI?KE)ogrV38s=w#7dzN1}R{vh|zg)Ps2x-Y*(T&vJORL;VcNpXz*rXnqWF8>l zc+nObOFUFe9Yh*o8-ny5cETIjc{j%Uo9bbf=3%D!YOKjYdeedXFi~}y^SiC>L2J7= zJnetTdzj|Sd~lmJp74*I3==)gMM7vQ=bpH7wWLJoC~vvO|J}sQ*D+}PFFnS&2~k*< z+fsC*w}2Y@@~YbX<_#bF{|O> z3FUbmiVHUd)bBaiMz8YA&~<^YpIB4H@aM-CFEYG=TkU^a$vWj9MY4efm=<_U4b>)+ z-z6x)i#IIvTW|H>4<}WDCz6CkKLc-6H!jcB`;XFphoQf3K+!x0>c2`K~vzM z-~a0s06_$HimuK@^WR3o$ko$7tNs7pVvZ8?ijL7Mhm?(z5YUMpP_ZF8^0p2Kt^YP| z%Lu88T~0-o>BIxH2aUQWbQIK6$_e_$-2e2VxE6AP6a0!^$r0f18zY#SHc*Xznw!#H@d0bQtd*b|Fw(?sxG@2^X`dd@?z9yV&W`zd%^&pi0;gtV% z^Z)B}oNeC^u=PZSIh2X5v6Hi-iGj_3Bs)V(C?*zW0tSNrNIX0Q^kNp)&L)n(S8D@j z6A=?5J7W_9dTA3|GiP%G77k{nUxrzz|GC~ht4T}CDQh&r_oe1EH9Ck0BtL=*>e%AS zD0XDz+1<3#X*%c%d@;IRV<8uCz5aQ{ZaV^O(PnCRID}YQX_1tgnqJQ|Wq-*PwMLge z@~CY0$Gu3kEP9?sqm3peLAhr6!_1E_a)0vh@#{5`uXMHeu{imuO3m)s@~PhEnfp^q z7H|A=&-fjEWv1F!HC(ri!mUnIXQx^^MYJ~R#<@$w2gz#w}y;a1r)+%WL zkGSEzU9-9K%kM&JvaS|iUpAlFIA$kpE`AD^`DDd$2To@r_$JceWkJ;ZfW!TXQ+D1( zg$+F%DkPG@fz5q|ZR7TamgP*Jf?f#K8$T|)*1XguiF)9ntMWas{KBv_mpig8b zc4di=9y-li%_PO@*rdyq0&#m$qV+j$A9(O~7vcEh9Xe~Tb@T6*4YXXgUV5yeKLQ`- zhZi%0U|hhbw{948_}kW6m{09k&!7%3=Vk}~$sXlO##CWi$H`)%ZLSq8;gS~BFB03Z z@@kc#ImV^wFkdKGHbHvjS9G5b@{3FMc@MjLM2i|omu3n1%(%|tl6`Qzc-_;tw{N>e z3pt_X)Lz#OW!-y8CM&6%44J~r392=S0n=@j9u~ zXRB9Z5`k|l7N{bg3}+b4W2HQ1KA^9F<|lctXX;F`u&pP+UY0$O0c1L9ac+BI^^qk= zdcko-`#1Ee_D2KmXHqt{VGTB32)^&8ARRIgDs+%A5&G8bjTtXr@GKCxj&Qj6vZnye zbyN8Xs2K^?q03E|A4ZJRIU=1677jCQ^JK!iJl`+I6IDnb>duZ-H1pVDC-J}X zZ)OV*`WXK~8cHHVZ43?A@$|G9bU2f_79C{ zJ@Qm@oQdCB<;pBSflK$9z+VCH0r=%0ziR36dE}11GAtH>SkB{>=fyx)IMxG;Yx>nLz(wkCP3yR%Us#{Ii zv{ao-M>$%eGq^p3V8RaZrF{I82ERS{YT8Z*8#>I0C zHk9w9Kd6*$fe>+k4LdlW-ft#yswbRzWD7hxhy~o#5Fet=oMyN-4aR z|1<^RDqIKaXfYEGoHc#jQ;0)s>KtT@GF5;->b_?3gV>W9gbMOpE9LArinpw@Cz6=g z4Oz7es85p(6^V(C^c&m<<_xa1#RVxceAj6wvtdQaDvbWXRbKUoj3PW|Q~>}<01VQF z&K4gHnV`f0qduDZUN?%f`Apcl;NylFPDU!*yC&4 z=k-Q;*QnTx-ukh)JxaV-)>PTsSGRQngV&wWWU4+!=A?K&=_}&sV*JEo*-*l(=qrzh zp=i)5=IoL6C+SW7PE+fdS2Nfw!5%1s6;$r-fG&fB#>+{}@UcRE{anxPpyQ`jJ#UPt zWC=m6m%k$}dqPCqN(eo2d~l0!17#6<1V~ff<6TYJ(C!%`A_`urfG~Km5*&sJb6p>J zGpxSV#RUE>P7?QcQufW6U(n@4l2G58yCYOgV0{A*3j-Z0w)pA@;&b1DBPHRLJs6HM zLC$JsVHpLQhL?M$(9&YSFhD3-08?l9Hivp?fCH)~^^O;S5_)qBVX3@&9B3$IiKa4( zl^pi#?0YTZ4IdMeib6_8vLuqdI#NO@$9F7p=a0y&gWl(gU%8X%l=~Y17CuH6lc8MT ziQRV3z8=@|F%+`*=3a^-nk12I#gll8_QoLpyuQX)Vg4@^XmMDA$sska0)hWz(N=@Au5; z1qt=Q0skEivbdxMuab*S)Du$+(a8cTB9F9~qrNF1pK~rAX$92P$OUrJB>*!phmuF! z0uf^jF%pF*MW?6{KAq2#Qq|ldy!JYZ{$PQ0M<9m7%!eSg>7cb`D@rk{7gnhQMc}== ztI&U8>0{eLb(BSh@2<3kdya^WO?*6^)f+QgzUq+c0_@z4XuMy)8+%{rKSVa}w07$$ zQi654MFScH%eNE)3C+;^WiY9#e~$(VWqkr(SC8`6X~7``_mqBhj&oUnc=;{)h}7ez zb@mEgdS6(cPmuHvcHXl3Igu5|nca5#D^kBGRu3P7u4zln(W-~v?B<6Ac3klEUl}Ok zF%2SB(6_#UtRi7aI!F)RGlKpff6&qcNhTI+I`%PdO_zV@ShX&A8{)?96#rF8)Lp13|noZ;C zF*ZcXHF6W&!lAi4l%7wm<|V_`s_Wg$jmrw#yGEs@N@<}}=41|j{Xr2uHz1hbkWUbC zoEg3d687nB5z^F?Ao=>{3<+K7vAO4{N~%Ed#cuHYMk+^QHYw0=tv^KIRSeB>DR%%dJp!$(HxMOU#@M8a@1 z=|l<9S9m{tvY;XWip$xeI+Yg6d)(__<_1$Kmxh4kNE#(LXiAd$2f<$tp0^j1MU$>C z=^ych(AiRg-{X+3FDybiajhicDoX+{0|01y!lex}rtbDX=CbkiQO&&>T1|nh!MOMr z2kMs1`?V_gQTsoLn3LgZ0KygT0eWNg{m6j?#xPpi#A5h<*5waLX(WG)4B$yE0OX_p zGET|%M@hP{Q`yaQwK5HoDi+>Cj87q+ipB<`0Wn*t?yBT3LSTPSX$W1Rh-Q$u$SR1M zY77p#itXG2Cyx2s#eD$v;F5wd8fY5jn_H8ds^dJlBN|X7i(Al{-j;U8B)upGde;%o zpQ~Wa;Y;49nU205;$-8TAYl0_9%|-1xKp{tOC6&5RR*fi4w4)zrn-AW-j&3M&yCB)Hr z#WhYO29SqGfD22)Kp^39X2l>1{$=P)JP#lVJB}Q1R6+)O}0Y2sLRgagVo zlWSuvtfrWT9D9UWdGF|mqE&L2I37z7{XhoEA&AnP-TBr5TT&G?O>i&KpkXjKI|a_L z4q_kc<~9j*t)NVyM(ZhQl_1#yc+%7k4ydW>zs!Q|5rJX0yO$`cT$Zn45rQf&qAtO56~z#{T?euszJ_#G0iKVI-JuXi~9c@@gqU~7W5#446({s@=a zd<|W1gjI1Lg>P-NU<#CoqmPf$0s>5vpGw#BPi@9k!15rf>2##%F_|o)F4>u%ZCX8* zQ-C)2~11>CYJMEpOn-1gzu4j&To|L4z^OcwJt42@JJquHv zb|_~t57WrdWt2W9*zBU?_^1ii1qBqtsygo^M86E&leS09jG76UwYP7vqH{mD_E)80 z$DclZGH>J+Q4d=+1ZWEC^v{y}hi^8SeS%&`Q`w-+Wy;q8MW&8Fu0SNO+{xsHp)Fyc zgFJWV(vqYuo% zNYEIi$ZZC%-IZ9tO^BZ2=-%e;I3?$^+YYuDIQ-IRYtpZCL2B)XQLA0>cvT5jI-l5c zJ<%tkJomM)4)Fz{zG17=c%lhnW%NS;#lp_|z}*(6SB!rRt_m(RS8v>PA}>PE(oYkZ z`OPF|$^LA)rZu{hPNYZs^g!EeTPRQR)7K4gtX_`GF46HW*&|`2zY8{M$?ooiaX0sc!?8 zZF!w9($-eRWpE@QP3{*3=r2$uL{mu0mrg)&-o z_m+fFsOJ&u`IinD!B9mzS}bjv3`8&QuARSlXt0+ESeU1OH}l)!?J2wmS~7s}{O3`P z@IHxOFWr`YDu0r)^CC8jxSijsA{hBD3ONtdAa1^ewXf#K^WlE#AuRcuQ_Szx4UyIF zlA*JIg1f78p^4!SyY06oa}7S2xcCXT{(HuiS5CbrH5 z90c^jcGh-|O7;duzo?mriK~T?iMXSI$NvqiCShV>X6{VD#K=lOujFFr>|y`AO47!_ z?DwMlKfb8T8CsecITO&USs4EUaEuJBZ2yhMF|qug!MLkFKtKQhfWX|J3L6Q5|A_y8 z{oH^FZf0%&UzvyXe>2bj1Cq)3t8>Qx)+7@F6B`R7^Z!!pf6&nXg?KWtvvK?;p!{!; zXJea*a@NFrPjp8IbKjOuU`Gdc5O+`qc`FE{tbL%cy&EjZZ=>qn=LDaBPS1R1Up?Jc z%cCkfOHUmY6(9dJgx>@;q7?===TLqb@Bj>rO$_&dN2yD>)-yS7nIgE>(}7ph)YUl< zC@6=t{mBTd#K9eq5#r!`Ve5e?(gU^+v;nMXXgVR104Q^TXODblarDnY<)dj!_zPzTK|_uu(ooNF6h0p`0y!#KG-+SIi)1HLEwrbl1RiuvdAr2#aMnYF~> z)wHbq!-F^lQX~T)b5Bq2Y#YI5Ik*6%160Xe(E!Au{ki0^bMwgZ%E96kRp#YY3rsw7 z)PIvJ$L5ZH8pF!UtRxr#21QYpNC5<7!2FX!GTw94Tyh6Ge5Jtr^WQ&Sb9ioI{Q6Nz z7DX3bl1SClf3$)9QFq~-9h$xqpZyvYx~PFaX|H$}`-kSfYd`?7s1ZiPW7F3L27<=s zr%(qEPYr(?8o#9SxlmiF`G*!n((Zn3ftrJVRxsGY9p4;*XaN4O_T``gj%sBB;%W!@ z1(M+Yu$uqqj=$u(ai8+{Z$ksVGnZ`jYywsb?D`eeF*$rutLkcK;OW~LUI74crE#Qn zaR*8Z*j57^^jP1><#Imt{F3Y#+S1YaIZ}LjjUMq^eADVI-hGWfUU`2?^L)DvN#ks4 zcl{`2{&ZV2i-W6Ksl{FNn1TS&R9^Y}Ep~isFk6N8SVIv-FeM;ao=4r^M4$AQ?!M~c zf|JFOpVHs(KZJ)(JOH|@w*pjl;sB()kR{tQpgT3T_^Dk*+?IInFWc^{@AB?#oS0ZDpmg;#6jD`7`bRoV?o#y>6I7f&i)m2$%uPE=)$e$>sOF zAI7FX#lCd``0~ld$O1qkbF1a%G=biJ!S;?gaOa!!oq60e7{$ zey|C?TDf9^LyH64^i1CT`R4s?dm#6)f>z|SlXDyc=c)_flnV;fkWYNtg=+r&vu5y< zC^bJdEju@eC-I>7^r=UV>}ADW^W*q|4^O*`pU3hOt8S$xX(gB2R?j**d9KI$m8t_~ ze7}851efC4`||WUQFvztzt+Wm@eyG3bM3@|AT7nA#f;Gc*+3a`I+8hC4l+QtDmN#rN}LqrGAaQaJ&q>uKI=?*Zt?}tq9 zud&ZRI0dAi@|md)IH~A|%J~J1$@ZK{Qukgme4BTJ9I}h!j|CzplcUB-D;k{G$p8qp_|7W^mukSi} zdWZh$SWEiJs`bMqr15jx`PuUPHe`*9ETEO2SpWQL{(wgZMUIPXD#4rh_%`!t>EZwW z8r$tblK=K`{vy^?UYuJ9jep?vkH7LcQg7qf)>B_=`~0R}zSMII=$ZcaVsmfz?F0hs zFCdqT*}tDvfazB%V=3%j)+6$5jLAYuzaYsuEpD?0QabcF#wNsbw@J&CD=#Roqz|SN zP|(Se6X53W@3=l&OPTGRx^e?Dv3@_r}*(nm*LIriv*?+{-I`=`TIU* z$YDpoG*DZlQZnzO`rMRIozc)XUvP!q_0#@Ftj;91s~FV@@5SxOA*7x?3HV$20#>-P z21twyk-ipI=ntv-K3Kw4d+6I;%G#mqQ>hmui=pUT-c~C}yeIOy>0bI>%n!@I3u@qe zH{J-D8SeV0y}}vV*6!6us&9R$t%7Oj^C47%Ct#PO1q9y=&=yl#Rs@Xml!+) z1~LII9?@_`)xq0#UDdI5_#Naha$J#)PIQ&*U7hptPudC;_sdaD&fsgtr0wy8ZBP~S zC6`Eb3TSCj4+3pnD-WMVf2D?ypxqLs3KOZ*B!OpNiH&d`+-N-Z6OFT0aU__gZ7~sk zaAPO~>}@9>5hHcW6+d%T>b1l&VF&M>LI~utiVzQN!<{c`GsG)~?LH@WwCX?nOJdfv zPBC&FJf+oj*X0ao{Ae;dS!El%CUxxVf=5ckGgd4|)G^E=v|#EW0}SAoTt>;SDzIK4 z743RM!-=+@a#tosA4iWFS8g`5=OU`uau8Ak5%5XkKuT1yrl2e%_x*}*iYcDFS{(hh zEiHwdny6#(UUzP`ccm|!_)A#5B~#!wS*D`a7+B#ov#x5hGw*Ez;* z8cb%Dx9_aWU>Db;flUL+pF7LTU`lLQW;LJ*@Kftg&_Ju}lGbZG9@Bc2CYgV3rOLB( z`+5LNY5!hV!#b1$JZh`j){qh9E8JbhDERo3fb%ldgDS(hlV+?f>u7Uc#E^p za#LT>&{|^$@Yjy~1W`jY(tG?@RGlQwBdzr%3<(R#jz zB~NDIa0gqc?(b^7cXgM-Po$3@YjLZx->EXkSh#U`&FXUU=mQ7n>p>`=AWh~G->!xe zi3vxrWc5ffGum2AXm*eDUHn z#~Gfq7f_lsIX3bzsGhJz2ESXv%WyG;q}Kp5sOk}spjP_}13D_=f0$DuE;`??eR@LQ zZwq1PaMDK;D{6_Yjc69zaDI3xhhk1@PpxK@j`OYPPMsQeWm0~DWn z|228Bv?xR0C|PZz;{lDHNg5y`w-=@%fh9F>Y$0|C0Y9Jr+l9>a>>H5u?@kmp^1$X` z+Zj`Te`%b+k013|T1vA+Y?|*fDgIbvUJ(&$Rfa^Ph^pG*oHW1`$#?9vNo=Za%E?=lwV^+y2{W{faY$3RLFRq=7V9hb(LV-I))|Vo-F-E;b z))fA#CFTvH)Lbb1xNYi(AU!h;B9fyjCN5c>9c_3ekPFGv9DBt>bo#jH7WA>wsz^NX zw0X|W2%(hRVWQ3|JJDNFVh00olB``)QH@dLHr(~0t8G5ogyl}}FZ0_G>vRDl&u@XH z$b&_!wv+={jzf=$%qNW7$|#24;YqGUK^73%z3TD{+d|_C^@E=`^+bS|JN0wR<@jB;25SS$)i(sdU7M+*Z9X z{6cLrYePhpK>>(DFkotQ3@G4%r@)VQ1FNG5vGEttm!%-oi^mKdw4;+E7G<0>-@90@ z0rw-|k4VMj55n<^JRrfIdi?^TLoqIuQ1feUa;Vf6m8;fzbj) z+@0Yy>f-~&z1$YDv_9a43&DNhQ`M3`$A$+k#Jtu(J=*%v$Yn}9GcNLCJpoU-aq4k5 zBJQDq$GqP^9o6_q-&$$iS$t72CTN*)J;q4PD3C+KI&681Wk-Ne`;cUl^NKlM&6SIA z-gszS_3v!^N{hH8|E!yB(>N9fV)r^%!DsBqofupK55St6LX^8S6h4 z%imJg3mBF33R=1xt9SM)u$PIYY%HSQ5L-VuJmx2HyWfyNwpf6%-L34w%iuTz&+D1jgC@4S zZ4BaZWs^v*9{}$11r)s}KM$B^rPZ;}#A*#!jX?$#IOJQcae%EpH!16kzX^*Z0D+G? zWPc)ax${aIL7EZIBNB@Zwo33ovj6OT8G?PAW;)dRQg)izEIZAo@gmyR158$ai}lmd z<|C0h_b^;J+Vsu~DbW>eo~V@@*N4*gXxv$gWSU$CisG2F6$%Zb5N;uDl{`w&MhdR) z)4El({qZUG$@857h#z?<`>a9!Mo3(8?m1<5OkCk4cuOU)7skfBMsUr@t#R&vPII(}b814&Kyu7XSy zl(W!LdiKp^`ZEqYwEm@$H|OVK7#KZ?@){Hk1Z-WI+u~)VrxymD>}5?TAXXv*A|ri= zgzQF2#>uLbkV|3pR*{PDz$m!%eAEpHrt#br;R8o86-nm*#&d1N(tf$U;HGo%oitPb&=O2VRm z+s?S69*$3D9_Di(dq;rX7kab=*uYSby^(0tRjbaTG%BC!Zib*WC<{tl?rz_A-s116+rWp7@L+~OiNC%kNv0Y`@xNW zNWNfBYbmN_Ab&5Ez)qB#R1+4Y8C?!BGX|BAPtIuz`iOBwZ==6*G8bm(Jq8r)M5>c0 zmp77?J8>cSuaYa8V>+8&v_cU}_o>A|YKQXOw8~hcp)n-VGKdquK&aQ;C}QzVwH%T8 z6QZL-bMtXzx%d{;TtS^4urQtUb_MI%X%x`pKQ5j3+(X%VUnVD%95a*q-mU1@V>+)O zV_c&2S?>NZICq1>q#A7-es11#>4xy`KP<#t_kY3Ev`o|Qfea$ww6sTz0FIh*R=evt zr@Eq?%KAh%lZ7HCQ76{dNGDX0L_pclLSJq&n@F?BcDD-%vk%FNv^-@-&2%LW|wCuv=OS-OUqbnO$exJW#nO`MTiJj%MGQi@k*@d{cAGIU%3 zV>{%YvE5l>Z!N-?1Lh?yVJivuCwI7%x|GR9#J)G_iGbNtoh`YUn-~$<`aV|Yezlw7 zgTtBo9S2;v@Nj7$4BElu#EJMtpJA}35>dc>8ADZ{VZ~U92I}VtpoO^j&Py{P$RGH+ zVAcafN|Z(bCA}&Br5Hi^?_W{&x_8W3|E!W+>Nj> zjKX#sRCHrJ&JrM*AS~gN-M$!U&NRbPMkh-=M09nl8}_vs;Gm>-KfkK7R?Wf{Ik458afB*o@!|gID zL$wHAK<6SD7BUgcluW-bk}h<&jS<~y$o6{8pTJukMd$)?i`b#Km^JB2IX(W@5^BPc zf9zEy)Am!-ed(}B%;+eDrT@poS_>!T3#S7{f&E*?wnaly5bvpDh2wt2U&8lm2LV#z z?iGhIaA5BC{okan3+2u!TFYL15HYFj5fhlGc+s+#=>-B2`lFmsUkV?YX{m1CeO%@W z%q(s#QndKZhDSy|@I02<0QD`3G5Ap$L#%2e^mSe}->{ zWmdPmLmjkG8EzDM2Q5)wtTYNfP-gzQtxPl=ye3<<}P+5 zx5C(Gal?$Ff#?3ZV%PO8(hk(2z?SLII$SxZb!8G<;x!h0tq2@_olo^g{WC8{Mj5$m z96rF6fZ1JZtTG9xbkPPr+kOW8p-V4hVS{k!8%czWVlg1|tLY}JI_a3HQAT{4Zzne_ zk4FQ97lmRRZW!7x3i5zUYmDkECaP8mO1&1X;=yuSs1lxO2n;s0cO!Hjj8C)?N9qQ@j%|l!)30uk)o=y{I0uAhRrcYU1Tp!Z! zhJ-o{(p?}`Jm#rebE9`k#XZ_2^ikA$h7s9t^9ofy=-B7VK)m^+%HQx52Uef}f<~Bv zs`X(;BMP2jRoYYwVD$lCuULS4>aO(_Tnz6`*RbdBbx&wk#^T*#C#?e7eGEi%Gxe=4 zjO1I_Ulq}O4SzBw*U}`vw8CY`v)YU?w|OYdFq#lhaT76(iuq6jUIXyg zWHg{JnI8E{RZ3zI6{H*Sqkby)&nzdO}N{G+_BpW~E%bIs7?z8uTMO#|waXChas>fr2?4_Bv zdcynheCN9w_O~ow-gyMU@^PpL9}1X=)JiksL)^A<7|&|B zSUrr(2M)nH%RpXFd8|q0;C6IEz+7Bo0qDA%BKQ`lWKL=g zHHO0Z$VQar6{q~YsKWB8Qw~)tTkE>aE}hLn2R;Jj6hX-Npt0@~kxx*Mqq%pRPE|oa zm!Nh}S9!O$KjUvl6jvan_}8q_f=xbZb1C*hTpIhh)jQDctsmCqFB`}1$CP#IdY@p? zP6Gq`m%4|kF-Ndw@igQI8&F$4q=AXFOJDA)Gt%YASXUIjErm z)&~q&Fquci9cA~X4e7+MusPE~19Y9Ysf~$?ql&;$fmp*ZO&DyU$b}8Y?a~&h0S7O` z34nL3I^fY19%U;i2})cC0e$s5L1{Y|Tz#IOAnDoC!eHPCVGUoBCkcFWaYUPKGgbJ{Y%p?*1S(w3}u^bP(s=>mT2-V5`?8g6bho%2b5F|r0=!YT`tv? zv44{4JR$GDo_CX^t4$x{k;Kp6KU-G&24UKgL1)Nuznzw)=2soNNj`m%O$F(IiQLrf zSd~J3r6vq6W`q$dp^kc=>GLdK?_2QbypU~hvKN5|Eo{qUi~gmV0oZy8AC0u z)AIH_D@ngg7}seLTfc;M1FvQ`I5pBTO3MLw#ln?(16<4lMx~c~U`UekXP8wa+J?oQ zd3W%aq=9m-g7>iCb8ZOX-b3Li72wMIP`8qAphU^rXF{HZVVb7+4P6V62}hlQulWXy zCf#kU9752T{RDRua9zj6g+no%D|&FP4y>tO);5YzgBi3%G`M!~HiN47(q|3gi_BTu zPGAgO*0AN>$E(R^wxRM+X$t!lK~)3?$<`n2>(6j(!M!{u6~0?zzMiJIw^VlvK6Usq zmoIO1FU{vHAeFi%ZFBV6AK5ye`M6%&?pAwYGB+QIBSY=}vFwwPXe(>`zoa=8iMVFH zF&i<&)h$vg53245YT=!MruLH_1kABb?2@OGMpl;PKJqPCnG zGKx461-D`r4tl+4$A-bGkokGjJ<--eZ4m2!#HrjawBZvo*nL+;r`N;Psn&KEe@O>z z;lhusdkMdhEThr{j@Hq|2CESH4(02mnW!X#TrD+$ZlEfK5gDIKy{3dd9RK__ktZ-o zg7V2g7E?b7CSninH96DKb!E5uaoY&rx3J0DVpu@P5U=I#%t6-@PauKMrJR!uE@G?; zqG{ggp`OV$hCRTU_e+mAZ3q*REjms<5#YPuf5+|`BNMXgs~Fk%Y$WIuBz7Xgj9^va z1s1jv4)H06F%m$eV}N0qmR~6h#(vREU6Zn^A7=e|aun}oF<`v!sB8MiSZDblj_N z&$WE~=4xf-yi8dviAVwAf)70#k-g8fPxT-JO=MF217P)Q(DA(HW~k|IQbwFp|)QMjVoC8v<= zC5#~j%y9H5O=d@0iO~!p3v?A_9>vL>^*D0v3ljtJ_uSry6V?Aw^$uQM0_XD@yK;+D zPQ`=26?ySgv0ZNu@BM>i6K4%gfxmWJ8rfgHn_;pb5?udYd-F#ZEKug7)rMocgic80 zv2$Q(PjpjVd=q0(D<;+3+~3dv6R?dC*$WIE5m7e0^?SkBtyYK~4)q;)o=7`Awt%hS zPIw7MFH`veVoe!dfJiA?(zqqn6wWa40Tu_OEQwa?~u$4mf`~oiZjFpp7G92_W?Z+3+ z2RLExhU!wc3^kcW^EDje*g&St7?%&-sz;2Q(9PPK8EtJr;#}#Y7W<9#e6$hKe~o$9 z%w}DU5OhJ=3(&pnB=Q+;5jIL+XT6y31yDatwz-VY1~}@{L>G|S87Mo1_qhbOxkXDa zlCfep%XH&D`SMRmD&(x=``l9kqbR}zZ&`jh1Lknc;2fOXOYRMw8jKGTs7HC@N|!%` z(zK6nPMLd#QsTG%0RkEKkoNTsD3TB!9z4NiW0kH=$$~FeWic)4q&Q8Ji9*Pjq%x(K z)>DNB!&Qgm=*Z{xhrWBOSgAwi%-7>?UBZZ$HY~o#=CS8MT%r$6XCz*!{h3iRc{dVT z%l133$tfSyUiO7Q-2KGFA}6jT{-s9RH8O}b_HHXO<4fCb7OW^cQXd*@m`MjVL zJ3l*{*l!!cYJ@^eE+`xp^$SP$9TBRQiUSoh$JwsS!{9;{5s1rhcuLGgvQ zZ5TLEc300WfT;QfNib3K;=!Q&Y-n&^oW;mM!o<0lDCvzJ#DZb7^N=N5J9Vu`A7rEi zKtUk*(@qNykHPq|bMatC?DLm|dtZy3VvWkv=&0ZW|N1hgC>JW>jsY!@6=G1r$kKXm zOGX_Mp}x-t?m6+&3KKpqgHXmd{0D*^0T29H6zFOYB1wOQ#kX@?U%(Ud@_b*v@=YG? z5T~PFKtzv6qT3WU&@0K&KU$P44Wd)|vHEEh|ab#q@~A+6bC|0?RUf@TuQflE70#OCGx?;`N9$GD+bFewK$`oqnUAK z_N3O72$(etcMW{CLv8P9A)ix(?L}S~`ZYT`!V7TqI89n1bco^$NvkjvntG}h&6yPr zD{vnG(deU6)q0>=v^54|jfW3uu(h@il}G*xnVmKo3nN}vJSO_l$w@p|J{J1>l2oTk!*K5?LH$~UkL{46Jb@KY-rgj@uIrLZW~SagOTmS% zmeHeFcEXY-&-AxIh1+Wj!>IqC^}ERaC~sy}=PtlbU`!`ahdY!57d*HJ<`hDLE`5*4 z1S(g*3-q4M6WO2F&JN{{++#fZxDQX3v@#yFa`VeFzA610PXZAIq@7CP8cJi!5YZV9 z)3r1d@T^)=+fo!u*2d1)uO{re1|J}$__V?@xCs!R!35!iG{jN0WX{TjnSvA|;no#= zo>vHbLF8rt5MNjO`kD9T2Bl_CHTbMIF^De}30m-}Z4PM132~x7-ze$WtlS5apllmp zJaQQCA^E*zDpm?n4|z1=jhVQ3BJSaCe`&uaI(`6*o3WQp!tKjeJ(=@|MW34c2lvym zl2APFmTtny33qJn6qG-9PG9QndP5LC-<<`Xi5tpRU9QJMsDd3vq1A0yeh@ji5>Fg2*?geY#0Gu8Kahlg zEFEC}v3y8L;n#A}mkTedO1$k=zj;Y|x0~o0@a-!B0A7h`X}>flD?u=+974RWGDlWL zcxJ0>2^!X^##$nZ5^wjdn-2#aff`NY@TI3r>3)@O!%ZpsAwDu|?-0_4bv-ZefLGQq{_AUNzPKjN zch#ui<`;hY`IHBO35oQWO)yTNCc)5+pyC;OW-Spy$NfhwN-$zDRBKB5nX z2tBqN^?erhI@E%2?&S0XysU=tO&lh|Qjf9+CRma5acaM_<(Lk_})!)rt|C&GnN363Gs(&fltUwF@rQdK&WSNbX{S@jo3$3cg6u`$PL&R~S82VE(<18FywZnPbHEJqTK~^MEiKyNh;G zXi#H%0_Z!RSKpiG`$UTn>&>_jZ*QxGJ7qYQW>E095euisob%SL9ZF)g(g8vkiT*lM z)H>!!)Oj?L{S1U_jn8?|XApWR??unRpP>;Q)kM0&%oMeZ3_;q&j4KPW7~&__(?E0( zAk2LuwZvqypRR5lw@t0Mh|y z-ogec)L9K2^f&*T@$4oEzE3SDI38mAPm^y$!iw2?=i_29p5?!sUpnBLy*hVJaUz~W zcG^@%L_a&O@e|^)R`sq$0oCB_w*w$ap}*@0%LG#8i>fZXZ?#8EZ=nF!YTSJkVbu*M zw#sGXQu)FAGVP_~up->VaG=^+lAnRhKwtBZ=^FFkEq9|w*#MdE#Y<5E=HKfW<;7mw zk`BJ%SH1PQ*xm)gBE+Ym0`azd?JTl46Lq*=7xOxAGLm8Xa{PBAs!^;qyqbQ|MD8l> zuicVb&(MIe5`hGVcegaqhq`$B2fnek{|5j`K(@bNo8aDe*yvNEx|x$NT|r*!#3?sZ zz(z>ORKMF&mY0Ts`g8lA1exi1^mAR#)Vl@5EFc-6= zi0imc{HE}TVFVUrFjA`Umb-ft9V5Fm%;g*}(k1_siD{^+>1CyW5+dV)L|;e|!9ako za35hjRFU0sO`#DJW3)+k^}}qwio%J`PQSf?)}~ByNyVeem>pl}^lSvGFjUIyfnjNS zDcNJ8U>0P*6wHqkO+~z1?B~!!@@ba=J8;sVFl83n<|2YYL)mMJy&=3?jQe_^vUXKc zPo+T?9O)yaP*-sEdSVT(5oYGNH3LS~rHU@jI%%-)Z#;<(#QUns5a}D8xeA)CFF1)h zSyw6xW~!BL;2CEzLA6JWdKKzv-lmDn1C*5lzYC^%WqF#fC3m0t0uX7{jlnD#X99b8 zrJvq@_8xL8zJJ{U+v!-VOF-(67Ux2{N`!%O$2=cgl}HD-S1?~WVKb^Zjfd(+67;L+ zs}j1D3Xt$J?RDY5-6IkBh_*&qi4S*g0??-Sw6;q7dF9}Kiy&M59jOxh zLalIC#_ntV!-{Yd+%#a+!IVC705LpE{aVV)?*a91BzJ9br~hhto{6)5yE1PDX8%TU4I1UZu8fB^%o!kC){GugyuaG4 zXjlX3_JbNvprS-YUslE?0aE=tV3>2`2s(_+BA zgYi;8%F*btXgh&VIG8G*y`n_Ec5UI9Wc-Bj?QYo|+Hh0Ns#|yc zTp$apb^dHgI5Gg`qtYW%5vZ<^ z-ca;Yc#PbRVCNH(J=U4$b7)PwNk5g8K6_6yO_VkC3DhYo3U@mX0vk1|Owf|k0s~~4 zCT8KB;I0(LXQZd^J0u@0iz+oPxxGiL&#lk4m0N9QDnrDUh(PCs<@Sbys_g^cj{9wx zg17q?bfc36IUqK$7Lk~2R*rG%6gPJLktXYZe4=_A;|`h7(V>PT>}Ro+ZX--BHDT_a zTw6z0X%&VS{zVDfMBb7Xp$|sRHk&A#0GE!n9%qn;;z3*a<|ptweM#f;3dIHDrYbOy z=_#(=acU7h)iby>?*Qd#Dab>O-aKEkjtOS`NeO;uLZ*E2Y0(v^hlJ3?B z8C*a7&~3?FaxoIg!uO{XZX+yW5*!rRmVr|qu=~!S&;to}GvGrJ+!Kn*_{fEBBY9qZ zf!G1JM$<`1c*VIdN1bG++0ew$;T=0bso-idV(;FxgTdB&iEqO( zki?F=S`JT~SUh#p^VL*~&9sU3o|(@#^ec;i&B0;oR3F+ozUou?l(wUwvp8c1i)ev= zhg{xUt&3Usr?R-h{75>VQ{O{^x)4MwfF&NLd@mut_~P!}=f(){^>$Rfiv@#5<}Z-?$NB<`n>x5 zx87d&HRZI-Jl~nFK5inE6)*V3mRoNRHTrp?Tz_uj@*h|<7GV!Whc0$kxx@=rvYUi} zKW{+78(&QJ))&Qx@yczHuHrCJc`RdRX%tMj&S)9zwqSAFi^vNz*QK-Pc#`$u z(EQe{8riD<;E5KztQmytQ*7TfrW`UMXo&Hlg%P3Sh5l#j$GP2b)?M>;DghO|I`I>NN7BBWMM$pt+Q?xDPTU&Md+MxklO%Z3m=4k6Zz+=?!t%=pz zpJ(!3?-{U0wSLyR2A>Wu(p+&Ue`pe=F5gB2M!`B17nHcc_X)g>j;jUB+o`K=%kpmM zJ$g*kmc6hQGkCA^(C}79d#0Q5-c8$AZJGs3r>xML|V1~*s}C_FE1Q9dP5&t7PexF$w} z@mOoig9Q8{F$&V>5ezUZ(#>VX8XKUfDD2MVZzy!odsZNFbuq&A+S9QyB1GZT*RZQ- zN#@*&IJ#Bv1!>D1z$LIhX}^4xig;lrovpVuD);!wIibZpmN2kW-^%OaFc5s8S3KDK zsd}O``Lc~DP)R^TkKq1guhiR zrw1F;;qH=UEtX;>C_*_9*>tgtH8Qbns_8%D^B`w7=beC=NKj7xoiK_@RtaE>*$A4=9txGmiV zV!x+Z3qqTPr#e{B?-%fdSg|svWzZ;!xDEkPn0$b2w>@}eKonAUxrhL7!BX5g zM${AABt&6;(|mnd&f40G=2f@!~vvLuw>D!Qtc$A$DX z?gs?Rgc&w?nY0@5HpebI0l9UhYli-o;PuMBA zmGY&iz#v23^!1AiO9qBD|MCPaKV1ZU@FeWh=jKPE7MIjQ&x<+YY)EZVvaV?h9mL%W zEI+H>Do2sesB{rG1&eF+m!Y3yn7kmNAAH_s5wXQ&xLf>x@~1c!vf)3$jU8zmpq0 zGgVuKRdF}|fr6S;Pb2Cnj~iuk|0ZmCyhwdA{bPd-$&wno;rXYAtSBzr%lJii`0<-+@~u>p6O2?jxlg$j%unJ8x$Kgy zet9Z&IiVd^{@#H`m=h?qqR|HWdDj`^uK8DbbUBH1GRYeuru1BEZ<|FGf%MYo-}L3( zT$7!M74HKod+Lzs%!D;QK4m~+H#1dr(|7i}m@QYOo`K;;;=5qNp`btJ*Ur?8yJbT> zWo3hY6PmoOkMTq>@iei>i%My2(C!F6jBD!gAU%JS@bTf8yqvJf6|A^`3GrqyB-hJ6 zmJX8Z_IvvYy`tVEx{AjZkd2g`mNIAxbu{AjdvfB(L!Tmpnr~N?yma^1Lto2V|JPQ+ zM;sfts5o)}#%D(iYHC%!9&VT}(wy9g2OCW(M!>eMtB~77mU3L87*GTXX=0;C=GfPw z(ZYfkb11haQ}-#)Ozqpaj^h>X;fn9=DP?LTZ{9eD$IO6Tz$tv!G0~t(K}5?}a!uky zRFrP~ST+^XFNWwY5f$OPxK#e4#4O<9lgPa+<_GjNdwCZ(P&$NnU7B~(vwPQ zGPEw%8Jd<|Q&TX%J#`989G)}Q6Qoc{l_kZ`#7=)oiTSaQHXh1OU2|=W$=3u8 zg{N&e_texOTZ-6iWSqs`g%05hr{DY)J?}6$KCbuJi%aV4yBdM}+Ir%1X_d+ZVwkp7$CarTA7E0=r0-xN3LLuBp2Plsb%7PU<&vV75M&&#Jv3>#9u6$jLYpzz`cpK)H1I|Z( zvDZSpJKZcW@ggAyi`zfu)_i#%i)eoHw7m_PWxBfe5_5RFci|!v>oslrm=!iW3WfFP zgM9*V&=no2knvds=eTYX+MO zO492A>WFqhv1(~uLoa_=_PU7&vnt)Kf5Hj99O1zr6uyE>@z=LiwKQU;rDPiQnIlZH zJOFAhh*C{Hi)gA9ZX38r$3D^I_cYzGPq{k~Rg4QTa$iAPLAW|&eZqhq;oPI89K*Qz ze8ygta{1Z9>17Lq(LNFr8=YR~Eaw5iSsu?7#rmBUZB(&o68VTEA{G@0vUUJybAR=uh7RS-B%)LS)r)*t|5d*Y= zeNU0|Q(;t%-<5EiIAI3bWdey8ywDi{|Js*g{;>fSOvmyVe^z0(g?P(+P=WuoAol}| zZfC0-75Sq3S#rX>8Vc=}?ysz`KP;?xvpy!KXt4bh9z%g=9+ssTOT$xmfWV|(g{V5v z6X>hkWx&y~`?a?wlo~$E6ByJ|lNJab0!7CtM>**8I8Bv1E+)aQI@|yKqmY7venqla zens;>Vl)`nwhHAY(Ni1=uNz!jvo~FY%CBZ59qbY0ZR^~5bow{DO3l%X+plY!r(NEN zHT@Z8v6kLSJ&sxFxK}}J&2}tcA&!W7-q5*(pw5^Sf|lbr24b_X6e~(nno3&Vo~zj& zZV(tQJrz#P#17+I^1DGE-Zy1vUm^Ku!rTHq&-N-O@Oc=;|rc^%eMU!f^CBi#xZ1(Q13C?Na z7FbZiWP^)GZ8TTgRMT%aMvM}c(r7O;_v4Whm)64i4Uu*<+kVH>ix)D#ME%hl+r)4i zsa&kN^flfAy+F4(t3co9^0u1a&^GHlTL~2fbZ(aPP|%DC`t`rzSFy zHRO(+Cd*wrY^b=FdbfINBO+&mn^yv%95J?dtM8jR`N0UoqYM2q<0oMzvud2E{mpvV z8|%4D0yUZ4XwcycsC~AAJfy40A~%g_0ruy#{h_;DlRO2YeF(fx@ohk9(Exu8c1lRS zRVP$v?5MG-O{%!>B5=$Hk2^CBC-?auQ{0XYTzadmukR1}H8^&C3}F`b{AlR#OV#?e zV6i*zO+*vCIawPO9_CgqD2~_V=ea!|nut*(GtzTY7?Cr}Xkw4ff9Dbz3SpEw;J!-Y zLR7APx3c^Aw2k_8JYz6pTFnf3m9UOUk8%1MYV&lM-9+BtIPw>-&85?CgCk8S!)kFh zu`t}?VcKX)HMb~S91>(CbjT}rK|uq{xZ-lX<01YB4WE|O38dU|Jg6&5w6p-eTfyu0 z1T5dfO+!MLcG##49fQ6k$1y15=n=ASQJ+<$cZDK<`Y!Q!%rBk4I2qqph<3b8_N*=d znB(6Tbke|$5yHQg3>f>d?=p`wdek+sKHZW6AU?n-P=+pp@ z`?W=Kp5!j~ec+|8`eHvzoAe7o0o9;)ow1J(;1{!~zfF!6^@?Jaz6DA=7Y|mS7pFp* zIzC2WeAXmo@|FPeuOLEMS>$2CTMmPd#+0l_qUyic4sG`z?Fnw>5Edvnzg<^qx_##y zjq60O8Yb?|bQ)=>f@p*ZI;a}e1{zNE931*nbjocM(^c(@03DJDT~b8IkJh3z>1-n5 z<-v4+Ev^b^#>O}!h9yy-&~TtP0| zv>bJW)+mnZ*N;z55ZNO3X*ytQE70@V9wL;H&3bZLzZ&7sHt(^R7<%EZ7htW!FWz&M#zXy;k!O(E;(sPwP#t zNk0{^Fi^=+SpzL0FMV16{aG(1!Gn*xXD)0c5~o@yz^K@;(Piz`-e$eLmhm^G0%rg} zv(JDE&Md;KYPLF6=v{i4F2VZm1WcOZaf7cm3Bd#Ua4x%DOne%{vh{Sw4_th90(UB? z`0Q27#x~OF_7aQwJRJ&}4XBUlXOQ6PDwoyp&IaI9-Pm*7B!Y;?_ylV@Lh(mYS}Zk} znzl;U79Z&G&HFYFymn8Q*6-;f8gqgvA(>#&PbS95H91BW)OUq@++zzLMQt-TO_+;I zU@kZuc)Bcm^S%^cs0SO0>YHRGOHQ?g%PLP|4F zxiVTlXA{K9KfGC<;nzPm5EMuX7{LgbuzQe7AZJR`WDQ=rgG6EscB!3JE9@ib@7}&R zCef%)C!vX588>!(Z)mvt|JpdGu1Wv}3Rfpi6Hm4=*)`d=ZQHhO+qNg$wr$&W@5_C= zKVoC8y}qp(rd?Y?khFD|L%)7Hr~(Fo`WBbN&5?tcO_9#~mkRCUWfyI)?fE zthsz3!V7M{{Y0be5))(xOeI(aCx2|H>lz7V@}50WL19!T)QkyPTpM%Y0aYw;=I{pm zn$mn=hj~vDTcdu%%6FsbH$W8Ga|80H&@?44c~-5Pay6Ig`DuW=y4(rvT^5-&YO*fyUPvbZ!=B^g76E(y>2Y4s*l+}0(PEMx z+K-VkPRdG2Eo38UJ=TKSHv}n$F@v3={Y1N2IIP=I z((JK4o`W(-0MVYUCT~k5cOdH}KI278Co?c3?k&KpjViPJVe@Kbqzs+nKA~@R@&=VuP|5G{=2k-|4Vp zi-ml{CsTZEK3qtC2J?xDu`n_tHq3~S>A~WMvV+Y^pXOOFYn#IuHIn*g=Bs=|3kNGeQ4vf+W&?>8sCGgA|oTbz>2TX1#Y;SXzMQP3R_o@<8)p6-=MH0AAJ#7 zL)6FP69@DZsNrML(i$p#juh8yZnha{bzrCUc;y%mSD%90!(evc{y_gNTY8k^b4u7Q z58s`%Rf!1%d}%Z$F>)?kq+;SqQ80?4QFG7`A2HIhrdwY?yV*m51n|Q!wY4y1>O-1d zvV#=}ME(naSwb*!rFe??&^mZM<<{4hlBH2DQHD-OY-$xEF<5^v=dqxhXDQR9&)d*~ zQlM=XtFMg4#Z5Jvvl8vAcT?F+&DCn0o|tJ~%hIpx;K)*mlORxKCWLY+?Yf9>nePJ{ z(A4)Qnmtcv1J}Eu3E}lMRXVpWV$8ys{A50c56S2Sl7qKh0EwM{^BL0`C%c~?W2D)z z7csX0si}V4s4xBY-fmP9vEeaVT_^Qjx*v1nzm0!mP^ z{aNETYgk@%7R3TMwym{WzLm;n9E^wQj*_jEf7YWVwIs;k}c zy~PSu_rEvS z6sEx(awOTt+eP}d5V|WtF`})QP=9p6ZEY`9Hn^64>o}-W-nwbL8He4bym9=EzU?+M z6N)?!(3B?4ZbjzTHADOGq^U3%cIR-y0{N>G5l`B6C#Df9LMdeC9Y$&Zkw_~ zdIvBpowoVMi8S&v&Up;D*B@(Lpi04RW`B&Ry5VmgD{5X=OBWHi*`Q?Bxd zZjRB|(%1y!N&H|w~X zW3q=DO4Gm9J1g2MXVJbxamL&qkkD=>@c{1_uq>Qmx7Uk)z$&*d+W}7@VydiE>kxYQ zNvIO&@`8p$wr)1-MG55*GkN00@SfGhDJ|(y{eCRNrH)^%q=tBdB0E<0YB@rCE|?Pe z0gU(F`_RmYDQY^GO+XCCS3Xf0!>S89l5?aS&$T)?-~*$y!233`VFSeRxr(%}gxw6R z0z1xB?z1t`#U4&~y09)$`*Tdwp9h(Oc~Tbcw)1Uq=dhy)NKfL;604%xlTt_Iu`(C3IM4-Y(vk; z@_%KsULRT+681qKG~X-9`f`zYnCI_Orp7KkxD{6i1bOnhSQuE!n8;<>jE>oN3q4*d zI~W&l>IRXF;{T}MOewqZE8$wpXl*?@}U~6U&!&k(>hBo-mzw(v3rsYGR2HUw@?R* z?(Humh~AMH0fZ_B^zchN9zEI4S3g2cX}npaQoF@`d;Y~tyYmRbAE6mLUH@q0d45Jl0Y{x?Z zjFMdbVqLC)lsVe!RRRZmnN89yr_1_crw(`0mGb(@DIXsr)v4ZolPxfP!2sOZut2cL zP7dO`fp6V#+A1a8qN3;GKJp1Y*$ke_#FB~T(7({HzKd6JTU?C4Q0lDB6}WWq$=68{ zFYJ}j6mm9OyI!FqV-lLnM#~KR^;dbvKXa?phqFN}w5hl8spB!m z>eoU(>43|cF>ak~9*oC42Z!-!EPy@Ml%uk`q@Nz4aBj;hJ_+ZkEjwN=6 z5Y@^R7!dxvb=9?W-i-KzeLLunYImM&#)!1qT*M1Vpmu8bRX|G2K97S|tIajhxct9r(t!d7>6H~g{$PO!rZFH8>N{ORpBk#I>9dv*8LVqraDki>Tvfh z+oXvO34~2yB)-T0ezA0wQ^|UIQseZY&2B{;)HL-2_~{0&=Kd#?Q>SIq|6?7q&_Mmh z-aL#5IS9MAwcxel-p2ME7BO}W-xi#O7b{tNp@R3C2va)V%KQ>fleSP3*UC2mUyukqX`#vaqRnJSh73# z@^vm1lDETknYv;a~KkQCwKnR{gXQ|$To-C@x zl;2wQc0GC|zyG^`_7!REpumY18$FW64~{+L(r|a$XC|U=BY(+-cArO($+x z_&0m3q691D)R5dq{I^%_bP&I2krVej@0y|WzA%TNn-`NABn`fX4zYma94IllbFj~L z7l=psUiZPXo~Cd^-PN3X)>Qc5yzW%DD^0)~aY8$h6fuRt6JAAcFV4UG-yn>qC+uRZ zU8g0M^O)#@cm|mUBI}aqJo=$oV$Y4zkoBkrqAaXA?KT$jeKD|u%;tpWej+?A+DL8%Z*7MwW?O-Qf;CsCUg-t ze9rNfA%k!s8ki=*)72S^w&%7(5TZEBl8#-M$DYyGxYfN@Xch0Ge3XUX>M!I;U9e~? z-4LhgWyeUC?xH57p|*?3O3_#IiYZz4QA62(bv7qPA2^~PrY0uDrP7MJRGKP4gpjF| zvYfRW(p(7m!~OT?feIE@=i9i6;Y!U8-#@)@Rg_+*C?kl(=AumbDNOpR3@H~>n&=7pBu=Sg~= zgO0>}&~vQB_NvhgxsfiVt489_85$a-ioFUoC^xZfgo1dcB!=E~+C=E8da5FM{jnsM zx>1;3(D0N{@g7<{&w>Z?Ao`m;Vvv0ok#@Qp zFA1OCGm(?Y4J(xLPLC(&-r6w)g+3|)NlIEd^*xtPtT6c>@bh{qCHygr+R!Bm8aU7X zBMU9Pz`v}B^B)@E*8Z{o;wa@6CwXs+J{ueEW!EOP11>K8q)~gnpYZ&TSpk)J3I4tM z=8bw-l6{QT)nn8fVKJT^|KJfjr25&YQ;?>mcnN{7hy-bnD+4EvxwnQd;+@;uDv{HY zseRH-njMt|tguTg#JA-a^vMubOZ{`nE7jL!J#U5I!TTkMRn)<0qN{$BKPad|d z3gr$=ZhD?ygu+nd%cxg%rKZ~XP30NEo_#e#FSBM*90-=q$WU zSz5n>MKBP-xsbqr0~F6$X>P0H^aHW}rW(Ke+CJyiycWnAZ@ey3nJ|Z!&bAa%=wmOF zRJ5mBY2k`(t9~kmAt8EI1_&H+54;s*2F!Yb?kln7I zfU?e`=p%#~BcU8mNUc-wJ1H&svfAaNlpZK52Z_;aCZO`G37+4 z7gTYfM8XS7#wwxF`KMr#Ez*B-XRp@D0xSCnk8FKj26YyKzSr5fy4Z0qgd@B^^{fz3 z%BIOBq^0j;i;V@TiNk!C)O`c~+~cwJ|EP^vxyqq&_Zj<=Qjb|SCJUMA~qORHE(m4d&sIhe@supf4QkkvKCNniS-?W`>en4644Js2W zoW#XU(T%!))o)Hojesm<(}uTyyQyN~`w^d$+o3&hB+)R^@Pk+-rFp}95UKZe%t8u` z$P;-5JAx9MZC~cYCL2x?0(D;rzs~1pcZ?c6%94cjN~T~FTdGgNuW0q$KvyE9HGsvG z5`x8U=JQyA{bSNqu8u3>mgHMCJ(!P3j}~vSkDR!~SL73i`ufUwaUIE$g-Vt!TOy|f zR^Y8e)LDSedfoq_ovJ179 zdBJe75h_6x9j*$e?-IjlqjFN)rD0zB^I5jMGX&35-(S-0?{dluHm&rMBH#BzCR(eY z+wjnktbPu?6=^c^cV!J+;KiQ1Mc0pQSRw ziV21|Es`2dyl!Af9lrvw))R4z=^=YecW2{c)0qju>ryz{2rI|^$khhOi7keV;S@|VB z$+d0|qP-C&8v!NN+&lMAUUm|ftHSH0ARdm~yb=UFU2-&5Dwg1+$~ojhvv1~2oCvJ= z>zhhbSdsP~=8m^$3!$@ynOW%3Y7y0^OPNClYg*=>!%6+zsktpum?Z6G<~wVp)PeKc z0)N$?Fer-p{@AbYx2n(DQGclpzq$M(ugfH?L(!uVZr=jOOwWi# zAO9VSRmpXyhq_J!MYdy%5v5W^6dld0u()O^#oP0G$0;}B*-%nvlik`NyYaaJmW^2T+ zd|>tyzZl8MvitpF-{(e+QbsnBGDP49s5>;$p*v%)JfW8&EGaKSz$wzgd}s3&)hw~~ zwdd#j>Tp?=`jDi}3zR;*tRq}49nCtw@A7(#D)7p_Y#x%rD8ccLSyrZ-o?h1zPW9+I zw|;ca4D6>D8s#N*q0PF;se!A___s6lm&FHlOUm{o2(qJojSCkiOxVLd^+0eVk*yBC zuwGHG{uZ;m!0P>Dnbu-UiZA3eU2}kM5QIzTFJ*f#2s%j97~^L{m4f1droNXFdl&dC z?c`@LGZ&gT_@QcEwuj!=(WY@>$N@g2tR?~QCabvlkST-103+^cix}==?NSpKQ^2CAvb5;^HWAaz!zW!4H#0qmlV_MhK85P^gVG`lW2PhV)uPhzaw5A8N(GIeu&FM9= zgqwvC1@EP$YZ*4#&uQpgXFjOwChA7H6j{ztJS+UcVTcHYcub zB~B_3R2LCK$ff+YH`F#*9H^DT-S`S?m#|Im6e9}9zzv^qdP~>jI~vmM^k0v64lkKG z%c@Sd2iObO%+|ziH9-Mi3{jA;*dN~7mHPCygtfAGZi&AOsJHe&Bg*mc{ZgpURk8SCJm6`pvZpaJ0A!TFakar#dTGn7geK@BtF6ViCl>cWHCmArGSOG7Yp8T0cs}njxn4YmHPm4PTN?pX!A)1W zo>OM#kx03%H$6&k4r(Ykz)kYY{qJ@V9_{F#8%Dm@etJAf%))cYAF4*hx&x( z?0ZMa)2s-8KqOyUsVG{$=$UYo>|n%Dm&v-e~C)NFcC3* zMByD=E8cspgv0jT^bbvL+$vFHx!I6aC)B}?~4-Ze@-K=gMGBj}Uny`i>;zCq+9^t*LH*LRv=$JQ= zdwFRDppyUHLnJ4m^nlSA?utaBJea3W)nr|hf4qM=p#kQ0_k!&g)(wx;{#H1G z^Ih|+36|Z)fd}2$c^aSYunCP?;2-HOLPLsE$f~5~_#;p%echf6ZwG7+QAM*>ED@#C zWe>dH!}#GuP<^sxh*ctBgYR0lUWRhN_f(Vu-(Kd;-Gp|ZrB(qx1twqvF3hRx!eP;bT`o0PK#X3%n-gXRd0SRd@r zHG)Le2O@Ipcqwb5K64XHf4H0I7~KAn!~6_>u-8nT$CIKx1VkkHEu*Kz3C85P-OHNs zV{DdCn$l|xJ1Aou1kOKf=Hf~p($7$~*Nsv)K0}41cHgiJr3O5&V#Mx!aBE5%x>IFE zmf`-O5MtZN+daE5FQ8#uCFr5fgmE+SGGr&OWbVq9XEQO@bJL`w3%obbm^`8SkA-RF zZ-R~X0MnyPZB>{SDGYDPc(!7vUP?_OytY1W+|!fOj+~56X)vT-ERQF3lxu~aZk#sd zW#SZ~f%^|%5L!ULJR(r3C&2EV%2+A$z$B<9*_eqGlPl#cYfcDx;gEa)a1+Dt5B z97IIssm6%e1LNK)+{UKZNHPw^eZmbY8D3FfA0S&cq|&Mwo60-b`_m0ZTFY*C%+v6a z61EGPhx+g{PVQfaT1N1K86DbZMubYS`9R;s!Ag$4sby^{TYsvfwVU{%O=y|A{2tr` z&KyG1^Y=EN3AtpWa}J6W&>KjzB%myxVj)h5L1jnQu`(Kavln0};)Ig;ZMg5&6#lN> zvZP)5!GA6v$`3Zf2WD!!jIhX_Wr(?5ksqOc&(7OkAE$sdbg9jWE0KRc9(e}r>Tc-{ zMyeLx`cas?23>QLa+EDSHHht^5i0BfkY3E=q8j?bcsCDUR3?(orRZ{n`J1Y~vr0?` ztsLvYBrXPegkM>bM)`S0aQPjxIXj(qAYxH^$2uV4$6MmS7YssG3d>taIzb~XEllWO za6YITJAu}}DmV?Ac@sGx-+L%@6D{$L^_ zD~qX89$_?WDYW~51RPe$(-ZT#`-D@+E5Esh;tZQgIG5#u=e(dwAF@wog7;sEv5Hcz=$ef3glVAJ_v0y&8zSH~EQGS4z|6_l#~gVRLMdLxwjIVQ z77r&qb>+3BM1PNQE85lRUjGC!7eVwDoRI#mrhAtn(s<^TyK+s}%Q&@PKYX2K#cNFN zf(i!i#3BftF*z&qlwlf=@>-QB*D*XH)6z|&gCU9JI;(Yp`X2Q`(H1@aDon}ldk3!^ z!gdTr%t8vTRuSu?6M0p%;mTr2EZ;7#*qk*J%Bif$lA)Naa!$F3!B)(m3PR}xx}(Zw zj)>_*PqiP($i`>xy^}{QMDsau-MN$x=^720kX%4!Q5o2OIT{JnvKP}fN2srqkeHpQ zNkiHlk#{CD(EuCMAjIFnpg5;EW^P%!D@s}5}`n3YQmLST3w!tqi75X^uAvK ztYhJ5F1exv+wsY#${FLU%(k9u+q3OzBIWb#bg_=0f7d2k8?us_RW4>=jtnDds0de# z@Mf?4DFq>Uqi5Y@sJ%GuDHV{VQGk~cvqTOLpQ3+&Y2$he2$|<~qb$;W-Cnmrjik%y zEXg0rl5shbAgjC;cG0Fni?9^2mF@;7z7wo7(C^H)Sma}M{h-6LDl-z8Sr}kJ;wswx zSdy`Zhc?lc3FmkPtlTe>&yr^(FO?fWKvyT}%(gvRZ8`f=B>1KQyZ=lbOYP%Xw4>&y zI-JN1&Q)?L6XEno0aV`9ryG%9+kl7|1&WBXv9oh^1W51>6KE-vo+kZkTpUtIl40H^|%U<@Wc)?=aj;WjyyQ>hB}=j8!do8_$X9!i>>M#NS6F zQ2o#s^voIk*F%l?0NA2_8Hcs|l*TO#+o;5*eYLMNq9{-iffMT`i4|N|uw9U7S=t3y zllImv$tdsc%m_{!5WRO^aabG)e9$Gv zF3Ipc=zpXRraw^5EC|0Nz|0!rG00}CH^cnP(Bgo^7izFB7U$`&gsa^jC>0{84Y&?0 zQXXZ@ZJ%ok19Y&E=aahEAw5cDKi!!C&C6rN=W4bFrEaCO^t~JLXFE(!eXD}?Vv8Xsa6rIheq4r#SglVtld^)*xF*N5OOk15?}h*YQBUj_M1W&>p%}n z0hfZz@w8ak68}l&yFs?v$7U${YIoUa#*o7w({L2ZEhb7$qfyA}Lt-7H#e?)`J&51G zhNEPDUKaZ^6hgia#vD>x}kpsf@p-DM}}NQjiRISXr$l z%_^?o3xAb~vUj6tY`Z5-W@W|DRb_w&7-_J#n(}}D4M2cr{nC2}g*&NhmQ>g{bIO!x z6)htfrIxnoP{RBNYYWsGFaB~+`nS-=G{Q(qDIAu896oqgw8>!JuO)WpR8%hYey0)R zN}6lZq9uWqFcx_P*PBUK%d3#VC3bWO--B=qSuZ_mG=r&ISJa-chZ(w`JMmvd(@ zR8rj8kh1x5oXbnD+KZK5zkoKp2hC(0rI|@cpH$4S)>pnmsBH*Hl_7Z%BZ;P3#{{c+ zt}EEc5LVAnUGU=<@cp3Y`$z}WDLW$HU{l^LF1-L>CQh)m*M3M>I`>sGocV8!uUAD? z7aZ|i^}QipE(xavNQmJKnj`BVFXPn6VM!}eMN0bz<$fUYI_Ju-)b35;t`-)~y}bd5 zEfyp(Tq&A--ORZO&DQu3x4N(1y2fu-+8V*_iFzT_m|s@z4HTzLV?E^FnM_Ez+-sQF zxt5vpm18RdDDxb~OS-6-{UvFATMC}X(g#eKydoMJLow*(KFK=dJi zb6?d>W*k!3e}k26vQ(0gV!Z++X0a)h?#xGbx?TQNfCio71~qkM!neBoy&awzD(w!H z9woQ>x7wP&7X~Nf!tY^AI4n!zcKgiP-MGm4gdppSkL`04^2XOsC^N6Nlt&XL8{l>< z(MbkYXCDSSzzbX02gMA?ERoWuUJ1Q?AfpMGrcnJGy}Lx-9oNaa<}esiDZ~sl%zuAX zt)>uZyK&|_%{0M2Uzd%1_0`M@iH`y@s|R_i=0KzIc5ZX}-y$BGi-b3EOGtSgEt+(_ za4D#%fYY}A;VC!`wy11^LQ3eXzegYcOM9>O2NE#ucW~~2mvnxWSA~b4NIC%#H@y8n zLdsTb&T%qf`V@^X{Qyjp_UgDw=Zv5q0V5GjXKc%L-}}UcT6X9%GVcPd;nHQ~L4U}* zS=ZiOQ2!VR0X_fT(n2X#65RjNGB}nM=zSprg!9f>F#S04TX2&KSjO-VW>2JE+PpGD zK1J5_jGa@EC_$I!+qP}nwr|_GZQHhO+qQYzwr$&XZ^uT=#>~d|Fi&;zWLDJEKPuzs z7mN&w^wBf?o9*3hW83v8K-}e;AjKI@$mN>RpfenPYA-~Ae{Biu*C~#sRJ<)f@2AXF z%+V8F%W%>$elCa4?jaJ+GVsHaf&?1{#=(;F~&U>?o*nhz**F(fQ0L-Xuj?q!^EjU-_BPY`kQj-J@ zIbTpFV%p5;WOtM}azxNqt;ti2t$&~$1dsBV;w8%nL*=1^-_MHm3`sL@bm!EUAIS+2 z0?(oT=Y7%4b>d(CF&9U1>+z!WnMZPI+R1>JIOcJiUGiYY2 ztD05^^?Eot5(QckZfd-T(~S{et`iEz`Q@`WufsBgrZmxak7ufaayGiY8mOKMa2j>i zOCNtve4m&J^lI?&=T&7&r6o_@c)$5jmygrZ<;t{k-xxx*ANx5w#q1Py93fgMt^)A@ z#3GrK)0WEa*ln#s;^OJ`#IhFf$hSh`tLtE)n?s3hjn1VaNGe27v~51Q_ZgA2FTmc)q11nZlCk}7p=6BgOf3I}9%CZl|TCt}Ji_t`IR zQ(w>Aos$g9?HBDAr5BIoFPaRs3M?7}G6tFO1LJ310fFHUEtUO~VuP7G~$Li|qoEZQU0mSItm>D30BRGY> zUP;C^9Dxf!*I8~b)gt7ufyw!!5tv|AF9O1VQW(f$-xi{kG3-ZnU}Er*Yst>seieYb zlA~r>Wg;zx9#lQRXd_iUP*Q%4907&^d@#VIz;cO83P2gD0CExZ?0lT!3czStrCGTJ zLmeOKvYm^aAJ=ziWKc*+3JkM=qO^t*6u`U**r=Jg*|(2H0D;_RbsFG!hTQXxa}JJg zDD@xn2=nuD$$DVl8^B&TJ3vk@f8W$EboO;}6ThBv-r&DnLRUX10JE9{dT>Ajd6kva zmC23BQK;u;L+5%gp}8fjtB|`;&P|{^UATfSdHm7#A?iHM$R@Dw)O|P1fD{2!f(YcZ zHw5OVx?&%ts(hfldv!ljm*t3cJDgYF4&WUA0zVXn*5{ApstPJ9ho?DT0&?F$0=YVXe`qZ=Ar?{h z-+p>i^umIVZn`x&xB;38c3r0lILCfC_yx^;4OIy+{9pk*zyaFRHGzH6KMIp~7(Q%Y z%OSUO;N-#mip#T+N3pFSJv{Q=xzaoQ02stg<62LCu)f=c3{ODS{igKgn0&ENoA}p` z2vXz$esph<6Mr(ki2>gE4aaX;3en`lvblSKjKaZ`L;7MWxtWLh+M||B+{d|;))k3(|GuOW+z+4i*{`u1=_c1tL^t`{Lf2&b4h9%=d z^!85=jSr6v+@*?MyhyC^n!lWV1O9Y(=Unz=e!y=e(o9(s5|EPvkk8;FpO$0(?0-1{ z^=JV5bx^I(8#GA&piSe`Q-*Oa0>RV++1Wh+NaNzlY|X9aEcW(I!QLAACNzVNUd3tw z2&h7!Jfr*`32_Ie$&WGXiA~Bx2517|KRbZ@0=BaU_FKM&TP7IYLEr~~2>!h4p8^Ho z^56Q_X$16RP&?iE_5#QQAHAdB&Gl6L;$86-nRtF7`*Ga+(y;8s$!{URts7b}{^6&S zEMX9@%jDZg++WSTE#2|$f%#GJJHF9>%QdsNFUur{M#ty(3l7W$?wua`c+@-Dz5f=k zdee2C!Effz9`;@70mlabYWJUi zCL&Fk{Obwp@i0`o{epajURF@$xA#H9FOfqhL(si7FyA^je`AVfQE*Xmlf0eS9fyQH zxoEO<|5L&|w1_tG~yjug}t90RvGBSEFk$AWQCJGC^HlyW8 zzIJhob7i+_Quin0kEvMZo__3#8)JM4u0dlIGFfgQB-l|r?z*OZt7KG_{dUn#)LtP1 z_Z>7BMX@+RI%t=CW)Cdcsh$Hl>R%o~X|<`i1W3LI_UFKvm?TYM%AuLFDcC<>Of;Ka zgd)7M3!}arb}lc&Q7kCB1P)y{rNkSat(r1gNBus8*bhsO*WsB^((->j?a7f_UIFoJ8P~8`EP`? zHUMwh_*tnHH?NkBwz{MIhHcQWkG~u)?^nj>0bV8c>X%#|sKhQ_(Q-<@mHVx)%-1 zlRNUvk<#RRNq+6Kit<^XsXItr(BjwjT>_OqT?UZ2G|J5s#$A2bk+@L=_hw8}=9gkC z+E1AqtSfV`302+6D>>#zQ@Im3G~@O!y)_jTRLNBHEBb1X(`)Iz#4`>u9A%R!5*v~qQ5)LenSYyo z8I-jf!o4p>WkZ}oD#|qGJxFww?%UGkJIzD>NxNxDpl5Bnnjf7jhA((J02q&p)|gl5 z!R`saB0`GYep*vChGehqOQ?AZ8L;{|i^46G+z!ZdV@Xz_u!}xv(=-U#Oyn%M{tK(d*;aL(53MzeD0jA4rHO@q=KJyXNxE5Zxs%1J1P`X~tH+VDIs${QM5zi0&N zNj0PVO{LoaVki4n{jR>{?0mDzG>a4z&Z(P&zr;Ho)nmW#lWlK+A`%`l;C@X#sS%;y zOFLa6MYWb-VGke(LCj}QDI}PJd#;^2%xrbqqbrpQHSJb)1?sYaL+78QlN??0SO_T+ zoiZu9Qt~*Hfc=pm3d`?M@nq}|vCd8pR@QRYmYB(pv-jEof|S>19e#H0_sC0@5RyHs z`_x#&t!Ru+3zR^ebLZwnxc%f7u$*lUspF(dM{;oB!9U?U3OkE(CFlz?U1@SGJ)C3Z zVe5uelQlKDZ_Wx^@Vfj?F#He^Yrq(Uc?;Q%T`N_*A4n+fX>aGUdKY2uYW~NESkGF1 z$Rj56XXQn!Ceb-DX5j5iqRIRJ=Hy z{@fnq1Xde#m|~`M;F{H>!H?tPUqCH+7D)}?20%R`1SZ~`e@SA6DT4%{wQTw))Wii8x#gpVGKn`-AL-?zSHYMz22naon7p;zuckO;zF{d}G@x|i+2F2rM&|FY6* zAoT2-|H;`SFXYed^yqsXu57N#=N-o0H4ySMTb0x`9!_%Gv@04~9X!cnqJxogs%QT4 zY0AQdPkEH@{|IL?D9*Ai8+p#t`g88lZn8MR8B$0^7v zBjZ-jUc(d7`1@;QJUxH`v^iZG9&Twh!%2#eLYM1dgEs}dP5f148)EdaQR@TW)M~rs z$`BV&*9gYNHelW4WQ{|)ZDoeS8|qRvW)a)5BkRR3@s}eAE2jnWyBqK_~>qgKl zrFHmcJnTrW%TK!kojfM0^onG<^h){LNBaxzx?IziRW(}we@!F#YoAfw!T+3 z5J?`6kG?nI?MDTJX|V%AmyyqlugumPqK^jB%Mu?&ufN;E5_H1DSS+=}%2tct~W{+PGa8*7+YF~3_9;}HXU9*ao*{5EEt0Cw8A4b?LPxbjd3EiN03Au|9zlPGGBFu>i+4R z6N!vZbS6>16j;@v$I@iU)_ly%0yL=WN8z?yvPvFa;NNq`KzFjzg{f5w0f4g~fyk&& zUI{u*yXR*tkTqY}E^L`nu$zBG>x2`z`G+W&Rt>0S^AD3RYj$$wpRZW^PDynC3P60^ zpD^%KJ+`x~IVVVv$ksw>3aak_kJi^546dFgYq~wJAXmAzi(Us4r<@xb{ur}y9OH;i zX)2DBGYt{UTb-5rC*EU3fe!Pwak@nty40|lt3)l?9<{IK1l;7)BQr<&mK*Uf(sc!o z3U(#vk)*F=5-=&!TRy@0@q9C;+$(G~(tM;g$Gs%l zedI*?g!N9$zdM!am6Ez0fXOyXO7x3d(D2~)TTE&N!GXP^E47HL#FCkb!)`c+XcVB3 z$?b_Jfnm+-3Iq~-61GX3IzYV>pKE)dGzr%E?Q35-v-cR}0pl_m)^F+UCehRRF)l+zRIMo` ztn5?!yLjPmQ$O7&1Buo1p#0B>sm9b9*JPy5!Pa)=2qd|V(djM`^h8_@`T3U}`5$F) z$}6M1r?;;6OP!pgwY7HxE!h>NQO5n|jX%ZvvUQ9EDMMOn_e0uZ^-{uB-Vm z-a?YnH@gzjkJmwiV(HuGYOZG(C|Xs-E6|yQB%IKMU;}n4$7)=;4ptt|=uxOO?$3l) zSezVcYUZ*#4^kz*#>0Iv9fAsenvFCz#%9u;RXQ(wH7{5gQk-JLS)>AN5oy$=z z7k5a@Q2PG{bvl;+0gO!Avmg629II-II~B7<2sh4CK4)s|3}d30Bho+O6(o#0Nq47| z*vjsCZLIX5DpuwH#`C6AlXF%w-%1*x#wW>!oek&e9$?(+FKCooD4=KI@|ljQ`L}Ec z*GSA7K`L?xKaj8!(d04uUjVt_wIZ$DR#(}+OPc<0$C_kb)JlD9{Z(r1D|E0vJ3hC; z3)M8|CT5~QNy9a7pc)^qptXxFyV3iBCxgyql+aDu`1+^&TeOT@Y#0Kk<%yYdMCNJq zInTm4#@G%4<#fJ^&_{yxDKwAbZL`woq_R}T#utFZIlYKG%x8maVfNNy1PTAC1!}31 zodJEXZ!`l%XS5hKsr6ei))$_Af-(jsZ8s6+t$Q9o1wp}A!#)`Q#)PzZw{xe^dQpTE zEzuJ`duAf&7+zQqj=3vUV+meg*o!+zQU^e2VEdBkJobP+;J-cDf9% z=L~m5S7~N#bsD?=@ONjkI-D~&O=`zaXWeR)fJEy}CjMLx?uBd?WTw+@M|S#|F3X#x zRlC21WwwIZ%b$I#aj^Tixp&T>L^FivSCXWNa9NL$tY$1?bfT(?cq>rV!E;;1%FTZ> z%89+p!R~$kF!b|@(Oa<6Q6-Eq8_{2POW*OS(N-uVXb49(iCBhKzGAZR=UPK#jog@ob1g; zT_W`Cu7yrsZ6_WzMTleKL>IY^rrdAfFffs0&g&s5G_YlNDE*l)!3?B&He!(AXPmtm zp*Q5%{JO=Nz**IF60=j%HT$3^gOdE$_@E%<*48dY zpW6EicZj}IOecSRmhl>ig^Q;PkymMwIcXvb)fNtWayqZz15*bXc|7RA!on%c`eF6% z6Hghy!)aCN5VkifnSW+{%K6iO(x_ebS!@_LOacgjy}mgp%m}p@+b35L-$=b zNsQd?irsYr&bE(KxbRh0k&&1FU~lQ-9m=X)Ob;zAWMdAg74mECwtn#8#iyjjk#bl2hvu*m6 zI9B&jVU62Y$bGGFOd?>@ufbH-D?7Q3b;L^u!8Fh_vcQS+^?Zp3xkvaymWanM0QTrx z13|#S`<}`Z%f#okHjspdY&4GF(UJ4ER@yE@oSsu>nFZZ?%q@NKD&=4`hFFNL#5g8N}EYS`Z#Y7Q?{bW*BP70N+AN zl+bl>Vvml!OW&YdP;WcCViYCxT{T>7af{_xCE#AD2X4becjL)1&V=AmogfH^YEw}jkLhaXN6bi|uSI-E{&h$>bd5J#%vF6Igq6pV;b(7%2Ts;Wh|etVrG6g-M>tvZ%wMo)RTJ2Ao915Qz`*7i zLJ5xhA@hKEMBud9DZfu(3Z?c%tOnf|p5*Bx6m>GbvmO9HxUQKS?PO6bqeJE8TfcE7 zbf|{Fz+@o@mjbs*DICw)$pllulg-c|hZpDI-3(*M9}ZgTu}*vTyTy^`RO)!|*%%+0 z0UZJ>Y9*I>6(!B}Km*-*G8BfhbJ2~N8@bS8&s^Ten!;5uWo794@fd0*EbR#8W84BU z1R|H5x!}F3NU^(L(8FJG25}lW^r_Lj7NzWTqhDN1Gng;}<)QE}2wfX>WqFm`Ug?C{ zt=j&yY!@)GAlK6H23;qSCs%P;;JWJx#XL;tu853`DLEH`qFyu{Y$u4qW5_P1W8B2h zTTMn|rUc^aPk2+pn!4(xo@lt^2t{#;(p-gUQX|!fpTVku7ge@uEH@cx7EnGM64DM{ zaP37V0W3<17PYjL%&iV7Z5F=?xQI1U)Z2azsLL*8d;5Aq+=DW-+fMe*K3G4ZSuy&9 z)$dPVPWYfLDRa6=CdK1NZ2FHv_ow>}kT(L-x0So6u75NAnRryQeI~H7$a+5}{cID0 z4zc_-I<(xo>U4wZTQB-8sHqK!)~a1%Wo?d&4Boz1StZ}_jC|1v_}&9=t7kG3KThR+ zU4)9B(vZOkQEYv!3HiT$S*z9=R2fJZ$sNrHAF-@0bHb!S#8XS|#GT|_!Uc;uKRbCE zhdTLGv+a`&O23k;EeA3u6?WHUZU4Z*Kf*IL(bvq3)EVgttuSP`vlbHl=WDk;*_v39 zj!=5>N|7Yjr!xW)&E{|_ozQY`;#^{S-~&+ zh1PIHrWpK71=)|NLLoUJ1CJ%R(f597}HO0pGIAq=kxd$~ooLCM=J5s5Q5HybwBIh@!M2l`%@{dn2vR1paa ztaSD)?-Oz`i7$+`nmMPB2<*R>H+0TU>kn=sl?qtNa4UycfO6Ms8z2-4ZLDMa5*I^J zqfi!l%78qg=$}&sa|wA{v3h`RkmbO!*F&~ zE+Sl1!pwOmlJ2g!fON%Uc)T4C&AiC&Fx9NqCNbQj?{1F`tC+TNJyi5%V;duRap(8Z zrWkW9W}CF1Vl6LcYpgG32kiP^8p82+_8bpyf{sxh+DF3qTz{NL#^x~K72Eje$##Uy z`7jy>wrTZ9GZ-14L*oglOD?pIVXUOdoL;{{E@T?xWJxMa4j5Y@=LtT?R{(TT1^=vC zbO_lBWOT|hlw6lbxb=hlAeAxP2)xS(8e33bWABknWR00Cx9)+>Rl%)H8E-A>O@Ph(MwlyFbuny+;KTg`DuLLw-~Q9646ruoUm^~C`>JDcUnEPn zAgoNnuI5kd>RHwHe9L|mtxB6oEzmFc@6-G>MEOD%UkmQsSQ>19iezLwMbOi@fG>~- zI!+0Su>Gs&RNB}`nG7neEP$>J0D7gaV@OWksR7XZlqGSj&cWO6WK|7N7I3Uim+I6< zjgIKzS^N?4>mA;re=l>JBvZ)aNtC^?ax`No6_=a-|a34L>-ph*mhXVE_EgT5ih z>LURgQq#MhFb}0(A!p$h=*sE$B}p0W@xu2aoCp5 zM6Cuh&1}Z=fF<`(TIaPoVOS7*;%RF^toj2<(WBSQ5ZaMr96hOV5Xm?);EYa4-*zj> z(8paUGtQ+jUJXP!vG1zxGB$sE5j_&Qyc7W=5fwqqZP}TiJYa;ryxz76;fxPL9hJ};skx`fdd~0h4qPpD&-aUSO`#gpF~nngkQ>?M#ol*rbf>l2 zzl_4qr&Pa*dwFVH$xJ3W7RX6o*qF>l(>^??lNmGf{>x8@JXITt{5OFLW#tJnC>l+O zFftri8dXpRS%<^NMk5v9u`-}KT~eI|NyUh5*cl8`X|ev4&Iu-ROys>^LqVI1%R?8> znwb?)Naf)R)W#zAckB-6h#A;*H&mpBE|fttu1vfvzM~dt?>M1}V3`T2ecc;N6i`9~ z%ofdGte4!{U@3+iFf!no_rEq{Q&XGL6xmXjpKqI8cEuDmA1e|I(sU>QX0`0g8zC!( z7A*<0Zk$%~+^dke$#yuv6j6vilG2RvgoPzOhCoy*bZu3a<&az?ti3((tVEcxWqTwi zzJrJAycI@cu~2xE7)wrJf<+{hu-nd4N?iPc?E8A36poZU_i*pNk#9wxQ9c---= zue2k|X%FpCx0M|Rn?RSdB9F>;52d?Tl!G-}SKrfr%u4{WM45%Z?spjpX?_RNW~I)l zYq820Bvzl3z)!JT!9=;Hs=z4)fzOfKb@;X;MhV$zS-V8aJRF<&U);X6&CG4DT1Et! z(e}zqd){K5P~lvV7Y$Tmx;PhDgVpX7H6A`w>+?ogr7EMR-d10j>RNlz4-Tl%93ISE zT3JU!QC$9=CR#M8XC81($>%HGIu=@X=)2L0cQav3v}LzDg5A^T4*NjbSXt7zCYuiR?VHiw(+RC_jg~SIR!B!AI4zh88p~g(Lt23uoyd*vt zj;jIE;vMehlYMHYc5u!RpFB?!OAOuMLPq`jueTJ=X<4qX%rWx zGi7Q>dl4y%8-^_YpN|$y0AGP;Ph>q|+p(4wM4<$8yiEVIp`0W;b;HTLT}3!oATUvN zHksG!@66~r;Slxg3b>wo2g&;a>^+r&?u=jp?q)JsB!aS&gZz02#=8;b;oyOwf)s}N|gz33x{_l+?%EGqKNtlF>c-dhADMEr8A?GPUID_?D z>>&4Y6}}?ej|$y|t-cd$S483dO9o(udvd-|7`wIwA)Bb^j{eq}G9q@sv86SJ_%rgK5kdca9p)_Vj*I7 z$tV?afD;!8RJ-FPZjuGM z!2J<$C9z+qo@uJo^=%1eXdo?*%Mx(I>r7q>$pg}|A8;T%Yzs87B||fmF)t29kipBc zQQ>J-jXh-KD7ifr>4sflrbTtaPDEoTnk%er+9`B`U4&KgjM@87CuJjbxb$+fd{d`U z#YxLqQeK75<$cRKa^~=U)QL0I`?m0$rN)){La_4r&8~S9pR(-q!ejZpV75reRZJEC zI$o^VxAx;nplh$JxlgtqSaVYVDzJAAP3m~MzRLQ9wg<K2c}-yA+s#p3-m(G*1SPcBNfl z=zR<5P1rz`3*1`pQAt*5kg?^iT!hBVLQT!Z-C)jF#=SW@$>_9NTRu!eO6JI0y6d6Z z?B7Fd0V(sUH8CA%6>nI#OvNl~eo&pYsK8o?eo`SBYvJR~2D^M$dhVlerAjpH13;aMvia_3H0@A*W!4{23ws-7!pHlaBLQoj zUHSe**HHWEQ=PBwN!9nStTN*-w+b(TS}B9O6FPtG%uTg%L#deyL4c&1<_EiWpAu~spO{=Moxvs%)X&Z=FJYx9l={8pjYPN=a!P(@;9=x$Fo zted}fn2#2x`@UsUb|s>pqX0yMMsZM0D~DXp>_!}%P5F*|#g#zb&c9vKkLFh98dD4( zn?ZMo>{`x^SDRbTDf)!fN{8Kk-an3yQkQxr>^{$_hf!yw7zIbYQL60o(R_9 zJf~goQlD>7amvNe!ng?=G~VJEKdq6+8&H7XyrO7RaSk(+ngd0|FYlsXcft>DO5O7* zw?6Pt%l4R3sz-;4pMxynVJBJizIY#WA=EfdiJC1PF9pJ!f~)pQ2stAK%gCfk4+!tW zqkF+6lAFk(^SUtM%cm9{i-+6SmZy(ee<)SA#PMe%ZN#IvVyIvuuylwsakD~^ru}p`y2&S_EbMfM<|>Q^FuiTegw4oHa-^g4CaYnG z;@(O6>)a%eri-(7Z_F_(<?hf)_Fzx6Rwj&1!DSc$UtymF~gM*{e z&P2S27C8w+7dGO2vv|NstD$3sxLu<=pMPGB;v*zI|4}-y2(dKXI^KLlA@4q(gthF) zKT4v_yh;m);_S&i-0mJ}6kz32qCXbIS5kThOt;UF%9=#tpNhSDJS}=Q25!AqVO@*% z=B=l~RJ?Wei7PoWIsKEXW~R&k%@3)WDm$&RB-*)-B%4R~P`TY5WTTaJ$ z>Z911_~NIizw7Fs9MpT+M)U{oT3R@mej|U+N3^B%FFI0e?wFoE>)HDyZ?bE79Xv%_ z^BG)%2UN+tu*C3StJ@O%tXOA`lOf;YWlBT!aR=wKN+!*x|so%Y8}&Ro`g2X?#E%u@3Yv~v=DK)YZAB&D7thye%^oCqHR zmL5pc^C~z~>_G-_Iy{M9k+KH4UJ}J$`-qXgP9e#mC@4y)`N=3!3|0(QW}ac1!cz*I zP_u6<@bnc|p$xX5NKA&3S^abX@gQpHBGbY8(@Mh_wO&X==qv{ts4V-eSI(2=-jv__ zE&66E2k8j(ZbY=O46;4leh4c{kkY=HL*{uVL=9#QzF^?q`=0>(<)f2~nCf>erR*d0 z@6qy}kzVLMZ4T%&J&jO6_PHoVxp8+iJmbUWyJZh6jDZUpBN%Iq!L}r&LM2I)9eCPI zM)PnGE)ZcdV@N}mX~p5eu~m$)i9r%15wdhlm8CLaz&n)o5Q0m{+>6nJEQ2QrB{CCz<>7eNsq%TW@dO=i4Y)bfDXiNAVs2ldPi@wkQh_cUr^!pO}PdNQI zGXFtkP~B^+2GkvX~Onpk%=*0wexG`}(+Rhf7m! z1lV%ip{M&*WGl0ZtE~yy4TmRV>GiytNeRBo$nG)OO~piC(}}pDg%bSy0 zccq2q7n;rkEp@TE%<~07N=CjP_>@7YknoXUU5%|Da8n|Uk)y%FZgA4afT$s$* zx7bh1dpg${pD!M)W4qJ&ijMWHn1ADs|DLx%sHaz~R8lT_o1WOyX+x#Ap95`7oZaI*3RzG(=~1vI3!g3RX!~9zP;(U-1S?0J&e=Efz?*@-h-p(6Fl=7(d9<3%LJPWex^Nhf&k21=U)?kRQh$hSB~K+c79&yRM?IP zC*u{~r2kAyz7KkPw<%Ok-Y{M{vpp|lr_sD%d^h$R-k}Z`Cj$Y!n7NgcvBU4vO5e#?#Msc*==Y>7 z0VBiz-E(ksBH(0T{~yA;($a9m=1lOp(e)qMO0(PRCL$wSPN3PKeM{I(IfCP#MIx=2 z3AaeNukZ1B5QtYHiKLT?3HMGrB?}TV229t*lgN{2>w_Ydw(GZeJqV7zB@e^cg;&-N z==sADD5CKSqpZ+p4ei@Jigqnd+_VfvrD~Q~Hw1YFl;Dvc=qwK@L=K8OOyMMpMH#MSU72^X z>j&CqM`#HBh(p^VFvF-1{g?&wYfu{e8mzwtr=Ap&4y6@fTyiK{bsi9nI>xGHC@mNj zMQckxc4gjv9#Z`fv@L84NLe11Gs?EPm4A-Ed6~crLx=zuyo|hmj`uLyDdv(9>?4i< z;j+NYoZj!J?Bx-9$w4+m01FIB1xX3*(fSICdX?-T_yMqhBrT%DhV_O5=KTECG04oL z*B^*PQ80TY{~T0@fdvmjQbCkV5s@ctQI`1u{%sqDL@~fs&t{@X18!96#2`R%|2xw0w%b~AUSE`24ii~XN?C~Ivy zc%@$6MOlUJSjEod`PhLecWDp3o{G0#G`60MWoO36!#UeShwJU<%U3p7t@yyiKSr3A zZ0#<8#33Zk65ke-bQy;f5dYyBWzwk8WECa$_&zP@e|lW@msJigXOYlA8&UmzqP|}A}k_go*d6c8ktGL<8&vs`mP%HPVXzaK{#VV9q)7<)cE3O4)eVT zbthUK2*4fP$>_y&L+%Y6*ppN5C3U=ey0ErK<2zH>9f-);h25!b?LY5s4W^qZpJ{DF zm2ebgkEExMJ}(Y??DdDOc$n~z7w>i*$RaMTE~UHuH)9FL=eA&|O^t4dcee4P1>cNI z`a@C!E#vNtUXJ=E0(~#LIp%latYTtq>HN`%C%oLgWP9Ix{C$PfD@Tjin99RpSea1R ziR01_@nppl9XYXm00m46Z7B*rclMe}^WrG$@)+mLXa&D2^5~y)vU9Th@gos{3Kc+#}<#cE3i@;|A1Y~wVXDcK*it9zN3n=Vv?w|U)P zKHump*j>XfJ49_TM{KP@d@Y}~aqM`~@pj;*Lz*7Mvv}Uvd_8#ZC+o}gJ(_UqYwdez zQ_Pxb&2MU_eNsFT+tMXaYD*KDMKE3{pP&6N_FCT@JPqL2CqC(Ra(xgORWa|GOm}S` z_m6FOz8ULfQEeulh*jGa>ymCntsHB-n7qSPik&b2WGSQW+Vbz0QN!xzv_JgDSNwby2mO|qSm z9ppgI&8SR0`>O)W9Zt&yuB?w#oqi zqUq>D1f1E5#0lkS#^9`V1#Acy(DSUJq_aRs;eZ@L`p$rgqX&9!z(y4?_2VP@349}r ztc?YO)aN>Zie<{jiwD`!t35@goI6)uOtHM}#=3kq3CPGqvhE?&O-AqR*aF>gj&`%Y z98)PSw&>T$#j1ef8L<6W%I=ATP@;(T7Rd`_k`epXBqJg8$AB1iA^O*}K)vjNAIFT% zAUP5Rm_QR_CB_Z-^(rBAScyzJ(Rt_0p?Tjy;?Z&hp>h#AFY8D#4||n#-PoJ-P86w& z+w8aiO7U_a8B~Pj0lUk@3-i#?d0NO17X>W<(4bM431?(4gwWXnU{f5u*>hrTMt1BK z?6|u#wPS6rP5Mzz*W3<|oFLA|l4|Yh+0$gnbtwUF6i3KV2SXfMQh#yTd|&t%bCjr= zn(RbDDXxxi?G#s2x>|XueYR0gi2975E^K&O2Sn02a>GsAm0!wmHThiFn_19rFIyl4 zB%>+Rm~-mme!sagp&Xf-TJQprL1^FIEOmLJ&0AmDcy2?0(v<)-l+_0}hrrev@nT6l zvW*Bao6yELh#UCbwUN5XV7-t&IX-_D<ei?An+)&4qALPYIt>P{{mbC0>G1J1cwXNnjSN0RX zgEb>AkextJ0hk7WT?}**iX)_1BqWvx^biQhf?lOw-d;!Oc?d`*|Fqeg>^R+^Pbw0E zN8H{+`%qT)1&6MswrZi+rkntxB|_qw$H4J<^5h}Dfre?8V$l60WW)k^5xOhNV>d*D ze)R)bjg}$Rf6zzGZt)2V-CR)hQRxNnO<5&mq}-hQ+TU#qufH!c(;4(}DT`V8U&h9nUb$rj3WyVEi;M-R zE;65TIxa?(nUVd;`%XI*EF(?xn?)d)ywgrIQN&cW>@BntFg6Sav{r1%Sg zYE*93#F3$!S}nuUwiVZPcNtpMjBrSfYK|b6Bp{1oHoecyShuY*mlf*1oPxyYT>Qvl zdVmEYuhc@I(~2%Bu(`y-BJQYRhUH2-`ChX4b}cM<9V&%atWj`TzNkMX?jAwt8;gp& z)!Wg>LuiyRp?EBqlyVP#BT))w3wuGw_dEVu|0>Ro=dO;9!NJ3j1j~Saxa*D(&KG=LC>_*DN!+ z%yfCT`S{3-U?Ar7Qsc}@;2Wy(To&3K^_pDV9{kCt*r}DnN46=H2mC*Y60T!d928fr zR5aC%Ah|bxmymzW5Te$)T56i#hG_(RiY67~ju$WGrh$CwKiassBYQnl4ZInu2yK=Q zP2^-{>88-DgJc^Rc%%oVZB}rD>$Jt9_00;+?7|kI`+Z8Z*hYI;Av#_WUPWnq9}sUQ zlw+e!0SqA*vJ$IxyePI)gs33}HD5Fv5_}evBdtk=O463^#tYt2f5V16`oczD_|Tv` zQiDHF-}#Sv@zo{2e9Q*CnnjED z5UW(w!kpIpK}?33cak|RWR`ewC+{;j)%FH@S6R>cP$N3fhWi4K_@Q1~xO(KZ9n;*^j68uM+Ak z5k?Q{;3x#m!sZn~r99E=l5vo;)umgyUE;q{u0&h*2mf~FcL z;Ji|*=76Hg{^pWQzg26-hjA#&O(THLX`*=P8M-4Q5>1Ur#Z)d4TuL`k;2Icnoqe-o z%&L;w;l>i@KVC}&%CvVXG3xTY-mbo9=0xKk>*QQFaA|1K<6`9FgJve+SVn7e&;y=4 z)LML|o81N{$Qqb;pdsM__#}88l!BHd*8PM%G%Foi zunqn-)QK{N=KvS2w6 z0aBv5Asks^95NEgyo}wI{u2b=hNe=^i;K)_f*q5u(S`0uN8fq)E;RtO7inlHQ_;bb? zrt~8*@`h39-^G97@DJy^qwkPz)SYSj+2Bz@aA2u>e$b3w8;kn@fV-a@?>=)9bEO24JKh|jzpMHUK2lB(QgU_0 z4pP#A-b3(vEMP#kw7MP3#ACkSAK?E z-iBOBzgq>~aC-tl)E#gqKK;%bMDHyimQ4fH;Q(v|If_Ihx97x#IO_Ck)ERBvLr$4R ztNwUK7g^zlDOryeWMbe041`~e4>;A;frroxtPS0^YLE;*{l(|gvuqap5`*F&8L+K+ zL9A#AHq7GhI%nvAyWIB_n(>kO_w(LZI4wb4W@Djud#oYH;EE7`Q_X(?EUK}_j?#V{ zttm^%isvb`XNE}La<#C_(mp|LYp!VuvH%%euO8c$SF^rIa z;g4Uw$V+@S7h$OlFK^M}|5C>XvCID1{74ZF3!mQ-9$L!nQX1U~Mcl(2UJn*4_Ek2z z#z#QjnN)rUgr8ZlPrHY>E<5pFUg=tDH2w=QH)$|HPXGSR)(b?(Y|QKtcloBP2giQ+ zrjHUa22+RLwY!(#G;*+%iv=A=PiSCfGRrBfI{vN$58Dw zyhXWh`?QNa58~&Z=01r{$V)&5z40~10Z()|&KNi1LS00c8)@@haal}qUh~&q;vyhL z+=B(qh|`Y{zCWM#d}Cf(pW6Y?+3B{rzm&TN?64K`S6naB+~q7TK$ z;e!}Er1himD!LB8yTcFQNE5W#qfc;uw>`;G)%{NxbzbWZR>k`Rc1X+Jr}BA+Jn55` z222g$^~%ZFOkE9Esrm0@0@)ChLb&v57{;cMa%~N@9h9uybZ6(!KPr6lC7w{7PjN${ z8?F3%iyQ!*JQ!Ws*YJFQt|XR00x+p%`(|t5Z%f9j_->sm!NE;J_{R%e@NW0uoD0{? z)Q+Q1Zk-JaOOZv{?sv=bBbH;sb;x$c+(sGG8>C+t^?($@FK7~kgvA%R*ajheEj=Vx zPxr4YCqnLv?HY-0D@)%S@kgEItD);{0jQmKQNfPk*}#o?Jal`mQoC&QSdAH$<#afA z&$r*2&b>e7lX~6%lOSVd_^$-n|CC_J5D(7ywSn||CN?fSTEzpkVk z$KRd)03*5_=@-As$VeH{9LcGzeiVvp04GBuW3yv$3W`e3wXA?Ojf^4Ot0};%sp{%% z2o)71IRRvJYw-Bu2nfs;#{iFf(GS!!r<18?DG+W^|v$q8`p{bRF$X7si!4nw`w0rZLkDb%w82W1EL z<`m=plr;NWA7?FQZ5{Iu?O^RcaF=fPP65sc?D>~8FggB=stbyW0vgyG!8!p3$q1O) z|DhwJtDONj^f$5Z6>!e{M`r;d(aFj7J5~5gpU{E^k&+OJ^(=C;>Ie*Mi!ha6j8 zzpUH&a$h%tBQUo%M?du!1p%U|z7FVJ=l;#jU_HanC`?bxZcgq5OB&cb2uw%8-RT52 za=UkUAN_FWVUz^*1E8m74~U6F?)zvhRw8hQ1o!@AUzvgE-6XgCULi*ZCzntCUANMX z59S{G^e;hMA#je`)n#{gHJJy|^mhQAko?KJ&VtyFn?gDRG5`hO2MEA3Q={os{%)e| z9k%Hmwx_*&c5`t6(MVt81oo|=4X~$&*qI@-69|Y(qMlp*v>)~-24TYs=xE89+)V7n zMG5gQoD8JF0r2@n>ScUyyZ9ZE|C}W(@#Zc#fg|9S58#{w3KSjhKeolR`2FQH{!NnJ zSe}s^AHtLP+=cztr$+X+=AQj+{lrJM-LtQ3{>7hJnORuD6L3bgMrW?_cYW4(;zUns zD?ta=+yLAqdx(>GeC2HP)?RCq9{qUD0W~l>Jo>@k-lWT14+@x>0l4w+0NnMu%df<;hPaPn73~vNpU;Zsgn#UFktYkp$z$`sGTbip70So2pcH&(5$_px`p`d@U*+7IY1#O62XZe_zS@cK^sFKXm$F;zcK#&asUGP;BzpB zmOK421Z!k1j89d0D*dlY7F2vENlrM1xbN}`(&xLIkoP3cZDWB%`aqfA%;mit6k`wA zAEe*`kuK#y4cB^p%_OQadRbMGc3E$fABcq#5i!RWjMiXg6m%bHZi$Py6Q@U)XwGux z&3;y-fEKa1l6NicyhHW=-lDxSgzOZyy3@gK^x3A?15S;DcN!wZX9PN5JAgM29|5T7 zwhPp)BK*eL_>s(H@P})Qp{1konRSLL`kej-aLBsL!Xg%Ra?nBxPY~su!!sHo)$zpp zG7cB$;$~#-nBi}9T>W@Ti3JYw_7-hQ>ITQ)#Edz0#}|*o4<4x;<@jV15?Kziyepeo zsbY8OZHvh)2=2dKrHI8!?;N`mtOw3=fy%BrQ~*AF8jwkNrx&`($fsMg;I?ns^tYrA z2~vXY2u4}M0eEg7Liyuo5>JHHq9N*Skg1$c-`3@a_~M`I_H-w_P=7yGfoEo6kU75Vvj@(2MfrZ7_Ilhc}A(PPy=vxsx^ifojav~kkO!73)7ef>fUl*HwYiEy%4?kQbAJAxEYVJ7kyDleyKtme5tyw;hY0KDBE{j16FQOF2uBS?EP0Fd`e z?g_yvlR2Z03S-DaZZ#A59>4~#V5CTgZ?|Gw+3IuZD-g5JjSV05NT%Wy@Z`euAIrr0 zq)j{xH(~VO1rn`WeC6&XwvTh4sp~y#%)dakNeHQ+F733oY6~KuWu>+1oraDMe0_yF zE8r}p0^|%DbhIwoO|~jT`dutqVBm?CwOnLRY+lH?MYyY$hXKaKFQpo9%+3p#mAIZ4V*rWtU=v1(v z(Xrr?lPR%8YW;&0Q7g-xQ<+9b$EK@RPJI^cdLYGhR%Psx%QxM+q~L#?Q*Sz9j>gKg z5a`uk_Al3uYkYxyT4Lvb_~vxZmH+Kb=IR$P&%K#k-l}6(FNr{mT(ionTY3n|qx3ha zW97P*%e>t&KUtgyE!noVR{KDFRra}=s)*!_$G8#wCV~?9YY{f!CB@Budaz8KLXD#t zB+y+#N|m=F5y!)jMylp!Elo6Jyz6&E^-_}qYcx)j44RHdDQ#0wGzOLWe*Ieb460w> zY1%P!DBK{1e&~uQiU-Xv-gt z&8Qi}<EHwiu%&4-C@9)^%LO?r)}rS@Wj1Y3PkKtSJAhAp;zZ zNwohlsYJZ6}md&CY5)DyAV zF?Y<6feo0~Vu+4FC|pLx;HoxlCzVB$2i8rkkBWM8woiN#=Y$bDP+J^=f-E$Z{-2W; zUM%yn<2m~$fz!Z*OzLoe{D7-RAIm8=c1_2lp|ltr)Hc|~5D9+~D2&9isN=!-%~Rx! z!ac!N%P`t+6R1crsV`TPOBeLpX>}+^We&u{rk;9dhz0 zq)&)m_UR6|ePbq@)YuXqnd@+VxWmvEf^{8hKz`|>qi$5W<}Y%i21y6HQMbq45f zx0x=@iQMKiF}MQ?h5ax>f^tmE_TkMaxVZhlwirXJ&aa%hvz;O zka3)awf@*_ukE?fdNFWE(ini3ip$Ri8)p-aZYS6(KahOu&I*Hm0ZuH+Q{wGPQwKFV z$a;qAd-m?8V*E-7B1RI%`d*43_!U|+>gBd~Ni9r1nM-uBTJ%He?4{{WvZrH6{`gm> z?qUGUF9`TMTo9I3;S+zm#@lj$^N2qtUdX43)B(vm5+x?zAu*SN9iuglyWa?le@S%NWPVZi~j*iB<; zim%ArU?t3Ww~dM7n7d;Hoh8Gv={Dyf^2@zgk#L7VHVVjF5QDb&KB|GVCkVABM_Is1 z5;~JcA*-Z2ej888LB><6r@pdG(eh!NeL+-0%iGHORkol7w6$@0GABHNm`wAB@6&8*&7f(SB%TllSDN}$TE0ux2Amyg=uJCdK~+h5@!r?x~&zx zx@c36wo(;~kp%h?MUaK|Gt`_an_Ev!N;=Wxr5eKDkW4aaUYqi4xZ7^|7v!2E-OaS7 z3;j!S+~rQI*cZ?I2o~ArQoKxjC4s&J#CHw3W)j8cW2PD`4PFvGHz$o2%5wd)L~=$Z z(1G`ioG03ODk=rLDzE{^Y0?2(vflW#>sD9;htf%zhC=>oQaxHDb?F!us=?cS+_YP; z3#ySYw_{WKg{{b&A=Qhqcay1EcIzgAg~@kcmQrGVDcdm;Fy+{*fs$tRaI~x!&Amxl z=0g46(TvpreaBX*yT!7#khv6D{Dei~2JB2P-D@-+LW4A5cwa~nd~2!IU$K?vVLPWD zJZ`JO8QP8aerG1uKS1Um%M#JnzY^u4HnYHk=NY4QH@wj-;3GB?cZhCUfAx7>VQTy6 zFYvtPpP()H=SVjr)a98PHJqoLb!jiYI`k-4x@$ALP_*e7byTx0qmM^l1N8ju`(cai zh`YK)=?e*5j|7q{z}?*Q&0_8m2Yqd}+baPx_hw72R9kJAR(y-cxh`x#ce;lSYT@LZ z0a0{m^xLJs>GoNBbe(eHyg7YzJ-NLW!s~&MyOX{#{P5^aBnP?a`ocyH z?lwX1-v^C%_PZQU23vekks^85`rTv!M)`nD0h93uHF3QeBl@MhW!mE*f;=B5pCzyets%VA~VfL<57RBEJ`E{yOdtEhC?~J<<)w|_+?&WFs$#y7f+*#fkwY*&lZYU9-n~G5M z=^!5?pk))CT!6fu;PJ_epsb+@UacoP5W-b7AJ)2P=0I_YDShkLB3Z0gCcSz?euszp zdLjhlT=2$pe%M@pn`_34Q7m-hjl9=}@!hSdd%J0hMA5+XTM{8Hr&2+=MsS%T={}+i ztqiwGb1$C)iXL=FHpU;GG%*D#=7os|dfczQtvd>hT!>_n%GaXO@@oIwC@E4qJ#7M+ zcT%@9&T~)`NZ%@XJCdgrt&t?>_>q||n_r_0SIR|aD6z?XLx6izw3hW?lEtsnCidFd zWtWvla$&#BRL10#t`p0H6P%jz9LyWu-eMTN7$yGWv#|$o??OpshWFa=S0BwJTa_`} ziFW_S?v2LG0PV~0v%4n;TEI3Q2X;D3OKt6>;(@0T@xpm~`oV^VH@lwNMK$5UGBsN| zYq)U>0I@y`b3;Ft7l6<5~`Tb-sCl%?Px)s{XJH>}MeOvF%Pi)`gMk9u^6(w>twegTf;WN;WnI*#Y zF6@QVGbX!8DXpcUs6GrhM9LFqN7?KX`RAvTm!HpV4L=REYW-{0O zqIPOzDCl5r3BJR2Aq2{8F0k!VVcvk})-ge9?1`Ore{D;OdxarNN{40xX0LI&<16UB zH(n0CPDmnE)qliqLlf-eSpUk%Ak|7-jPxZitzK2Eo|9F!H&M1^{#ys5%Dept4K?*K zn+||0AFRtV?=By@STHMg;7nBpbuiY+1(Fg8_duB*%}b{&LN0v1U( z09-Km8OVYUtPfa7J_HBLJ{N=N$dA7_lowhL)8|g8Pt8 zk&y>@=CTMCkyE&mib-$x?Uz=Mj>srE?Gz0{d=>ZNXBdgi%!*Vby?nxJ zLSIX*vR9HU=iq8`8T>U5l8(yK8?8ua&%~No$pH&oW2)&m$_8$d`d|_)LSh`s9kty9 zikW0tyNRy9#;Tm?#HLvU7BaW-w4X z47T4myjd0%K8`v?J*9BZ5;baS=0pDzUQ}aL$t|JTfS&%jkECPjmG&@HEYYt{U$hh# zmWU&Il;lWNG)~66Gl&%?QRfC~zv(4VF^KX~xVxkk;%(ZXJmu~6pYUj1?PeE#Ulxz8 z@2WgdEIqyHnAc(hq~?&9+ic*cb7EXA>r{cxYMohJ0iR4#XbUtnxW6w= z_z}*~@_*;8f>tmjRhH0I?xWs)x~tZb)hpACotDuKR<<2;(r_h+U)s{*&fzMY)~1D& zTW_R5jMG9Po%H1#8zcs{ZmfyiuNbOP0~Q+(JdBa%VK}GTH-4s8ZgtSgu&d{3Pn`I& z(v@xoZQjT=LJpz+ApBElANnUXB&G=bgIVS}NyakDpW*EWUHMClRL#sZkUy=NzbFIG z@_7u5b>N6yThY79N)B7}%rK1GAXTw`^Gwx4jyD zv+_W+Op6^(V|J zMtgaJ^)J_buu*iN5mu&RstrA4b)JdIAW67jv(uyRKB^aLL=+UsRk$SleS2X8l44@y z@bfx3-Gi&BOdjQ^2|d-z>(bz(N?+(2*!Mxg7>ir!M`MGWe4`{Z3M6nM-t)w_yAKMxiup)Yn_W+v*H{@Q82Bm+5||&WYRX1ssrf&_SZC1>P&)v zTHlMots?V{2T3{pn=)<>qe$c8Vbz4GSnAG9@@c3# zH=SUSe)_6O8J;4jDKfCGsw8S;DWz2%19=tYvCLG4Q!r5#*Ee`lZYqgUub!;qi;$dr zsvu=oibf@54Yz3f13{ZwF)8$kcpfkf7I66t*wul5$FQwbj9&bYuF*v-B{Z?baK)iL z8P|5&R`snv^-Dh8mz*W5RPwt@p}4|rR9L<4iy!OuLAp#0f#u$}V%G_!@&;40db)5< z!jtB_dWVs2muQ}RGE7upWf$z}3B)p<0GjW*JikWu^h+D6ix&{2~{ zK@yE_$GQMap3hP`XO{#j3BoCS5ZwZwSLP9~Ll^6>{ESaFE4W?Z{>!*kM;cP~^k*^P zh?>LD6&Fs*B<1$xyv^e#kD5)bI~U5${P#ujDR&vUvvp2Y`qp%m7za?Wp2wlbkDT_y zX|wrt1}+ePW(3UgjnciWxX+w!qrxq+BTWvhF>Q4{JGaApHTEQHsz`nZA6jBdo_hG7OQPNF(I^You+hBG)q6;mC+&;cSe>O2{g%*qx#rdlmnrj1s61paq|2jN$FP?iqVSN)=ga81#_BLn^Ds=70u_Uf!}Y z!AmS|2MuFjd&tzltpbAFjpW@ewG$hE))kvHfm|4@6Ps0O!KP5ZKY;-a(s*aL#rh#C zYp1fDH9U?~EUl?SP}sRj=Bft=Xg_`5)SX7Ze6G9(-3<%agL0Mcj3~L4iwMYV+<(~n zT>*XQ+FlgKncujnOfnf?jMe3T>7Oyi7@$_2&3_hf>Xm+G6T%;Oymc0L)>{xH?i=#p zAPjpKW+ml>jW2eeXmb1S=(BVk5qskbSMl@UQbl`WeN9{Cx~}SaJ-w`!!)zxYfLEjR zDV$Ix+pSSHl6Ppr%yaWGVH5gG^@R$sd3>9PJ7xhkS7M}3k2R3ZVO^gx)kL-QZ@>si zi+jOI7ykH&c{*PH5)rt|e1{|LV{H=n1mZBLg+!L+Ji4lD90x!DP^B0$*(ioN^8=Y! zbfc?5vdm_X{4ArP>?l+e>ZFQD5;vkdLDSO?L1@JZMa%#y2VRh;Wy~2Y4;|)6wS(wf z5Y$V$x^M)B8-4~*WyUrfyV<%Poemy+X8lpQD6wH`k6Ha-yp!zJn%XXK?bJ$j*uLY# zI9mgiYYYS3NeNwXz3HSQT`8TM%@jF77Oqr{?MjNVa7K*OW*^Ce3-;OUl3*eC3Y^63 z`ciCjorK!?cAk*zr2{sl(Ey=E)G;6IX7~e}+lb#SS3xY4MWlxhtVq7|>Kd9K;M@%} zm{BlBmZPS17I~Gj zB#8}LImO2r8{9JglVb=!#Y60)sxIDHbE?;zI(J%IeBSC*N?u9aB7-Rh{&fvkC}IZI z`R7qSP;GFsri_0G@0N41cO)9Zv&D)vkIFMgpT3j8_pN8N63koJ?zbAeK-06k)#YT5 zHhA)=MlDASns97J#vp6W9~}^lY7;olMW8b5owjcZij6^g=iTW?Sevp~iG}-#(!1${ zO}W*&kLK$B1w9qZHP-Ck{0aoT@hg$cA6!Q+wRctaWyFt7-os4vi5DBh5U(A(xj#n3 z9HgeG(jxTugF(59)VK5N-w6nWItIv0B-uQL+;Lu6FQdr#I`9kYFcum5Sac}`tM%AY z!SN$EP^ZzDGOI$se>)(@g^IK{q)xg%@P&j`rjV+biDmYm#pjY*clfNT+FlFZ|8yPl zNMv-pbF;rH4^_NeK`Q07-kD{$k*%JXkG72iVNuuJS=m86V(8|nE(vef)qr=T2XKn2 z#}Wyjr{Cc#P9d}$sNWefgd{eS!)tV_sxQ-cOEkS+7vUl#-U&sT-EvmcPLpyt-UkXE z9~aOMwR%DXP3n08UC|>3gA(yjC>wtF4txp|7(Ez%l&-eCC>c$*EN#NaZB2O>HexTBL2288neWT zxsPAFYLI!5jFDtdf!XU@k-|jYS+?Ez7Up7O!M3aYOvE41=4+MXWLSG0{UP4=OCra(E?f^OrM*6iHajWE@^oWXl9_&O3(A~% z>re1JFpA^!N+{JiCW$Pb%sJKV86pxIj(+P!gVD%>aAfn0>6%y?-cDtu_s}w~vwRw1%G!RJ#p&P$Xu@VtMRC{&wT2Vn4F+#AbG680<53{dQ>-eH z`qb)0;FZVDlDC9awGx*-UAbYKH~!wrpEMtS?kZwk6hkeG5^sZi_~Jr0Q8+cFptLy& z^LK5du+HSBM`$0O<880)k2_5N+`u}US<|g?Fy}RDv`)7T^8DO)zaOZmZGDLEx*~jd zuF!C47oA9Vm9?fb#r`iAv2aCk549p~yF?3hJRhbVu)>_3&^6&~JZGrzp?yh%6$&YS zR@9On{OX_!R$~pzvF2i~tt3Ep&a10ZL$7l3{{TDdhj39HvHpjCG^II^#4fnXR76$O z-L3iq01e7aRpNi{P1v$0-iZUPY=)^!n(uMt#PgU=AIRZ)4^E1;e9%SjEz_p{c{CZm zJvrIa`?RtcvSoyW)Iu&4F)oD$Et-_5A*Y|^juO$5Y}8#dYF@=~0?$f=U>`$S%I{Pp z@9iG<1a{H!Xd3o;`kHSJRj(bY>){ z-?Teg;+5*B!)+n*#kMl19Su~@Wg z9cE%deG;v9LtI4c0Am=nB)uPCC%Fp|H*Q!&GZxr@7#E_m@q+P93x;ME%<0#2KLSjW z7Eh39wwWW++D-;amshNw?C|iIbUzY0=}Ba|p^XDZHV=4J7@D3R#@+7lO$u_s?3&gbmZF5&Ajg~$1i)Q{=P@ANMzH3AFE^@tiNnYLl6o@* zwyw+*$0AV9YjGegrq|z&R_4#N>mFugo3uj8ylP4IpAeG^(nfwWzLPMDCf3A;$rlvc zWnz#7sA@N>+>OTU^Fl;z)`%$fKe>~nM!joLf?laMEtO_RN}7m6(q0`h!hXmYuoWdY z>OUG>cBCpf#hI-Mm;ltt{*sAi%{Yyt&z?a!FMW62+N`;oU|%lQC|=m3dzFgAo=p}{ z!f*2@ku87H;@8m236h2VtsmCDjD*oLP&|jvcXE$Ne?ZN!%UxUny=TrK%OHtTSP$zy z!EU*6=}570G{lmstW=-oiLTvkpS~>T`MbIhAE{$>H0aJ6U(+jmTCi(pB^<^7WjRT` zeqg-fwmy^-o zOPGD&&8CXL?U1da-4K12a!F_Sxgcb7-`}^L5w*r+rkTlZNPnC~5mERy!KP9hqYXGQ z(i3QnWW*MZp@PNwV|N9I`P;l;RCt!!RsP;1YY64}8r2q_-hBD@B4G{jUkL0yu3FH% z@uu7{EOmt9#z1!4_$t@l&_jH1Ww*mmxVD?h{uBfzeG+V9+BQiZ-msjCSAw^h1?3F5F1M&f>Fp-q zu?I=aMGH za&wipga=AT`jR0t=b!CrtUc)nQ)lb|_kf#fYzL3UK> zPZ7jbfI&sU!b(S{-F8p&r#y%TJ|Q4>?kPl-7sZK56&A$S57ph|D60dPDJ{llePI~g zCExXSlX--W!J>+of{in&43FDm>}eJmNO&I+IQc_YULhJxJou(|T7J1{nlZd=nj5wZ z_6a2bmBN3NsPBiIIqFU-#`Cv z(`YMAsiC}cZljfYmHVM)@`a>sk(*NxM7yF^zI0MTI5F|LMo~W@Ai1Mqj8TF+ULsXR!Jd^OPEEP{+frN z;@6RO!Y!EL32$uZDNGcOIEQ$6t{|yA*yRZV*fat#Bw#cv8A-Xta59m<8q}WX*Un&dd=ZZfK?zcCJ%3(RTaZu1q|F-6CPS_6; zy0JscDU1)zC3iA~%U|F#NLh?C04O{YV%z7ew*cuAYRA>BI`^2)6K^snJ&5Nbg#;1c z7U0AV#0C5?K#2cc3^86)iofICY?5M4Zu#B1>Z|FSlc7^SQj4vhB;kTwy>-qZ@{FaJ z78_AkbMP-@Ra7+FLFWz+bmOmLD{^y&3qzIZ_Lb8$Ss=L3_w+NJnP9GN!q87%q*g9^ z2zN8BVQ)Xpql~tSDiKv`n`Yfev9P1Q_aABJO4)D;7o&B>2Y3(W@3~y|u+f(N!EqkZ z5o*~M`)%s}gbJL@nQC)>g=sx~M*{413C*?)dspX0n(zBL1boDdm|hls;9uQ^L448~i#5BpVS z*{)TtnhMYgg71D@gt3}=D>JQ$Drdj7J-62g}> z=NfwB!9%~_H)j7K{4KL+HI-J_t*NaYpNfqN+o=#mXw%{d0+B2)cORXBU#gW?s+h- zj#)?!0X^!un_tV^%Y1p*%5u3y?&M6pO z4k_Z>N-GI6*P+5Yjf0v11nsMDZ;p?kw30i%R|-E!meZ7x3=b{w34(jlLeN+Do%nAM zBh0C`*EU)f+QrdwiJwLip+lisy`W{VVo z{aEj@`89-OEW@%Kj}q%g@7P)Ye~=ot3u__wesbm4w=uYMob_Z3vTk38a#0kc_Q{?g51NUn(=Cr;VC%u!bY=WrBM{<4=n7^2>0n< zE3BY_Ay&wvRtC#?n;lW@zh!^J5X2uiXF4U}8$z~pE%dXv6ppml%#iA|AE|zx<09BT z$%^l2sXFu&&%E`V2H;42Pt=%No3DD>J@HaKt2>G8X;gKQSkvV7BHwxo>l0>{K8j3@ zsOj8|L9E3nq1H51ybvr?J=|O6?$e-(l@b40s8=`mu_x5AllA8{0hcohl*6)y6uwAG zDhE2J4q?e?BFn!tC$dQaz&#f~3;!8}ZwgOF|zvSk{d3o1zO z{#a)9ht(^m>|SKTcZt7#4}4e3dXiX{0Fhkw?HE-5-LVa6R~(G>#kWfMrM=)$>z$p1 zvv!4k8`CU_ABdO^py(CLrh-&ku$CiZ-Yi#hQtXzmZx6}KOiSsO#>U4xg=l!*S*qX7 zfx&-8mz}#%^x>?lFK&(+y!(VX0LZI`fRi(W4P&^%lUh~BQpQ*F=VU+?(HmcZq?=d$u=$z=L>*5|Xuk)MUZyWWJVZ+bq z!BPj^cAciArsdEoOKWvpH`MbO^g3jT)oD4OT4mf%C25(B1&Tj8oZthu*mkq~%Ohs= zi#blz8ROv~6pyD9k8Q@i@`r?p$3nRK?~y^DZ2ixTRmoMz=D&5W(qfdS(9I%_hZ;e%^6qd z;Yrf3h*I(wc4FwGsnJt4jSWi?N{`Is8RKcg4vavz98!Z3`D)HuRb`)+M-A&ebaZgk zqAU;x#jH!=8RtvLAEA2s)h1t(SW<_1M%W5eiIc%A^~leR1bqb=CHIyf=xl^^{wlv~ zIyk~H5bo5}(ku=?URPt{Y-k=T$=5}fjw_L|`1E!P9W%gzohuM0)G}5IIq17$t-L6L z@E#72B>LbxRvfA-Q6Qobl~u?OjuP=SP%uemIWYJaY!rC}2#=5!8+K1S?h}jHOErdX zwdrg5%Gi*?)wrrX`MkQzK{|mF|<@hVxU%@SHKo{j}67%5ULT6 zha0Sg%{i5I2ZT}}9l~Fw^u4G!_;L7_^23h_vq~-5QPI0orezfnOeeNYdZDq=vWOqe@kph zdk(w#Bwk1USS2_&Owy4={U|vY?k@?Kr<(%DG`+>7AF;=!QIc;W!4kvbR(PKA-LRUsa9vB3Jh4)`3rgHQ##Y zkhsakLk1)Y*UbsA{AuYr;VJN<*##!WjWNr`(3E;^@l2Qrr%@pbUk874DwsVc@}LxZ zFQHanciO{YM;G4~V9KUU|bMjH;L@ zeGf1fxX*hg$uGLj(D#b93P{vPq=__=!Sv${r)V|VF!9I7x6!8H-h&=jbsOEJCq%FL zgoAGTp&!6I-L>0_^D1RCWS~N0)PvDgW%)aW&}L2Qe5Y8I)`BrWd6{YdnXL4y5jl!e zl;Japh?0I&#t+C_3<(&R<};96l5G?+3n}C0)g4O5u+s93y{=uA5iEZ`dtqGiB;vTN zAJVUP2DpeXsvW=dGIgn2wnp2l$eF9nlv+(xx=Y{6zNeY9g{glRR*3t$eB9wALHV2s z@7!7qAS;bV#8xOuKPdW$>dc9n8VGd8M_SCM(zty|w(onLQmnDbbPeW;%8*QZ%QrS+ z@#W&ABDKmI#ViPu>KG7>YrQe0)D2387$l!6yOdN3PECS;=`6tw4?itxW4{NT5{YW>L7q7XM)<{KPmnc8H0w<<% zj;fO~s2Ay^mM@Q4ACh|7{$)c zZt{Div<{?R#S|^iKKu2h%7N!3^{jvOUd$Lu?|%Bcb0omF(;c-}ML%0+S7izhJ{l#ZF*5WTha!P*S)RLdP?8 zT9)w_&eQIB@X-9Ak$a}&iGYzNx(-@43fa`QpTZdc-d0z_&LfF5(9aGpC8X=>j&b5C&&p*+uqX5325WA#K_?4-AS1(`=(7QAz8dG{`JBPZ+*19x^TKe!+wBUS zJ>!_x!>?udOK!ss#Z!8JE}XS!86grWz68a0b1J1WMsVj}`=g()*qF80Q`0r(PR>_C zKIIr{jUDlP(pT?;R{6qspH3WQQ3MR-Tx^xNTR>D#f3>aKt z%C7hpwmGB@AIizV*Y<*GcSPLk^VCgc(QluZe4GwUrrsej8z+JVJpIZ%F*Klx-ynie z*PSbn{0!cOe+idaRfzyo7R@?|``nz6vgNn_x>F$X@e}%@`m$a;-B)A;tUVJY=otqz z%2fYJ{s8FgjoNlI=?ll*kte@Mdc6jxvLSzdwYodNP~Bb!3U$^$QEfTx<9Kf`R2@OV zANgtG^}~Y8>^h26*i_*nX18Ij&lSO$nSj^#^0v-lHBJ-YR8w3=$9RALr$y zk}Q=sO{fy-pAYKevPUAHC+{$CUR34u09Rsvuf*FJw~7?27~;%+LtPLR=$JZZ=3)S zyV9&Fn?o?@(z zoXReP(+;C4zE$=?L{fWjPC zeB&c!Hpf#w?xWs8wsjnDUsV6#b4Q$y>YBW)(1A;5vM>4KE4_*}3&XKAP>CKYSJ7_5 z(QXB`j^pU7r zOy}$GTOM8CoK#|m7X;St#OHT1E97uv-UEj;Nu>)vE@NC-YW5OtN@H8KF!(_ z?~ftlUE}7Av+XaRY-IrlYL?{(DlO0YnlHv<0AymS`dl%m;+Kq(5Ua`{BA_CrG<0 zr?RNsLj+MrXqzfhssaNJyngw<)*+bHU%l-)9-vwB7Yk)y2U}W3xX2fQ6!(K!)l7Z? z{2Ygd%~gl4xj?2MSK#%0b)X1W-!a;KiPytOWSck8xP@p0y-hmb`h={qbkZ-Sh{$XO z*odOI=~*DB{o3ceC{J`T6q9`6ctK)K>&n`~x(vY_Dwuu>P1(TBi6y&*kiXa;HiWbA z7r&r{bUX8YaW4MV;*ur9eGef(sirvIx4pg7(A?cAtO>W{O={pG(8_ymHu+u-PxsEK z`B2&1)HhJuYv1&}<#v=4!9#25kL&MFC$xivto>S1vG2(*eT81OhG&yUR#}HLD+^9v&dp)q4O)?Q{SDM@986ov>etFN$XQy(^J8!L(Zfns#s`E(U)37 zZV{IJo`n+1K6xPzrN~SczWC?4N=`zLmT3<0XQM`C75uiq2!LT7@)sb$zAZQxx8_Oz zc@W2CPi>xP$lq2d;1l7EWcWk7kWZ@6uz~}jUL7u0M)i&_lE_cmG2+G(Iah|@JPz|^ zareemQTX%{8SCkDFw@!pP`o@%2OF|YFBTEnz}@14w7n4IFgTrhHNsyM^GNMppo&H-> z_OYR9TudZC+g7Ktk|cc9%K0r!Xb9-owsY^okluVDn9Dp~90mHJ!8?t7hcFpV8?6Zd zWi1h!=QiuWCF6lT5n8<%HHBFDh$>0$rcQ~xhJAA?S@EBywpZz_m9z9pXnNn-*6lsH z!F}++4q{%2Rm`Xd42Dm18Jr3tJ9#N5v}#4`*NO=Gv6J&-E_GGgvEGi&Y7lLJlEH9a zC;$&_cGL;8mVSI?2?M4`T`?z;IYwn!#XIiKmyW&21k^IM!9Mjw!I*Q^Rg>8A`S#X3 zZ`f!}mB;qgFS1+5&G@dfI&3|d4x^rDWE<^}c&w{xAxWzQPO}9d9$8wno)-i@?FUC5 zn|7ybQfU{V^^7ah&Px@E>(u1+bk~l1rv8x zl{(fRA%@O-$&kIB(HKK>psWOqme^@1YA?cM4ARJ8DQyiZe+=OB4$q9{EIcRxf`|Uv zByDFCDZ*87?>>d3Kfv3am%sx&V=;LOrR4AzoCrFNbOe9wTHJF~*|0leV&;V$t)&i^ z5g`s4Y=H(vpPG#C^zbSH)P|B*eOHoQ5eRL2@G_wxoijF(Em?ofb1W@!+X;CHg=#LK zEvP2&0y26d2EzHj?^0%aCSiZ+@k3s@qljfrKltfdYAT}n{-~q%u_}tXvS2isU{$5o z=!ncmo*-F9OF0LX##LCjhmioy-19o)%NypBnCP{O%Vz?uBWsB0OO?@S_XEFhRrLTnl1LVJbSoYXmV0Kl)IsxAN#^Rbd`)zM9WH+}d>_ z1ML92A`$9bI@Sh7f3^ZHg(;sgHN_~?i_e8KjZ7*BtsQuF`4u9VBsjI;_5026%HQ1X zjODBPSsN@iwbXqKQE@k8>2EhM`tbPIll$?Y*k<`(|Ejy+3f2t4^)ZSsY`gxu{J}Mi zHPs&!q|%p*3{hO_E8{0Tr1CnE)p&T^b`C$<*-)~szW$A*M~K}VEuH(;c5CE2vi_`9 z&W37{7Qcc1CGMtbgBFKWz6eC4{j^Tk5{;LYVe{2r|69n!GSIq$Mtr2(e~xV{ZBHHx zt0TP?*Hh@J%~3=5^UDtRYe-&{O@6`b6ya5`z%C{3EZA{)Y+xt zh5j=hdcJL8h$}JCY=QZ13VQC8;zyRzCXC%#p&JV7_QyYnXqV93WGO~G4($g$FVAxo zE$ewqrKW;^ELR?wgy|2@pI$RwAXj^@M;E(sAA(b0IsAp}y+CDoDO=yK`bRR_zpL%5 z)G#ag_RJyt)OjEf=#a#;oA}QRKqRS3dN~uM$>rf;3p&tQwy*To8s{nNJ`f%jd?8x% z&k9Da%mSurl{=&|`EvJ-3ZNBy#RVKsbu(X-_uY-S%3h+Zw8!O0<^4d(f?88C>!aJs z!t%TtXh9`6&&$|*RKpO(orbd{ZvZOf{2hsj6@i`C4&%SYuvpqJ92W9iV;p3wy*od} zrs*DK&#zv%5FB`Y*RI3uyIXTo=;<|b(!^rEg{2!1sOP*e{j;-DfbTud0e=V$CKlMw ztxHsgiAz^+za{DMeZ^;Mxtb&`&7FBCjQ)6rEx?ZJoPYYLwgi~3EW@yFj*1Z=*LR{K z;ggY297GjYbe85dJ54Me^(`rrX>rWIE+T6EztQ9+a0h{mXniWgk|l z;T5^g6Zn%Fbe77?&5ay87gPL@9Yct`mGYu`GA>W%%zT7U-iyH;2W^F8MbvriBsuxQ zj@P=Ax-YW7%kTQJYIta)?NwuA*Nnf9-6YWSR3oRGB%{0mJF6+LUuI~Wr$8iigMDsn z#x^mpgkmB>v^o~JYwFb75Xs%hc4A53;3r3IPg-9(4J-8$CUFQ*4Ohj_N2{oKJ^hYk zI*Ow#jA)i)<*9Mg9X2#@U7h46h9lEY_+=|xexjbh>OxYbN-5g9GV&|DTKFm+0y%VX z|F;b$6P%{|1ja16a3bq@#?2vJVYp5oJOLAHZ((5Bb#tYZzp2k@j$y9S@}^?bamPJ| z@B(c?d}XmnMeDjiTi#1hinH#f2u`EfM;N4M-=JImsNpcc_)jqyO=$!xf32L~ANPP>@!!|BR&TmRai8m`DWcoZ$ zdF5wfO@n*)L2t=lY|H()l0#nqQP2m)CwxnTQxkLRWeD%Fj>szJ!q~kW``HsU9gj2) zVMQgniAlP4He9H^h41?fn{^ecM9?@wWpwwYN1*3?iUHc)2ipMj{ubGO^4^Z!lTCf( zQ%x)l;WA2GH*5g2Y2Y^tuy8`aIf9=Jeqp8D|K#Y^E2*3m>?h8z2ZrZ2OS>AUxdZFI zrEH$x5|+c_rVjtvn1*fsvAIH;m0#thw$1`8t4Y?W7I=k3{6J5y@+_Q!sLQ`t_qXsy zwI8b3=pj{x8@csZy7MlM`0e4dtW}^3x*PLn`pr_qzrBq=>Zi*(ZrhEExL-yE%cGJ- zOu0X1``K4lh4r}PsTww|tb{1FVDZL_+N0Mwxx2tv2Y#!F|L2a5=Rt{x(XY_%_j_!o zZ6`%Q^SRcsz6)b>zx>;yl^ue#idB2fDpe{Y&FgXYFW8@{V+BjkapEmq259m}l6I*m z+z`3eRET4GFef>h^Tt$a`Q8>t!5n2(brjpYn7k3Cu05>WqVm@>-l}Ho5+HsHB1vKB zFW%s7><4-Oxis}wA_aB6jzZf&Q|QtjBKj-E;TSl(GXVm)?-Kv0I-7o3k0xncli6sx zcE3f}njd=voey;etxxQr*)Lbp9MVuI29bFpC>+$k;o@D*b=V-!gL&B83&Lr=?tL_j zp=zIl>au^tp=r4kWq>=oKbyzvI{(tUR6R06zSVf{NUF@$@B0%y@st0&iMI6U()#k> zh-yw z25+p+HY3X}S%(P-v^DMaZEru`R)XSH4}*eg+G`K3CKwT95Q1Ke{P8Wh6l%x8)mO`z zqPuNvsUzc#(muH{npywOH5P^wp0|Z{hb<$mV)5PQ*hEnGrFNI|FgDuIU&Sx+TZ{Y##+$!^M zW~bKSde0ZTJlMmokyLeuyCK3CF~S3w4#d)|C@@aZMR*k10!-1y$A(8zGT4BncS4%c z;^)GB(MYarm1|TN_JO%<-(!RvPU`5(Do-VkkE9)4Qj9D~Ky%R&mT{6s2XmPue0g(< znb8ysu$|sb+oFk?)sC8)0=|Ir%{@(|m$jJ_alx3!mbul3Yep94Gy(48>CPd8bhjsN z&Q<c^wc+jl;%`703nevc-hU-Giws9)H0 zx|b*5p8L(V=YA%y#nL2T%qT`rRto2kdFLK)e8eF+c&&vCX?;uD!1q~7&tMF(OWdz$ zdueUs(q(}TMToao-IKRmz)Feh`YRLIW4B5h8O8Hfx zE^w$s18Nzt`{i7zqjDW>w{tOYBhglQ=5$=n3RS5uc6pM`9v4R55wW>>3W_C;Nsmud z8Qouc2Hn7%kLjkl4&8k{nmOM3-7|ZQa}XNRVCvVO$=vDk@s$l&&vJ}Uk~Fbkn_J6| zcsfK5}}0*_pBmwFgd4T1}HBI}Q8Hwmi!f-|M-Y@i-Fa>#LtR-AT!s0%UZq+)#P061+R zcd))UjTzB&YVMnGc#AoAn-*aKo9Ab%KHwqa*!0zfp^kH^WOHDjks`JFMHAUAis#xe z^AkP4b&n>94#FM1LuxA~i;+XBu-BR5{ZIf<@7z(C!;&7Lu(lS{)@o06cYusO4p&6i zB}N<>UqT8iDM_lmW2dqv3Nv1@C^+x9L5N;jnK)ZkKvI>WoSCgN;%tLs?NZeYLy+c+ zrpgT45H(Si>(!~btz67y#;1ls#<)nNUd<`ssc;BaW;|B)c!$0d%?(e^E($1$7@yS3 z8v(bvsAKQ(X;D|AG2%X?8Dw|M+PETYFS!?ETQ7B}1CqHQ+slK7!j>7Z*s$cvZAgtSZyW9VKLhlY)JsU#jPwuMHnwRnQ`4Ia?^CL? z7zP<3R70T*9u#XH8y=$HR%++Qql}P3!F$h^;0nAHQkuErGj^DDmHQ_x#DMWpBz znEWd%0!}7c+b!$xRqjhN%_9`tmVEXLv`g(76#rkc_u4>^r}AgZx1_(-S{LW^tWw!j zN(Z3hX0jWFKY-f|W|8b80NP>G{TPM>nO)-|Mk3%q|5}(OGAH-uTjgW4{c=6KdPeJG zq-{#~u;+aXM|~2%fVv<+d>EdklFxN5U%b6konjZq1%%_StZ>ml?R7dr?FKd4RpX2j z&XmF#GW8KB&MZKe`|6Ux*{d$aoI_xaWl3IqQXr11c;}u}GPpVLrx)=p-MZE+5Y-b0 z#;$QCpa!`+<|2k{>6b+j61K(bMU^LEv#e`4uOfDa8o1Kt{0EFN8IMz$?V~F5`$jIQ z1?^p6ITBVEMVy1~(ma!;dlmbSy#nti)Ko8xS?Kgy_Tva8zn=PDg=@jj$d$NO-(Zp@ zHCp98nR5FV2H0+u3MdkeA^r)~S^QXWn`~$;$LwY_TBy zsfV^t(B;7=oEAuyZ+C@l8lF8dvL7p#{@ z0;79U(G5hdGF@4jsCdH^4rC_aJnqegN-T&tmA3SrGjhLu%Bf^Dt()IDW@zH#HDPs{ zwRfPi9_^`W!69uYaP=HSecdIb; zrfIA${?W7h8foxLPSqHjPDox;LJC4ovv{@|5e<>@2eT5xz|v<%7S2_@3W)3MTOuoo zf${w&(AN9w&rZ=(F(rmc5jpzLsdYg}P622fI+9hbX!{6d?X!|F#cMDVydR`#d)la`dEI7S9Smkv zA0WXdHe{#Wyb-BLp;001p5pYM+TPNYLKrq6Za(-JB$-p2HQ|EJu$NCQ{ZL`bO~r8t zlbJ_@NlI+}Vjh@Fo=S?n$WFA^gKocFfzO7Tsv?rc`}fMQl1^9dfUYe`PvlUNmsUp6 zu9a2cYl`OIVh$x9MktCty8IN#mU$tzL*^aUQjQ>TL;qMQ_60lX;~0i{H&YA}&M7C& z8ZQ4{ZqoHFv7~|#8Af{1s?fNc%!AhNz3|4JhVax=I%*QS)X1ZE0k)=@8tg$gS6vX> zo<-v)o}>ej&C=^pA`J;+aZc7OqW8=${T<_WUpY>dIQT>_2v`*kBoW+)F*>(vXd84f zLVjJ{j5n_Hwx|5C*`_ko%U09{hAk!8FeQ$DG5yFjkAIs|-FfCk*FfaLs@kK2pYYou zMNDn}2A+sn5%I(e#2{Ml0)(XYm}(bo;dDeQ&8M#uZcu*d?`?jTY%vZciY65yuZm^c z(7H12_vVt(ZUMD2SmM`PI~dmm5Xx6R-M<#6W{;#6kx<(=xGD`O&pEuW_{bDlW^Wq4 zLS2d#JrH=Ju?8662S7W9l>$d1g-bF-J@rqxy_Dko^EBe$LBPXv4@5oQAOStN!Q7NxYqv8^^m<0H9*>EBFqUEI&x?#bTgveKA)!cQ9 zt@9Ug-Vs!kSZ`D@poq?7aDP_Gq7IJnQ@rWbgVf*0Am+oBDt_~=3~P-RDrLqWY9N#y zn6c}p$1%&)HZgl)3S{WTIH}lDIx{dvMr^P%SU}>{9aP1(xe#Daa7kRW(W!80fT-aK zOA8rO%ao98Hm&WF{=|;9DRq2mdo^M)+L#;t5CL|5C$rwfeWo$#9T;SCG*#GJEvXjd zO~f}RF9m$>%i1+$PV)$G6ak}!el!(9$2#m&*gASpdty_nf=}QWndxdA)B!}!p70`r zTPM;G%c5(DYL^)6*`KMc9Q32}qJy3^CC|GTdrr$d465iy`{MRt>ke=!I5izID0&&Q z{+(kwT&0eZpeOuIeo1?>=Dv8%v%9H7do&Hm-)CNc?kvZwlU@UXESADNSH&E(z&v=A zNu9+?DepJ3p6SZsyr?uUnqG4cT{1q1hsct`N4BSI+&xCZI>j}zY_KR8*h)>sMl#~dSO>>jhn)yGK@Y?Ye~6}rmkBtYhu9k41=v&qA{z(?NlWP8 z*n;VP#Z?tY-xj?Jf~=*TZJ`bCAU@GGVh4_HMa>A08uXY6M&`|zz?Zr=2cz&vPzJME zhrJt8x#By7lV$(Gj>YAK0!aysABgoI_0=0;SF9ft2ojWzQmxor5y2;!0Nfl_6V(kY zB73N?f%NR1KbiU>a^=VGikdFu+pdQS54IO=x}{aM&K@uBC4_CHNi^ghgLM$KVzsa0 z0e7N@i%-GFsftDg0MnI!7m52JG2R&$hgF-uVdw*5qR_wslY1(QCzl@SN&@LRTRSQs z?c17Yqv1EZg66S`^QT_H@#5 z%MX5-Pem6o-4Ak>9)13_;G`Cjx91HPp#wXQ6MOUlkLY$#B}*uRE;G2Zw1HY#wcLe; zwDMgqOUhx7;dZ>hZpB=>A|d5bGkOXEY@ zAqcFtZWEtEO=icsFuyza9t6@LC5+96Wu&m^Rou%MCGwlX+uJFQm?pRR9YHKpbp_$= z#OdK~7vk8JFy>Zi2LXqJ_s{;dsH`%|1h-2e+kbyt^2V3wt%1v{+5an}0o!_mLPNCd zv1nB-a6g!_i_gYZ5alU9b~;(0<-PC}rV+hA>{=Bf#5dwmyODCs51$5p?!-6GiP_KIUnUJ(Y=Ze+v_uOjw060kcE<_g@U0!!1>K4xx}ik{$$PC z|6NrLB;cQ zj0IN+sK5V-+r`C^2!G3Oh-92}508kc10)Q=|8Q~qh^m=YH?jN^{_76sLLnXjF>1`Y>&nV=DafqrH$=JF?HiNI7gT#f@_Dc4pwASRskLGmt|MgahT z_OgpoWP#xW{rca?u6G+RDPY~g_?g`Th0(EXd{Nxyg?X(1hj5HLsBan6d+bpw``m%u zzH2P#Cij3I=DM1KaDde&{WW@df0asf?^okbn!w2VZ-n2g6oEwP2;xhtT>N~cN~C$Tyt{_s zTbXAjb`gqE_`HIep(&oUPen%y))J9CNKGC8EjQA74XpUAGLwT{fi^|8M(t5HF0G(* zMlyx}qVzl)=rx+-cp=mtSqjpb5ty{PL9F@=IP=4C@^&mdQQt83;A!wN7pk{nBC@8v zDrTgqd)l&Y;FemI_x{@y%%r$?IWPk8>*4`eit3<+p2t(nd-+<>TlIKC`(MyA+Y*i6Zvg zUoTrJ{ElUc>Ly`69acwEDzk;>k~h20;t?`WDuT8tFNqG5Tb^bCG6k}ydh9=Y5ba13 zt0QDpE4<@SdAeGvd+l>VJ9)Aa;0TKrg${V?f4`}v*=Vyj(YOI$U_Na?(qf-tuctr8 zdMe?0uDlSV^T5Y5Cb^<(9U1}ud0ICYGN9>j4U9sDpsuR8gW_GSjO%N>tNb*1zgCBB zQyON*;jHJFWwSU=zYtVMiR#==?pWx;POt4{w^X*TMQ@ z5g{#8p%%+qd__K_v!% z6aXBQO^AD}52L7eR@-PXqNJbj>p6pB%#4x^OdC_Rb-;Pu?JdG)wXf|=C~$0umk}TO zad-JEjEqN^#ui-4A2IH9c6yI8Po{+Z*=p8#Gky^83B=7mst5}o)zVgdp+4k@p^gA2 zFE!CcIutAT+tVSMtPmzOvmHQK1>_z>$ZFAEa>v4Tt7$69MD7oCo?mXYbk~|X-yfQv z=s1Dc^(o;5Yc`mi+ZU)t(ob&lm=&dR%p%|R@<~=;Y*(o@&3n&P2z$y?@iD_Md^DK9?5V#NdRg+fa3u)0feAH5a6w*2p6H-ADc%98RTFaF>EiA& zkL*k9RY8qzHR!$}Hlb&W?>MJ%o1PQb;a?vQXv9Br+nKp7FK=)!y_!S++FUrpI!hYW zDCf6r&6yST>+5-7w1cgqvXZFJE(iEjq55)!v|VoTpy!-D$=Uw~6N6RFn)FqrI)!An z4_0qzvGo3HHYqcd0hVF;kz57x^xZ@r?bgut2v{7bglDTQL9x2l_6+>;M-@g%Rpipt zi{tDpGt3If@^--j_YMI;dvYvIgCT`(@YL9DB~}8~<&EGDQt}0v5T%Gf*BggjXx%l* z2>>sTGN6(k97rGvjS|f=8JzM^GpGGdYSVqbwvma-U?6z4F^vKI)9nFFfznx7Ds*)w zH#iCt81Qp_Mkj>5C;-oS;en4e(Anay^H{SHUY{1Vx&?dphfD@i7z ztl>*PK8gUA6A*IG9&06_<9mXvZ8-*yZ3Wa65@w+ZL{#XJft&D5-D}TULE~-PN76aD zU7#2r{usz+QV#*P2`8ShPnt(O5NaWMWD&r~ldKgcl)591Gk8OZEi7F%c2#9txsyrEGO!EfA@^UuAR!Ho9(YDl4*VcS~XI{;|466f#~( z^KqMpIDV8jBU(PM4Aj3jV2d9GPS_J|cqZOtwOK9MZ7-a6*uPSwckPGFu|Az+6xV6I zYupUrW)t4-FoduMc#1V85<2v=??_MY#EVnegRIHV(t*A(SX*+W<8A);YAiVmbC@1o z7l@OiH$)F7a|s9y%eD=$O8 zqyXcZ6%2dNx2I{8j7Ey%H|4H5ujzOEFG zcE&J+ba{Nmb{>JJ4P@8X8TbMZDa7(4f%s~AF*|YgVG|aJN{%b&t0v*RszhB@q3UhG>^bk9O=0{JM(Qcomuw!_<=HRZba)M`b`#~k-C z@e?}r4454&hH^Um8Y-KC@d?NvJ+W;HLN8paDx(4S@Db6SKu_Gp*%vC3ETvj~elKY< zGQ&d_tkK|Ne1VyOsUaUM12ohZkv(BSMIts`yP+(ZJ^ZcwygUA~V>&-zasC>H8by#P zgxy_{#{e%lBf^+!B5vi4{;oxD3#GCQlAZLDy|7J`t4MBU{?e;5L`bEnVu)8|2-OQHO(w=LHy4Awf$WMbWm+;xW5$FfpACAC0 zp$(8~>55eGjf*vA2{BR@jnRWHS68S@Rcwn+ZVk+LO7>PofnoOH49d`R?A?Y7nH_OO zXgw1)e=a_>Jo&j|a1A^+%sap+=y@klqO{W-fKa7}HNwjKoKV5yZ-p2AsQEYb1H=8{ z&O0t(i}X4PAzxug%L1;fhQA@*0%7G9SdKaEFmarT5^dLnjEE|4j>}Q+&4B`>k|PdF z5LGH_jaLqoM=JaWMG=yut4SfQF8_%1-`R%EyoXy6O*tXgNbK%Q4nfI_2y2Wcbi){x zLKP3o4*J@HHU5Q-w$w1XU(N}*65EnO#9q+tO6$?V=yoR-1udT4*PRQYnNph+S0OI1 zFN%s76}(nlaG|;r10~B;O{hz6Vm)CVo})@w0H8&zgD@M$FPeAKEWt2~H|OnQ6GR0jv6dY z$SZ+Q6X>GbdAxT?0u$E~jcH$qHHtmJL{Iso&~O3tU;*G4N=orSP*m@*NsNx19vqWJ z+fZG#Mkl>KOC;j2`X#`zJSVm_7pRgxtaxz0QBH0@B=3IcL9Wbi-k($r0Na93C z7@#4!l2kzd{_`9?_joB$4*GY-C#DME*_hJ8dse1MYc3=qSSRAWhk&bRa~)sc^nH0r=YUHMcsy#^q` zxaEw{A=%qQH*BfHpiH!^Cj!G34|UkE3-9tI=~VO3E)o;gnf4kIcW&i@pSJ&)XmrMk zuNwzKe58$Dw@LCHXUZwr7)+Q+UldY?`?p4t{etV0VL+9Dfn5!M^xRuJD`e)^ktHHX_ zp!_USg$4rh{fqZE|Cc{y=h7@LeUAr}V@$~!vgP2mt6&CV{W3SuBj_DvUC(R;7M)Jm zpw6c9mN0ips=T;-j$`eu7b;Nt$H_>tV2;HK7%h)cNu;iayppXu=RsqSECJk#UPf+yRR4O9HUr3UAM8M& zJBV!m`@3-rCqMq-~IRADe~6tm__Q(qZWbC0%4xa3nQM>7?CWXzRXF8=mxa5 zPLSoCVl5%dfS#yzJ8nj+5jGQ&Iz?$zU9QBE%cHl(hi5B>I%MB8=Or}py0lSPR@O|7 zIW!Xk+N9+(1pIIlQsx!1b=oTzhQgT$%S{$mX;OxxwwjTkHD@Sk&}MH=b=nvAG&}rL z`;x1-P5e2{49}mRV6@|Fnrroxtmq4bQ&Jxf82?(SUWF7RJuhzmS<9}3%XC;gWIHgN zjxx|hT)BEUKpn`yyJ3q3dbn8@%i6=l%e+_HE^e>X0sS62QCUof*9S;@rIU zC~_Hwy3=a0yl{1w2IxS5HHORfkZ%E$TJ?vGjo~LkwK)B&1I`rW(`#5MTNK$8gjiW< z+&`U9b{!1)ddw^iqF5Dd4DpY(NK%~U7R~gFYIt3}uBhC*SnYlildiTFl`g>`e+^IL z#vT2UeZoF|>Geq$aKvZCBVhQv{0?Cm9kDxXDWU?7e6q4Q+>u-8%f{#M{Hkr5%}gqa zEKly)F?ke+YG_gEa4{4L6#x3M@m7!XcT^D1ZwAv*hz0q zrY!@F#HcCzf=s+~T1QCAp_ix8M$4;yY0-p^J1u{GWs+1yo3iq&c1CnE&Kgk{PQd_= z*Kf|)Id?u_v`mv6mJPUr@N!FS%dl9lY1=tFtXnk6UT`+NZGuU9p)D9Phwu}Js6T4u zhEKjgHt-~f?k_t_vr_gx9fh-DAVq@_fnspBU%&y6=Pt6yha8Z;1T~cau0ZSV-^05@ z#Noo&JG2po5JIcT1Xq*;yyW=-W7?(-r2%h9r=DJ6tqFdgT)~A^Uw^Gtmlob~-MyO5 zX1u`@GMDqUU&A;%l+|HcM;GFAo-h2nLjH1A>%xM9H_$S)RD5g6PzM=;OuV)D9h*XK zSCOhDTul%@@r7MEZ6SO;u_v-Wi7zKK5D}t*-P`<*zoJrnSz0tUYbm&4@ng-pw zk$HND%a&+Pxp--C>_2x|r{JVK`qlUVCpS&!fjeM>&|k>3Sq(`y@P`bJa=ZPQLS@8) zmW~hyDy^%b-s&j*J_4d}$7_h&;n!{`XdEvuXV7RTwL!Z-wRus}JcN5Zr*{*$R&rpL#s97#Sc;NF2#`Rr+O_3L= ze4XG?E}`nZEM0&v`Ip&R#CQvgbpp^0cl}AbkkE%0c3VzEl9!>zP$-;e4Rduf5yzUp z-$YZv-vcibVJPq8w?NylVoSma=)V}#k#}i%AB1Nx7ya0*Fm>{nZXo|M6LH{kR}rQL zo`_XZvpebHp38w17YDiS*0F0R>{*>`{NIUzjQ?{nkn_KY!2gJW%&h+f2>$3Nl=s?{wZO3{QR43Q}TavsUpPDnsVLOMD@Z5~tubBmdQ2?FSP z2B11-diwf4CB;yd{|KELftQ03Isv`^^&F$c&w3UUx_eZ~V!$D9q8`xv`1C`2#l?Ht^uBL~pk! zCSb!z2^)c%88)?leK)2+s$u|Q9)zPGvL;}e;0pk9UU7XjCx9`W0=OkG%MyvoGJsNr z6xM_#OY}c7P#YIVK<;-?R7O@uI1-P5uAqz#5WsQ)xQJzG$?sPMe53ciH4mJkqWAxj zfdu(<1vP~<{R2<`$Ux}-O$I)5U;8-}djB6Xa0}A-RUQbSxi~5cm>xN)vy;0pq_Pp3 zcx*gyzW)W^_aaqo2EZ*o33lz-v`tCX?k#nkTFxgO+)y6=`4a7-&5=sNo!8Au}d ztF+PG{4jkkd-GstXcuGuj=T7xcLHEWaL2!(j>Y|ZNL^b+17FA7(%=TTfxVg8gV}-6 z*~SGN{gcq`13X>!BZ>fl;N;@?kuALMrz-a=_t?86DrlGDQ}_JB>iT<&%-rJU^72v1 z{;SbKIkvMtu|E8+hZq3hM6I=F_cGHvJ2mlxpB@-q5M30U!4>qixwprQm=&;?*?DhB`!&+Y$iBSCsKJtW}jYFCxL@7F57bx|!(M?fxL@|$6a?G@kI z{pu$@U41n@<>$6NwHzQhxHB=2NhRzuTD^`G3toB6l=8|MZmd$<6p)TBtAnd6TPYL4X{; zh~2+mwHJP|xgWWt#U7gq%<o8~cv+)3bF|Gya?C1c% z|DfynbB;m&yb3!0tA~hh)K3TppiF@u0=mEC75o7hL&UEj4**$&A2FRNK++NZ5RiW2 zN6;F843i%Mx4+~IyyqnOFL?K{@>|f(1LcpP-Fxy+Oxx}Q<*#6s9qz}!#qxic;QP{V zUaNP;KWvb%+K)Jg-T3R@5$1391NTEis0JVI@!ym>hF(R6Pw1Xym@nuaMW$b1jopHD zt#8QD7wnGj`qMWXI7gQc>b}f~VZFO*m`Ok3msa7%C+9klpXpOs(`R(=%8?WLH#Prj zy}OvWZTztZwMo5v^&LyVZ^;)^=!V})wp8{8IKL%3S{DE=jes9Zuu1IAf6kys;mKF& z3VU&5%BbE&D+JxY9pgGQ+5Ud4r!m=4+-LM*tsC0^0SnvzS%q&Mzsn!&x4nt=Jt41O z!avi?&}UDzYrpN@6TP`yzuFa3el7QYc%#4itmx$NunLnCSJ$dX^vIw_=;YM!yeF;L z3CFTOFVDTb=fC1Wk^E`zdOHOL1n_oXFRk7B`o|}Lb@pA?Akfg*X}5lF89$ffZ?iUE zt~OuvKi}#=00DRc%5TFS@sxVuE*EKYj{Fe=#NK>jtc?F@nnY=UN424TSGXD8)yQDXww~uj zQ@H1bL@VLy*VesdJS>R$6hX5vi-8iVA2rFx_3Tu139lJL9+O0?oK*(*p4Ia1AcF*= z9R9YfoAUIU{5#WwfMux4PSLv>sQ7p5tth9nZ7%=%V)nIe3#B5H+OBX+6?hi4H-6m;eMTw9t1f9iECw{ zMZM+KiRpaeq?mQbILLi;b732Yr4S5e=&=9kHrIDNj6*CJBU2UJUIu|&v|TyYDb}sA zHm1X`RFOYCk0aKMd)OMgBiBypYwa=VGiHDK)_`7n&HQt=gln1|6PpmR*R%sp)|0xBf)`FNYmUd{HLfXM;(SP;>&Dno3 zoeC)otPv53wJ27|dZ!W>j6<56CcV|WIWv`9GT77hz`k1X$do_{zeK9ML>7}D&O_C8tM?-X2OKz#*ke-lr`0ngpDo2JCrHL z?bN3v(xY|f^Taz)W^uJxRkNcZWN5L5xx${jq>ZPV2Z2~5O%;oK03apRRzLyrV^ zGQj|itaMS%3o&n;GN61unkU{_!JFyONuNU|kGmb$uu{11E3IWE!l^x6!DAqsyF00$~HN< z&9Qy3-RR4S*SC`3G~HnW(iE=9S-5hNX<>OJb`=Hcyu_%4#q^TZJ|m4hukj&`JR5)c zqV=we;5b#VKWic^`JsyvBR@YNx>t=sfhEQ^_>7oGm3~SFz?d~ju2ePn-YWA+ys67Q`%92S(NDJK%PmbqV=suhM&zqpZpiT?#YK*7IP<^{Y<&P4FU{psYplNYn$ z?Z~MY$AZeDbED5(T8P^VWpl$l0ch7;0Fj*tRt~V^jo30MdSdwy_buO}AV(j*CgsTCn*z;c zE3MoJyt2GMb<)&jCLtgNUH0c@_OvQ$4#bR;Eu4?pn)2<%>bM^Xgrf?IG7sfNeSSn- zEB!9XiOL{V8Q;oOL?)Ij*tqX!+aGh%&d}23?vZ5U@UY2Pzm>q}W?RtK|Df@qZRv9S zrnvz7DM<=SojwFhw1Z@sa(O#qayn@%5^plA&Ftw&YG83V0!L&cpZu{am3UFe(+B)# zFti7Yi>fq9bIi+ggD$5;HIL`abp_7Q`>{s7k170iCzP}&`iEp5R|Je&LJQ_8i{bCY zg;>8?=?xkEaF8^!(har*CmrJP<9wO(pqiQ-_%NBkz6rSdfa3XMvGx)+~7xKMvyS3It~nekO!G{QXhkYb3gr1WU5+JMsm07w$SG8+w#gFbY1iS zH8FW(<>46DT1KbaQy9D2#3k}>%LR2j?_yPy>$|~bM%zVqOPex(dg5_M?|z4%M6G%; z$T+RRE*u}a0h;gNdFP;2FW9lWaBzegO?q3%6oa%GSb|O&d;8&{!dlEm;8M|dsJfM> zC5PtyR5{Z`U(JDg=jd0|NL-U#B(pXrFTFkj(WYzSxKP=_r>Rt5ma_@TZ!ky2#Y^!l#ZKWp* zU#;{Ff^znxgb;!LZW<*npIKDbMj8$|cjAj?E#`wT8>QP+*2D0FCns0)ua?YgxT&aV z1~T23G#C0MGR~4TlXGS|qRU_ngEDh-5EMDVgtyKjcLRct?TYph4yV*^(q+na%ub{t zVk>S|=IBEy@2B5eg zl^PtG+fL`IW7GVuwt!cOeEe;H$?k|eGVFN(-^|yuaMxB&_e+DsK8z}x44HvVY_s<9 z$pH>u*yjpN$H52XGr)1*ZT7JWuh>m2>CN zcij)$I~HB#?RylFfW>V_90~LwMdG;uB!1yUfqdnGB&%FxOG^IHC}WpIQdlBF*Q>1V zM(M8~`Hx>QBL?}8HcJxp%HIL=%eBn*Qlb38xJ*q5aIS1v5Tu&19%{|wnlD64hAo*0 z%d(EA^5wU?pT=(3YrKeNftA@DR=>(^g=QHWz;Lr+o+PgcXC5;Nx>P=Fk&_Cf0n^b% zB>?2UBGvHkoMIvt8BVVd(2%vY;8RS4D{_5ylU!BRtj4T0GS3k6TEM>xg$ojT%4G{4 z)<%AqhG+L1C^IQJZw&QYgNH)C)GocwRSvv(fs0eC0#jARA#~(J*JiS#sa_5!ps2^; zJ`x}(g$42*gS>`<&iX0YNQ4IuUa{Zq0;zd*I(ge4{^C|8Zt*rHRqi%RVmtNJyerJh zLBXWv#a+6O2P1mo5f#y`pCej6UILA_1i*^{tC13cGwUTQd{{7lR5Pb zW@_~{PDNKW7c?!pO}e*RN?K$t9q&kLK1{@t+RVDmh&X;Z_Q*Mzu&E+x(SC+cT)atN zWe|Rcu|nD48mHED8H6$f170X4`uqqaSrI2~#16=Jlt9BvJ~X(AMk8e|!9jqR5I0!e zN)xA6b2a~Q(5@>YI^**7y|`)566r9`w)94qDLTaa{%e3)ZqRpZco?d8sF>K`{Uusd z!c!pL9k=Qlf!!aDL-wKf&SBqcQs;LwG=#NULut{$OirVO6k|%UA@`DmsGPEe1w8wM zE9=10?=^bL`#)4+7+RU>-<7JO+dzj@Im;9tsgxO0Q?Jd6AK>_3npE83gyGiW3!~0N zhjVK(QU4&uG$eJ8j=o}_=XISZA?c6&;bYna=Hu5cKvQrLBF*@^FNRApwX)Kf_6t+0 z=8iB5in4}F_p@6z+p{?W;mcgC8t#PCSGhVZu`G!34>|5C&S=QDFMEZ~*A}wVK0KS3 z5GaG!SFK5r0;^!+pQSka5HNcS8?7J~z!7Imryr^Kg8P;qKc7+9IBI@UAw8R$1NGuKLW!mv$DV2*{63F{en8Fz1l)#PtOO8gy$t6Z(rLA)+ z!IHz(hk>u;w90hi$?|w_n$-Y8(6+RWpla!J{Fr40({ct}sStPN!t7e&sl3=$&3mAu z%MUHnerePo+1M6I*kb&mvDVD%o9yo^q>U_S`QgR)mwKNFLO)(gOEi01e}?jmjSH6IWkRntVvVk5nvwwAzj!Ch z6+&IBIvase1?d;=IAw+e{}rMY_NDO0dX}Ee4<7isZCoA-J6DlRwbW;E(utdk${7q1 zn9AUfebdWP+mur|_gNGjA$Z(fEa1V$*%RIj&!a0Nn6Qe=W3bY}c^$qbFPe|y>);af z1KoaH_h^3E@yzlj^gr21rG_{C6@Bpqbm%=bsF$W~PHT<>aZrT&s;_hwzPzK$ufvan z;Sxoq4PBIGs5I6{a3 zU&VMXJh#DzO%r{SupBvEe^jo>4A#c!S&o@5bQ|l%&axuVJteYw6$h$AKsnvr+|k5& zqRD}VD-oYLra>kmJNws=)y$YoL|Z&}@`&zFh&1$Aw8ul4e9V*81?oPXUpo5knC$7x zAYBa>%1vr?BUkGiq$N$D(x3;#-Z0F4#3zeas#PLXUvvx_e^M9?s^UI+ zIOZM4Kmt3yzBr0Cf)dOR&&53@T%zT@9zI2&-eu`0-%qb#W1B1B$XZ_A$ZWr;b6$}# zWxx`GhKKU!=pWqD(y(8&G-~{a2FTO9a%*D_#76p1ii831Ph!l2A>F&TKV6d7$62V` z6Qk?*TvU#*p?-~Jo1nX@opNcj4*KVOy;=QplCSXoi8NDxRD;qwQYFj zs*jyyiuww!U(Y!k71Zufr3YG`qw9#bERWIzj(riXw7E_)CwzA^MCF&eeHD>BY_DzB z=*7nF<2+H;4O4Jx=OWcjXrY)82SPn%D3t(yD@P!Mo!4fg}r*JW<9q>E&^{j2}Z>X}yBkx6c_^lFFSewrAcHkMN!cA^36ExVtccQGRHv7esxDUdN{vkquAyEb`^e> zh{5?F3DA7VO(KM>j z*cIfS=8eu!ax^Y1&(&>Sz-4qzZKC-Ea1Su?NkD~dTIK~iP;nmY@f&jb_c+^p6Dy;s zS#h5%r0vzPukkt;wKuTXjbV>^yXqt<_puGuA`N@g*0)q?`6i@o9@9F4g6?JmsjUa! z`&|+SMU@Tt=B7Frarl#h@;(+M%09efDic_+Gd+A&CFXXfaW}5e2>ce921#hXS7=0( zQ+`-L#v5}pcO*ciqAY-w%@qX2nsj)Ah-X|-rH^d`QBtE+|E2u%%ydzVL=o+}u{|#l zxb2XPObY#NB7;5rGKps>)Hbz-GtbMb7WN&EnNMzloi>{ zSOcHjh1k3HGrSTDJ+;|>N!r~l-!NPRGpQKC9~Fm1n$6+6)aNbF+6_XauersrL+Rgj zpl+}NwP2>O*BKwRW!K|^wj5@7$9)P8cbTSUzQRSM3B0n^O-$aQ67G1PeKgWvupGHD4WlDR zPQ*;Er@2GUlO`pbW|)BOC%;RCx9JZgCu`~EGs6*xdx6!yhIK>yPJ3t}<)>0K5x-mE^Hu>qMn3{DJR)qeN<~o2B;I`ehQF79k>s;1??PmS?Ijl$~6Z%MDoN zrDx6tN|lho$mi~5Kf)W{_lB^gdeE@!AsdFOPE--XJSyDwak%^ZJE3~7kER9o#vXb{=wcps+wf_TY>`4*qB&XFRq>7NbQEJ;>Ub}~es*k2x1`G5G}5DhtF$z`dbLj0m@b5Z{oj%ZZd36|Kj*dm2-7J!SL!S@av;Fp>taD0r9@6md3?)qt#N~M#r`hMMce4B^sa%4 z4m3?P@|`fD@j|1ClH$>z&xUn*d0&8`pB4~s#Md#Wb-e@ zSOvh&$EG=_i?LlI;c5d`mDKYn#`&dl7ni4Xi+lvj?WNjsQtI@q&vtCtt7P@=bJ9fD zH5gTHnIXxtI*79wee4!ovyOH3+MMcl2~+zNSrpVa^A7Z1M@8e>k<*Q>8lKE8%sDEV zB7?@BvCL+hBAPvZYFyGqRElzeuV{#*OVF9hskM$H zT8bCeL!8HB#o>#X$mae$u?wFObY<>q*JG+4Y?NJt7Vo+7?ii~#fMHSy*n7ZC?A*%*R8}4r?#Y z#Ite>+NmjvrO)Ov8AYpwZA1O>K_6z$4KBoSHF)o{^!rDAMRIj*$-AQ}&Dn8blROOK z13dG0OV`UU16gwOWVxGK2nChU(UaI8RcVrp=V|Hsde%arxK~wCHN$iknz42coXB)Y zW4vQJx{g=dC}chY#G1mo)i=E;!&ia=H1DlR0`BL0PJ{gjZ zFr3nBcf-vhy$o{S0yWsVpdGcR!30sbCl_`6ZTJm?5yE3<~ec83Cn@- zw9;dj^_|vy>BVJL#jA+=XT=r~N{*^#tI=11kMq@fqnNR;!uqOw3(y9f=jo{aSb{=N z4qfw83ZH{emU_TNX6T>8Y$;tsb!e#CMKM%c=>wkCLE`z?ar!%$iR(S9t5;{0Q1~h! zADL-8iYj5mlQynttjh3S@Uuxfg%2#2FK%XKWsBWB?=rVsL>IT^SOdF;ctgIume>*U z&$*}z-;o1}-JSHFDmh(4Lp_AdHrBmu1qnI2}5=ysNCb= zJb0t5?k+i-lH@FhT`5sSbDcjXvWhRm8{T-(S+CDJ*r)>0(x@}3#%$#GJv{p<=;7(W zw{}5Wl}GIf;S($7@uP4%ZVXrR8H=4)=MywapBgtc))qWb*RNr@CagUGr{k6=?+PpG zp)52eB-K}&HEiq2-OyysXnW9zIoXE9}mps1dAW_12gd;XOO?76Q-lUu5fXrBMTSuTKe#eIgppc%C z>r51dMo>_^p``GYw0IWSfy#ihDe3(BBmjfIZbW_O=OcE!PNkglEzwk4K^BWW9!h{1 zX%guUaq&ZlEfkqW%7X2+$Ok>ABSbu3SJ4jC{rhiBt1rQI&@cFQ7+=@G$~NS}M^BF! zT96cAn&=NP7njxpQ@L06-Krvg&6cl)56=^VwTN;NY1!bJK=|LnS$zRgdkuo5>2Tv>s+8xD^^sat2gofn@Q~*<>nneGzPjmh%8#E2-{6u8T zqiW731zqnrbMPdKjNZli6-oB3iaVfSjvnquTU?tKHcjG!W&m0iZO)kzN1!#&o?a$x zCGE}380|dHHyG>3Z4Pg$c=j-ZW@%apHTWOU%(v1%yUAcDqCK6BJqnZ4fp;CfaSW9& zE|}m|Ce=MM>z=hj=J#wz->Vc;=?{DkD6wvY4|tjl4^^iqM*yM@U-!){w)5}i+Mb^U zs&wt)2@oLXqFnlm@)DiHOF8e>)Lcr-v}`|5VDLO@C+`=!(wJ5li3tmyW0I+-6!Bcx zUSl;oV+o{Sbxl(cEAC;K_G1}BNKWMH6z+qioN(=Rv6@WRwgqulmkEB|e0%L1w6r{| zLo?TVygXNCB$6<66r2eq@xZ^<*O*yg3p)%`EH#grmnXZ{u}LOA6}fCy(~d~T)%bWv zm?IRmT`*nXO5=*TZ_-1!m;{sFS-}q>#r}U?JZhNRl-#q*cc4OQP5}a?EzZ?deO9jy z8eV18R1MD4%eYL4F?ks>Vhh_4Gct;~J(%imD#h<2Ka|6vL80ejcj_@d4CSWZc$?8k zzobBjS5%R>nXa;{!{!CF3eYGk)k}==;7PxdC6icn6ZVTzo1W}#6Mtk;jAtp?`^lfJ zFKV@g4no57QSYGfQZd{L7iY`5-3j>IH!3ed*!f-Lr$JNxW2i@%G}K`0c#Nz1Xm}5% zJx$@25{s*HV0r#BqrB`6hwT9zPwh?RKGVHpzxe_Cy6`GLNh!?m5jm)v{-h)H&P>4~ zGZ*LeMEqJLQB@zW;^sZOXtlHzA+6Lp)J>_Reb_8GO2c}TkoF227Qb-p;&XF;+ba_` z_;%V{wAF3huA`TY09e7pLTGc0#e{}RDsyIdh0<1A6kW9slLf*etiLOZ3?$qXlfXwZ zNT93=wL}lBMl0#x$lSpPZ)^%lRTd{Z%n6Nda@ZM3K&K^CH2d|~bM(b8Q=_{S>!&9kmx)LhziIo+1cx)m0ZAHJ%j$PF<3N63qnTk5{r3x$U_l>6LDe^8xkOcr0-ctw)(cX;Bd&fyqJF> zZ!L}tkbJXLi-iG%eY4#&)f$MG^K6D3POpJOWJIs!t>}!HLy}%Py&s1;_Z^QI*_)RfH@gh93U5$r~@5E)7C7EIVe(4Pc} z36Z(u2GEHZZJ@)OjnKgPiw&BluP2^=C}_Rvs-QfODtZ?ZN?`pJaFd9qNoCOnHy{^B zY}wV#?cXb7#=bS=MaFJ>p^*fg*0*ot*hf3B>x(yo{oUY`5voj`(8CP3ll9LB0PnX% zvtUdo9sQ&oSU5U;ib2bg+}xM>M=o6hf0001OuhgH9+fYP;5PcHv&>YjRKmgPiq*~W z93DPpqEP&3Ig?x0xyQGF?cikh>&kjfy>mEc+SgU4TiD*Bd>F@`01JLf0XQuBZzUx8 zv>$?p3^c4Ym^9VG7G`kFKh_tA!qto-J7O@F|42q{>h6l+hVPPJbwNXzZ;Pfl{uVl$ zin&0d;Ll|ncEc>OJU!O>wPR-rRqSI~#t9bQkjG5{dlKKG{!29Ox~yP+0ta~n{ndyA zh)$wsYSW9k_*{aDWDTZ<;kf*GE&LSn#nf8tOQUhckF1_ef!$KrCrW!f{`DO~QaJA{nus8^FOfsAgA^t9)bXqMhEINp9;KPvm>>tQ z4cmO3MIJF_Q-TDDBtWe!P!=l<+RJ}~t2Nt(q5=V6=nu~O^+at+NSIFs0@)Q#T zAUosxp4PvtMyNK0kAz(+nfFWi`!eo1cE9r!2dEabL`<35`8F=UCQu1PD2Z3mf( zj!rKwldz$dVHwr`|=qh`MZHidg=k(=tkHPM)d`(b~bH#R~yM+Sh}T z{qX_xY)7*D$e&*f^JEFLkEIG$csAo)khc=ToH~pN`%E17{nDH|Q^U=4PGD!y<0f6G z7bc<#RN{TNy7`D}2o$2pI{)az-aS|rsW#xD(0iU!nJ z)$0-Ztt|Hh%I2_cAVu~dkb9ixFzmEIH}3N(0=^n62_mKfF=46idrH}kJc|eeW^r7k zrZ)Y;DY{Z)8YCT{PI8>NK%G@04Wr_L$bmvYU&5%05qF(vDB~~BVvb2_Y?&Ag2#BA( z*Q6>^kYAJw`HqPa7Fl1)O5^)*S3%#XFpW&9IUdD8COOr#jor3ffz#wa>V@*m$==T) zLWZe5rv(DT*Rmq%J@!Z1jFpULjr$oba3*t1Dvzf>VgNqE*?Nw`0(9-Q@`4DSFPD6; zS=OFl^Hi^(&pShLol3ra2}a~mG!f-RBAX+=R2V;$nZrQ2oBoy4u}fKJV~kW^M?I4p z9I0CSAg5x7Wk*fd-1(&OS(Tngj#phoCn~Y(K&A7!^AYwPE?cNJHH)*g!&ly6u$I)A zWVb0(G&Bf4U81i50jwH*u*MugOiQD4AA6a~9UVok&Ym8(2zLBvdE^kmG=$k1=`ae@ zLu_QldYgOp%nSOa~c*Q0IR&4)*aHebjcf+VMq6c#eEX%V}vtY#i*IDo|K~HLulCa$zP6 z;V6;Mkg%Jl)Q>^J>V6g>!$*KB5QZUxFefxQ7{`O5FSUR=z=1&CTSTjd#*e1i-Ti_g z_KUdaiyt;q7}e9m!cXQ8j%p~2IJ~$Uo>+FDoROGe+B5ho#W|V4{K1rsEdX^}YvBUN zf-k&{t%f{BM2v_bl{2pkKnhmgN!cLGj`S)*jV73qdgfErMvZ(8l*q4gd%$eCW)0Dw zH)HfQn>Z}|=4XVO6ZqhbbA$VKJjwRymGhTh_|}l?aeWA|sv)!8Y63_h^?R~Djn>F1 z-f|R+D5UAfq7`(r7s9vuP2R+A&q5*brvXX2zXv0mwkZm?#eRUdQ#rBkm?luR92@^w zj*Ih5|)wl zaPCXOaQC;@j|2k6@_mKOkCzrRO6fDQqjN*CH?;-kQ%SmlJe3y{@LF(H(NBVH8Hf^} zBPc%ef0qa6KJ;*C*RtP!`6AND%Y(z;TRU+z#b@X~NxuCG(=x%aZVJdhC8k0d5adEY zPAX|%02@Pq0Ic#jJmEslh-}KAsD0~CvgW?rB$2-+9~Te_0N)226Lsw-Ojl?qkg;O6 z%fK+wUvE9AD`QgRc$8u(pdx_)@LZ$c5_SAK)v=yyHl=9E(jS}@PTIOC!x`lMsbqF& z3$V(HR2x&-zYLW8`aXCd&y#oDQ7O~Yo6@FH#XdY5IVVuphId0JIHuwBDrl}+K`D=) z?53h3HF@uYlq$rp*+%;`EHFGOd4Lo`&tQdOgXSFWsn&>E{~ zvSZwjU}bZ7LOtG}@%uP&HV{pvahx#RdRzdN?ABpRbHY^>KN=i9<+#*YEr@@}kJE|R z`n-1WsM6Wp-sj{2zXW302MB5Vyo1B2!rp$bzn`otqEaz4Q|CTTH;4bk#u^H6KA6!cUyT7)&pXkSNwUVi5YtN?^NsC|h z`p|QoA5VcS@vNDqwg92V%P7dn7_F8eTth-p;QVJx7)INGP$v444T_JD^)NJ=J1y** zeUcR+mmy8pyWX!a%rHGZ*U&NJQ!;xPW@0 z?9Dsjh`o(sFY7sq&ySMXO6y@~rn&#ezvw#NH~t@ghnr5kttLjip>RQ^AHrt8Tn!00KtNR8p-lnQ{ z23ykM(a|HsQg-Cgx~?i@?VQ}VEl+e|Cko1@ZHA>xBF|#CtatANOE?Z7*!Q6_Bcjjg zGdKjED~UG8kZVobwL00-gH_`PB4e(YN*HQITa-2rus>jh{yRl6~4@{OX<<=Q`9N+iVA3Nwy6FTZgLD77c@Fx_i3Cveday zqDdioK*9(fL}=XWUx$et4`1dwZj3Tmuy@4*9xK9+l3!(#2V_h(Im6HTS+y-*+eeo=DfZ|9z+pCkCUA_`Jr9O_zsS-dx-i97vp1Pf;|<) zP9RRTjx;-$uOxn%SLGnc00Q867d23pY*C~)+Uo>oBGu~Qn zW2AT?XwgtwXve@Cf#grrm58T%eo%Q+Dc3?sKP9G^~(CwZECIwy#_YcHjl^Ip{Oa zRZTqc$Fp&WJ_pi5f&R3;A>owP+Rq7-G+0`W6+BdVt{Mx`!pTD2 z_I*jbW@ZZPYziws5JK2;58FZ|6!>!*T4)&lUVlbq4*GRc?|dwNUZzU77yn)sLPS~58!niD2k zByG5FvZ(SgddxlBV(r-C>BgokW0!-o>ls8O6fiYa(7&2&e%XP?>t|gMH9BdWxx?n5 zvX|5;X$(^pwA=47M~U+@B^zCGm#N@E5!MF}4}iJ3NPi&KO0<`!y{>%DWq4WvaEc7+ z_xtR{cT0XkqA;gp-{sd`t0(C$o7^TB&*|Tmby|nT!jUI9&B2e0uQBzKs?LsDfRc+W zBCJEGTwv$mbAS4fI7*bUHA#$1O?xpci1c}S*+1+=>^Q57f)W_kM7IQRZ)bPS^gitq zx1Mw%WC+cZ6Se8%gx1AEoQ^E(i3qk?4_Sw}&W z1d$1RuQeFT)Fye>sFV}b>mH!ggqn;k<Te`R;+b6gHMWl)b@Io4Wao%DXkbF3snjR79!~^ z)pu$*-xO)T8oyf{{y_Coo_L{r79NHtJ&<-_FrVw$qyG*9Ir`fK!u8S}*AtlH)RZo7 zFUijs_$BGN@#x}T(>9NBGa$UE%cXRgp)%Cek1_%j_6bWcvA&FGlVb%Q&fFxCp0U@9 z_c&0UYJ~SqE{@W0kwmhm59(UmjMRhQQ;whQpj(S;M@9y0zFl$jn{C7(RB)@)D3ma1 z-SteFkIy<5<-MBe=6q{I&ISV|pTb0E&8Xz$y9@NaZ>u{N-f3bfOrXE=uQRxHUEhTC z9y#S9>T)pSg;+Q-dDXNMkZ61t(n+$k-^uQ$RC_AdL)gOhqC@)Y3m9D+nC2wMGtgIJ z_K@8t-5rSW;s<+QNgzY^BczUP(J!G(H&#VBAXxI& z((RxPzFjiEU>jP)QPJSS-fA_Gmd%@x?sO^YWeBftStd>1LNb7L;~U zl%Nj?O{;)7;b;1p?&VM+cv`5`aq@GRQ#MzR;D-VWG-D6$Fh&2681WCg6#hC2^(=`k zTMuc4w5^IF`9DVCROI5_>Q>u7ZVhp7H1W|AIf@9PsM3}y^@WXK?_s`W{-(RG9z{7rsd z(cZ;taTVux+Qr&tNWf&tTAZm6)E5quDM&~7P#NPJmBrf=*Z2fz#uM7m*msH|zJc~B zt?OkICQO-#o;#+xsNWiMwv?+;HipAi(PtuToO{js8QMGHCa(m(GbK<&37QfKj)|%| zMi1^mO=Q{04FsmLeuisWjCtaLSdgD!i@mWi(;{tRRL5SRt?)%dCH*oI>@LikCpl(a z(S-a>M8Xd|Bh$A5oT@Hz?zWQD7A`dn?X2<(-OXk7#_l@(%e$js3%Oba{$Co)Oj1b? zk(%aq%-fP(6;&8I+aE2x$Sc9cHj?QmF&jw`Fdm3oT`Qs3$_}E;Sm9-{==MWIL24wj zLRoS`3HS6ROK~5#JInkH%U~^2sBn14Y}Q8zRNTqKy=Yqym|l_&p49Ue>nqnEts!Dm z2yp!fYBt0pi!DWP2Yd8iIh$Cjv*6E2NgBY?I^GS`iu!x9%RH}scro^*Z*q!$iK^q3Y9s>|H?v?d>UmO*=O}S+4H%p2U}RK{}|Ohkof#q zhjS>wLpGzdEf$xyGyb#NZ|RPzDf$aKO!mF#g1>{($avL;3}k)DujOh42Z@~5+FkoP zR>f#n0nOUcRg8F4EFqe_+AOqtg6*~4v12U5()WilCJj=X4}PBG0Qk=x(x1_ zA%h=$)a4>mDOHNEl)xBWoJ9kw>yPDVROUpsh+pUsA~g_1%^r_1UtI&OKdIlO>}O>h zs`8dBx2sxx>g_3fjNWh9&ms}(?u8xs#{fRz$c!HUgYo@57W6nRNN{Y1$WjymW zzK&EwvJX1L^ts$a>p8r3w%knpdSJze)~zZRh@b1xbnvR85zxOw@KLHj?lbZ!grWFy zrQB?SSqAKc`3SFvc%4ewy0jfV^u~8qAqMzj{ICs;rHNtk5bj?m${*=|?0ntqX!&4i zJE->SMazwk;xyTPhQaQ~qec2@ugZ&4{tcqbAikVe>nRADdvCZcK1#?p`r*L7r|8)? zWps~)QmY;OLkF5e*ttV(iIVb7Xqq@OA>19i`gwAY2^9HHyD(I)lLwxi=+Ne%4KDJO zV;s}6CR_y-b5DA{#sZsl9t)nB6cALmhU0zi(A5nd<*od@o2{WVo9~8H$EdFhw*i>6 zBn{ZRZDkKNIceHZj23T!{TN0Zf1-(B&XD7ysjrqkd#nm;*4Dw|=$UL)InK)Q(>4Q< zpS$1TsBU-)7y=jOl+wZ=Dd7rJdN$Z5wf3Y1d}#;PzFrIC$bI0#QixXfbpkO>S$L+a_=OeT6TsD(UF2INUo`BCdiW^ zRn(Ts=wqWZ;-Q+2GsH{d&#$C$>HGd=vS90!(CVh}$4 zTzk!-uoo1)gW_^KMyFfPjGdn40GK&x<*cW3xZMEzUXzNKp6UssHdf_{*;^Sbb=uta@?;L|W3XAza!$JmlnPAw~3Q zqRQk_hzt#eb)Hv^24_BJt|YGBp_N(xn0|vB28iuyJOwVlpr^PFrf}Y zNe>|x8ynO~fdhSyi#q2zhOETN$#r_StXG=J*v=35prQ2HK~Ef}5;Sh1ZKF4nsC7Dj zRSk1d;CJYn*j_X7mc>jbYBK1z==50xKWsshh7{GKcv2h|sAx~4hLPzOQ)w*DnorP9 z>Eb=ZtZ4RLlW%4>iT21u%Y8U=nvs>h1wnE9@F^f3JZ zSZHSyY)=sWuRd7h*CZJz@(Ly5!4v$I&MQzsNOkeP2^sAz*z-vmJ~;n3KmUDs3+X0Q zno=!23Q;^I#EY?`HSDEUT;1=n>$03!=J3p*zcD!apiJQuNHx?DfM2uo1nG9NwlLmQ zjjE{Um)wYpWvY<;uIqxqR#R{|NK?XT7kK>LDi^n@MvMipVGE}TCv%LKzItLg-^*Fb z|J$PN%vVATVIceV+6>g*9AM|Eb~H`+N0soxiJlW*TdW>`U)`UpozE7_89A}>=fek=THV^zZ{8-_O8pcMFJUhS0JBP30Z*oV(C zZY6Ws0MjAfS1w;f{eNqxVOJIo3ozZ{0siF_^qCt0v#Q{Xk~Jmd=$^TuQEtjUb;#sX zMH$Oh8+goy<#6EH4)Sdt2xf~tWTy^`E3<%t;<5II%XDJFHprPV3hW4wvhPZe?t13{ z*w*J{)Og!K9oxcK;XGdD>GnL&U2T{}rF({21dHp#^-H9Loshxt8$ndM zvJJ*3uFf#vwD*oxN2*90f~PFHbvQ7td_%*Yst*9A&GN zd)jP%7dqeXz@aX~TkF_-XZlBjZ+Kc|R{&DK9k&W?oV3?bF^_Yjsq-)IBDoXQL0=1wD|tM^qQ<< zz^$=$#O0}B(;oPiQfCj6Q3}d>{c+N>=yXMClB^lap@6UZ*>B864(IeUB*sx}rBOA-DQiIVZ|?Ard;cS~Wn;57D8 z&bU&YwMbCd=-Yz%_kXp#Eyx6IDvKpd^dIjUTzrXMp&f5OwhBuUeS84#yd;rNq%a16 zCF8e{Xij5!V`^Hfb$aq)&XlfJ`PAh#ITMU)n4tJrO;labu}CMWqPWg2S?|x#C#@2wUfpgwtNuk4~k%A^`~0!-QMX=@6fno z;44~|8zQrT?aCM^sN&ZfRG`M2I$sl{UexE{GhYcHPdpTlZV$!5)zamS(ScI;UY2X9bTqWHYV@l1$PebnRS|K8%H>i5+w=S+-~zEfQEvYO^Me zIbJBY-!#ss<0cUl^dKw)JjA{|Gk911RO0#l+Bm8gdqVB6%mDAhwZNI2GKF)T3&QFu z^PJ)Q*rHRSbqxqp3BcVvDhK@8&4@VIeBfBoo|8aVpW_I z{@Pbq*R$|9NrgkZ&*nBhGQ+}#Ok>ArM>KYaFJLz|G)%u>cyLEDas^nq&9PINM`0ir zrx{U&M37x8AFTJC{BWG7wTrk1Dx1gq<1A-V7#h-l!ffSM`sC%3`%SA;j3^GpvB2-> zfPTbmmzSN<6<{86{ZyV#wk4VuzUc0Z%N?PpQWY&rw=K$Nt@SFQl{NB0_ zMre=~+D;W|5&58O=3RF)P7dUZ>kJ8G+Vh=iWqgF-PxmpO#N*O%gy_kt&@XcjU+b1; z$J;HO>px#JT6==x@5jkIVc;Ek)a8X31DC|cY-V{z*0|v>%>BEIaOcDuciQ9J9w9?1 zl9E+M&`0vSp1MP;yQm;7Lk(*6CseN$=Jst^iK;jdb{Ou`YdQ~>t&ENSLsqL_FF4l^ zr&E{c5`WXwea{9WL7=$1X;}QKt8sGR;1E-E6@fjRRmw8oyezlh zWj?7f2B^+8(tq~N`1TsKdR3jR5S9N+`QCFl;bZ6@gJm$V6n&aBwwV7kWFLenGmPCs zj3~giF5tFp+qP})wr$(CZQHipyKURH&AZRwB=077@J5wYsZ?tCS1Pr>>-Zp-scYtxl8{+jSse{b^YB|llS#ZU!A)csPL#D6;v6yHk7vHa^5*0$Pov*ATpWjfA#XBphE ztjotBij+ewvWsC+ND-n%Lf3VLvn2`7I=-4n!`nT0Br?O6|+9#r)@fB;Tn;I zguoIHmF%}@)i}yVq~Q9^JdT(9kytk?jS3@%x+0PvyS5!->P58l>I!cH!vZa%z}?H# zS3QLR^VUVT#0b$H(m;uj=ou;V_wV_Nv7#*8v60ylHk<#Vnj)n7YvIV_uT9y#3(PV1 z4z7;y&q!=8?|rrZ+wZJ&%=(O@Zx0?PpafKedte$ao*>QLQt&+>kZU|RAQPo$txOS} zKDOZdzE$6gCc@-2mlw48tCgtPz6eC$FdjPU6mJHXlC~| z9o2lQZ+U8eYT`MlJK%$G6KhK|=sluA=ZU9smeAY8ZaJ&2PO#xH2bYMKJhnt#&w}N{ zdW1^et{Qw({KPK}l^htVJHGT{y>c+Gbi;OGO>8ZNAR{pq;VJ|P@se#j;F~;&q`3%& zBo7~0r^~8(DA;S2U^C_9qqq^x53qAa{}ityQb8u8q1k@@fG9ntO%#Fcg6Uu}0*d^X zJlKI&nLIm9Xg(W;))!%4Fmyh_7Iqa`3_z3SSgHuBO?-~)ESbBBmp*9GHEz&~$3pY4 zI7YR`WxU@zP63!(S|NuqkP@;q8D-nYLPH*W>0{$(Lvjf|C(r>(S3^L(om*&|d1qsV z^I7@}^-uggQ;{$2gq-j`Z;N+Cdxd>AT<&w`j1(;E90;FAgi3HmzTG?i)@dB}&hxu+N~k{* z$jFoT9{Qk=OxIx*z7j1jN`*odry33*_#eDeLeYKA$mz!Wh3woX+lmx6b?Zg54@k-> zrxy$Fg(CRuoeW(t3fJ4KnV2SHxSRYon4#H0`hs(tAKIC7xL&?S;b7+-I|nN{$c zLJG_V z${cxO&FF(GO2&AB5&^RiO!H|fG;hjdf{9h!St1yC zbn&_~E;Z(}n~1{GFG1)n5y+<5885_HXTx&qL@f}&OLP=h-l`n@8!X;yr4Vp&4oL8S zaDK%Td(g-w|DsU3PwtlRejQ4@i`+EWbN(g;y4429k&uL8H`?lD7~M(3-5)~Xn<442 z%OBRiY_735V6u==t$J623p*bZWK-r6(`87a82)U#$cC*AF0vWbMYg5@#=wj~e?@`@ zmgc0-|3Y4Aa8mp7uZZ_p29RdUAP5(QCMF(#4>+}klQn3;Q*YP#t$MexoXTlH4aBKj zJW<<#mUz7ouqF5Sjtj%^(0kaorf#Tt6N-{w=y+DNpgDDchh^NADL%gLXdipH_t(tG z3Zm?a=1P<42&N9Jd%a{Pv zviLx&mKH|aiwLvL$(k8dU!S@?tiA(dsXmvw;i6nO5n8YmW#bxRitg4Y86>FP1=dpU z1f8-yWbj6V1SR$OH-cgS#Fqd3J;RP+Ws;0_ciGqA!+jf+ykF3cQH0n(1kCzY#JD_7 zanE-YBSg;8L+RKgh(U&aMuVkDF?NH6(!Jag!|xMglg*Be^aY_Ra`^WmWqzo3?00{U z%^G|EwQ+1xY31Wr+^UrO@hx2(79jSU=WWBd@=(JQb_x=!CjLUdHdwY4bXz_^{9%#S ziaAK(I^pN@rJ8ndoRv%}{Cy5dGEV(Ezdo;LZsuq%GScyPG-W6G>LiQr?@t!1P==EU zrUG0Jc)?Sc!SLWqJj{bspkA5B zV-|U_JHcg>3=6|2fym}Q{cY>qnAA^2Ab2Ti<;OC^VAX?W2Au?O{w~_I#SWzgf+8w2 z1!Z8@>rU(l(fBb^MI(?@HeCO#o6-+*OZ{Gx-=C)u`hVvi0`Ce*|JAiFO<^RNz;=eT z_Q!gh^IU|_U@XYSiCr)R|JO=k>1h6!-BWYX8?_XjXR<;2n|{UU6*Vi%qErEDr6=sX zC&0B3qNmzS)x-XUkViI^Ca>3d=DEaE{>3ze>UGZXXbrRQ{cVkg5$-OE{RS)XKGRkUVrUFX6%=2H)2H~L08S7U~R zKtU5g=+-51=a@R?FO_n2Y4{>(`={NaP83x#fj>3nPXfZG_M@MfL@PIpjglnxD?D(r ze2QqFfn6}lhcQ>uh;#i1a;bwHVi0a}2>R)I2aM|{-fB-w`uT$mJT@T`S2 zr{E5oV56xmRPo=MMt6r(#J0Qd)+Ke~Z9)k4mhx0rW-7`WMkj0sDj*i3yh5#&K26z| zB@++>-cpm6k)8D27oAd?_`^IgOACPIwQs!R0>?QI3dC81CIjTln|K4X0q8NHd_qC&H;1I+1!kyWrh;1-lN3c|C6Wq$api-!KNFkd zUd{aGz68LlQpEQ~;J~aeTA)I{HVn{$KF>7F*uj)c3X7_l08qIoDNpp-Egt5jnB7AB z_P`EC8=)Agd1hlK2O{xDRvafj=aN{rQX&ljY{ul#LvyuTL80?|5=5K;Z-yB;1k+X> z9%*jM(oBa+m11#wz-V-ea6EZGP@=TU%X)^k?)8wf$s0Q-cwB zkAbVoWRWZ)f>r z#F+0z%AbVcMF;oigedxhr_6{O9p5fp%CVB+CI0Hhp_4BALr0^fNP;a(0hPsjVseU) zHvOKRi?5_*_A#_qm~D3fD7A2^MT-b4DUpRNvA7HplreH8aQD5P@+lyiogRv!zJ^B- zxat;C(hAT$#QdUSLWI*nUGQHyz}UoKr4OscfGednt{4@(L&NzZ&dbXkgcMR1k^tmn zoThbkM|5;*k;`;xo@cmQCGs=JzM zE4czuvt`m{b(xdHW*sO3@EN;^yx#~^oZK_ZY=%}YYxzjZbpQyNA#GD)6c;Ge=w=}w zY?@h4=r~tB=<A?+ZfL3%`34U^NHrs!r2p^1-&_u z)--|NOSm)p{$SmjQOQ>9K$frWqU6T%&47FyTpX2PaIFD*>d>BJ^#zPoQ&ZHZZ^oZ0 zzf59|JR&N1BgXD!!~Xm;=%FtV4SFTdZ5qEsoVjT2U2^;N*27>_+Al4H}zbu3FReVSKOYb?q_a{)bw`v_EtoO5GbF&$L47*CsNG;McI&h_Fj3&Iai*zfIR zp;iI9FAEhrN)POldbOVplRY^)*->YK&y|~)SV9P(N7(C$lyyDF~jHpZ1eJfb?WQ<(ISkEg=J|ELUZ z7Qo6yO$JJz9R6&O1hymV$c0v{RKJ3elq%EK$bG}`B$tO)>o3waZ!#;hWG(8$xCcN) z4+Fhfgc;7pmG;|Hz#=r3ogS%kO#FSHIyXa3hB7jV?Bhlu1Gu&x3--XHkzKY~^yk#+}}W?N7DlV2ymWKiV3 z)I;Q=L$+S*7MjtfDDZy4gvjBEPr*j_q&J(IxsXp5;MnyKz|U*dZRKb{OS)+OF#h;q zPMtR}FQ}=0ezI3fiG`^nTx?mGpZ0}NQgma7e2&0?YAGLyJGk3M`+X4N5rHMQzz>t> z#;Q>Av4K-q6H2eKaR!+-(r0#ZW{uronyOW_T)D8t2Dw3!YLhtU2$oq;h+H$Znl^V4 zfDEt-<4C>UorY;AL16_3uQXxG0R^;WMhf}jYjgdjXY0E{qN+WYHs%}pTnGnme=n4b zb?N*X{7UF8o zLk_;Q79C;hyd2}kB_|Pa9@>#(s|Z;wH5a5XxdjuBv7PpR)a+}nen5vlAqEq^B+-fFD2sxbSzZ$?irFw#U@Nn;n zMT>e>^%Zpa+~YE^DtCg_65gP>l_}lAE@>(0mM=f}*or@WrUKC>KrOR^ov#WS)`KGY z!+(--ZvwXH0VQL9cK$utb(231fG5D?rtvr@KuQ5LOH#MEg7dWc`ag8`JG&re_*-Bm zmX{uAscTn~qEZk(h0HlHSNKt9aQY|P!Pl~fd)f3QJvJmbfn#QDGlA_?3F9~g^6Sj` zn%2;?bj^|eqhX#_p{Y>YK-F*YS}Q-tn}46U#?4cp@9qP~lq-owB5s^=0hl01Q{hZ| zzXcyz1vcpHBd0Q=v0JVWi#6LKQZJK7?PlO;DP$?U9;<_7v)qES*<zu8T?Ge zQ9Z*gN)4Nd>r*a`Uhfo}3Z2baJ*Oq0B?Bh+*4k?A~5TWio`_dy*OvR*N} ztr;f-q_XKbgNocjHx|1py{7O-sqgoQJ$kX@c)3dt8UE@M-Bt;EN;IrC9yhKZB&nG< zggUOJF#2jU(qCj^jZ)YMns1N&Q7OSu9@`#z8!MdBe6VI$l&gx|Jx`LU8`MWaCGN-} z^DKvOLDg@@N!PsqB@A^Y`VIu5zC4yS=J%#Kb*mN25=nmT5@u{|JbyVe4`g?^h!&%y zCHb9N>!qSfpNGiphm6S z#JX-j>I}#p{CB7q^ZULJ*vWn~DP$M5-TSD-AwLF_2SJ+DCy)kg>kDX4Ne=o036n_S ziW)x!F_w-kheX9y>vxSf2d>F82t>Culu)w!0~f%u&)9}^sG@`YV|j`}6M}7cP2yn%*(HM5j1&m?`!gb_$`*Uwxho$KB2UEXA^HFgWY z80Qx0B=|SECz)CgTNHrr@XKLP&sky)e3D5q&yM=5GyPc??Z4X zxRFme*$jGw`7X|c@t(!}J#MU!R8t~bJKOHMXcYLWaOB@w?<1<`Z}Q{!3Tl&JmCu0I zR<1)UEue)jAg9RE4{w+t73()u;cIN&Qql-U@^SQaK$_F-??B*~GW)>|;AE~zSa z58sY|%!l~;0M1*T7&&PI>|bZRBdfDp3snF)ra8!+OD(tbIVvw4Wfo|5is<}6mG0g3 z2}-mUX()VYKC&eg>$PYu`f71vJTC<1=4bpqNtnT_cQ;*9)?O%NL&ipn#x`7$HV2`v z;c1K_Gdp^D$i}#}S*dq6ewH7oKT}EE@4HF^b{KrQCXvGCwL@vBKRsi!nkxw12pvmF zxB**fG$m!Ww%JLPSpCQTp<2)Xn(UGCkc?<)mJ;EMMxx@dz=D#=a2b?}mIe%RnpOoh zuk*?c&r235^fG{fcC;VAgeuw4s2dY58Lm;-lEYU0ZLLB|o?yun*Ce z_H+!7xKT;%*M;QMRB$hu9-c*4ZL5@K{L1hxf-ha{URFoULH(TAjl3=9BK#!PC$$cD=eI48NA>ouxG9dV!7Xur{MNtEhSB>S3n!NMW9oC2R)MnZw!fS zjh<9Fe^p7v8(FkG4uCew$KZ^YgwAdSifDnO6YBC5b1r}&WF zA+5Togr?TC4811$cFjKkzc6?oW7fgq@l7z4br<`h-EGhJQnRqBdXZ0M;qWr7!R6Nr zfYBOKB*POjvr)C)eKIh`YU;pVW@a#1`1w0i zcX%G762A*!4;zFcC%2S73h#$115Z@ik^yZW8ykcQKz#PsF|a}8IKR76 zqSz4tTQFpqjBBvi*(!E*r7+K24?jNhGTiZ_Os1r-&^tLmm-noxO-8|lrDN)9k=ebv z{n}^0v1x{#OXpf?M0@g@)eMVq`>$%)tl%Ni-zcV5Z;$%Sp$8i$go^TS{3P^_CYBE& zj-4`606o7tA!q!I)YUG$QIx_*=x3{X=7jOt9tSHD zz+a8)a9jRu55aW}Ek{}9TE94!y@E3oCu#{2c~# z5o;fNP~pAw36o${<{cm*Cv1;`a+kN?^BBJ4l6mAmh&~)+6WVUc zLY#~{%^9OueN?`AXtJ07>H&CYCiMvEmiP3xe^^Uy@qAqPAjRVts$9%0z!KquHYvf231L&ThKiyO#mq5^M)Ev<~Zj#21f1f?hG)xHn-VoY*Ii(E1JsTu9kiAojB z&FPl_MapMBz+(D=P;x&>qfHj#b$u!VlM z2MmU*a=!_Yu&>Tw*O`8iv2wC@8acVcY-ffL5T~jyN?gKa#W0qoQc^L~5g2T9jpFMd zBo8AnIZU6lh7qYJ?!7AO!ai5= zhyQ@?$t(5D9bY1eB!EMIb5MveB`oIU6b^OxQ&&=I*x~FU=w|U>^*Is=4lp0w0l4_m zY1@I`>oCL-2}zwlF!@K=rJDTcE%xLr^ZiYf{O-yPZgBW>yf2wX!LGC&bjYt>-^j#7 zSz7Ig#RU`5f>HMe-9uVphLLqg?nD?!dQ8nT{|T9X^j=3ySYGDD6eJ;IQp`!xDSa6< z-L%?Wu=lILhtpX2Jcn&H6iA9=p3^`y5`BWDD2V<+s%k^6b`Ce0I=*Bpg!PCTaes{= z_E?8}4>AsydKmy}k+}&6HPK?!$u#z6h`*NTILW-5exCb}Z9u|(%kS4{lgdun2l|2SF4;$W(h(23vI$vcm~J#utzA66TZ?x z`N?(zgx~4cx@SGuK6DBMG1)w63E+(r+2NMIiVC|KLd%7R5*b5~d_I;PO7aNc=JAzA z=Fbme^cl`&Qf76#iYUGz?!I?b-lr#PiYt_Fs7Ypq@a~jq#4S5Goatr8XsAl!RbZEt z-fgpOgt;oTqX=%9m#@hJhsB}&oRts_YXTLK&A+u9r6hr!c~7k9Wd}AO%m5vRG^&1J zG<8%48uSd#q4TyZ&R$oJSUcc$!_;yqtjgI)V4x3mcuU4hC_xI{DvDJDdjw7fMhtq= zSwTBl1_)6&cX!G0)1tnGCu6D)myJ>B?o2`~lpqmdk@KSh`vM}) zXBl*NwR(~l#43&j!h9Gl zc>swPpkwm+xzMr1f-1u@*}uIN&L9jw{#6gQDDJpRN}NJkk^xwcp4{J8m-W`JK<=he#4}-nE*DdIHq`7>U~L1UfCnsTcQR8A-mwC-9n- z>S85rkon*}3TKzMEh0~P-W1zoAhiNAcOGJMjR9j{2jau^7+hAKxx zQN&KsfszU>UsoQ+>5uJP!9XaNH(5UhcrHITg>Dt{5V$zE~Nw+o`oJ% zhXdsn4!Yo44P-pFcAI-Ar+J~fodHDEz|icA(*v;>ebLb(r)$$=i0U~mkY=2CNUI>~ z4BRg>b8=kK(A-D|@Oo!!)k3W;;%InG1ua2#a?Kv51H33t`>mdZ<%VfmBoPIAZt?utZp*zHo1giM(ZaGGds& z6&md{!xZxkvq_$)a&uUrm=)$&87m1|Tpk21Y~0v*^)CfJz%{Amu>-EBRjJTOk#?b= zM&kpGjp>_KF7q-uUCH}#U`$a6dipApT=Q2<1L2Q|1?^G}4u}ibI)2c*TkYX^o)})W zqZ8#yi!1}5(WfJ`@K7Wt-CI|m+RN;DXbbfePNQ&86s>^Y=JVc-a;0Cn(Nb=sPO~C3 zgTPgiNF`3BbVPak7CD4vIcnNs!wd;k+bPzx;VpH}(P|eFIMlI%{Uns#6sTxDSk|mh zgHhOOBf~L_{Nq%;5!iFHGlsvFTL+j+c=UC56D29#z_NxqV-s6sxt!zZrozv%6?uJU zWx7g#s8GEqgIt-3>LI-*cE(9aPNZpoevT};yT+K8ck)A<3*u!BYND}pRqP`{z zDI;3Mqe2|NeuEtDlIuw}AeJ|Jx9ff?s)H{gUxoe5LMVD(gP8w#&%d8e#PICRH8HkU zF$JrO84s2z%-u@&#VBY?Z{YL$^;j?ALrtaQfr7NZT}cMU%Q_~fzHqzfB z0U~%LA6}PY?L+S1=FmCm{`*vMOqBJK+MrPfj2m?M`bsn1R5+154?I_vqKHOyBiNQ= zq3Z6;IIKJAc;GRRPXW9x>1g~XX>0bwt2PuhXt2Ju@kMbtKKT_T2!X*|2@YM1w&tgw z0G>i7c}YlYAhh3f4(in$NW}on@c=4a&@#;F-y3dIxj|T^Lw)sczC2YU6FbB5Q7iba zjZdB%9c$o+kZHi_vIMr*Wdm-Y;Uhaud9E&R%oCFdawpOnY2&QhIDBus2#6%q5zSlb zi%pbT76dSC5UWrXTKDB(3bk6p>i*<-kT#M@$C2}jknSw8h?FjlYii1w3YJPM8&eF9 z(|0Z^TF6cbPpMKLIV;jjBwwDso>zxtG|Al5mT3-l)m-va!Lo1`4asW2#qDit<@0-N z-|dmk{!i>+6qmfKiACDgC!l}>*SIA6-5Cm$;=Y+SYLrS&~W zB{eGIbo6-35(RSQWy^XO0{N16ar2R;pqq@VUpg-AhZzPmQ`614p)LTjyQc<=?8eF2 z4iV;C4Kt;g%O8t-&WZd8Fe9~--6+@Ft~|%^evH|;0mrwQeUJ8LO?js7O|buqDP;Qp zWeOSD*f|L3WlU_%oXwdC7?~JY{#*SorI3k<>3<=GEy`ZnGMjj-1%?l)>>3=LB_|n} zP$i=%J;|mhP+b|Ui!05q&9Awp3uVd=zFmqBz9^8R#10e%hj`}&iXKOvBBR8sCCG^6 zX4etIIz8MC=PtGDkuyU*XWm-B#v+G?bjaJ-U%`%oT01SQu1B4$@FpLtqhP2EXKPApi(yc;31!8{WqLC$0aVOfxh zh?B$rkO8%#>%)V;9Y9IU#mhdBZ00D0OC<+iP3`t0Ai`)VE#kqMLvS}o^qU*i+6nm6 zhY#8Z27riB$P)xHa$(W1Y|H_t@V5X5_v?r&Si|?x(_P%M>E(ANn41-3w!6#hz7H`7 ziPQgtX2ou*#)HHx!ffjz0-@Fh(0FF@GG{Yvd})81xl08ZYwH#B240F2xkBI>=||67 z^Qyt?krSXv`y=pH|0xMx$cc^=Dz|`>&I6BiHmSw@uA9w) z6&2uL;LN~xqa{9yD$apO;Q$&$5;qcRRLvJ8pU~g19UbrO;DZ(v6CPGNjuum$NW#(} z0I7o0%Rf$FvG(OhR;xrWmm_u?@@5jDu#@Qb@cNrTZ?Zu2Ec1-0n>u-{B> z>puaItsv5@v}J2ocO+HLieMjt><$DTQNB%VMlKsf5>rc>-Ln>-cia&KU!j61DUb;U zhg1Yjq{|f`9yTBZ?){oUNZ9Li)mukmRGlmzFO@A1jEF=nrl3wE?tesx6eXCAV4({M zEA|b*7e|1Z4Ajg;hF2~T`8W3-1ai-cFeFK8W#&Ab*dJG+=mbPBz^At>`jQ}&h}eLk zR)qbPoe+X11p5RsUO#v&)>p50kc*}Mr}EjZaeGTb7)Ol-&|cKA8MQzjfGiU99+wIA zIxSqGGQ5x8$~6!$7_5A6tCkcsNeQjwn0EG0{7oH)wdM7gmxFP|X^)`)GypEk3xQurY=?f|JHMKMKDt9`G z-0rn2#r>vfuwqrmdBL-w@Vzx$}8%gA^C)1~l1?mM$J7LwQd=0;b^$F$`x znzZEHu6@}}ec(@^j8#V}3BF0QY^e)mC;bjNffH_`K@2YOCJ6W}52esB4CS z-y;M9{Q)~}wwW$I?#h$-Wyo;K_pl`u^-`m!+3Iek89U79`9ojQ3FHNF?4z7A@4z(L zu7#$4(Us-T*_>^Z?evZ+FdP1zh7U_eHzVFyE1&T3 z>$N6aOqjh6MIpeay3*AI+iY?`=&zTn*HKXDbe+Txr|yZ!Re5CFLF&=)0r%--dp3MI0NjQ zpUoInrNh9C%?Nb0XdPuKuU;n}JUL)(N1pHX7 z>Kcr9f(G}5WNzwLmJFyFu;4zp9NQ!omB!=l*%$$<)vEsHH~aDoni0pAJ~bT|iCJXv zswpbBVc}B;?B8r(N7el(E-w?t@iRF#9`SEz5jh? z_Gob1K`aA;#+t>euKhgD8vuTdMAxpNO3X`h5qpwM;7D8)$v{{D22B)Knb9uMrDN;k{ z82kh8wK2-nvT%e&emhvYb(L8{-Suev`qD2vd@}B)HYSs3NoSjCJx1F{>w2?tEjGMG z`aIE{S)a(xeM#jFyFJmV&-ZwQB!o2G1=~6 z;WJ8oiMAJK=-Hh514>35KJkC$qyK~(J3~t-9v%XEF$-&F6UTp}wSlvVh>4M%vB`fk z5F-H-CkN+$D<%SFRt8q4|Cax6Kw@TNXJGta38~rDL_I~TjV%WWXko9gi@T&txKo2A z%Jv-L{3bcxmZX@wxVQ_uxU2Vg=XASs*XhR>J#CNo0naFwF;7ykuy}&N+R_XrxgMC4 zxt@XP5$Nbh<;prnfMyDYO27#nn^+uxnwXiH8woU|oOo_*eF2Wd^a$n%)Sgod07W)? zpjRK-)D({d%AW)Q#2JKI7H|X)z$3pYAv26O9#p{Q30=O?$|BL!RkaL|xU$-a*Ad?u`imUCPl~e%YcuIj#grHxAQBanoZqmh^2*Fk5?%orbwODi0Bt88={8oF{)>9KhM<*vE7B>e+(`F|Yv*xzp@uDYP6khV&UARLc70hkB3y_-Pv%*~9S#&0A!4cRF4p6Lll{arn9`hYMM9206w zQwwO5tI)?XAEUH=UiLK(Z~D)xzE;rHmVoYW*x(eBvE@tEYbo7}%msloIC_5b+HZQ0 zs6?O1iGUdZGr$=DKxdXZ!?)DiYWEjX(-%?@_Wrerof$j>bE5;0r&cE5fM0xfcC>Z? zfSEXYe%oqZq#rTKKg7ABB!by}Y2Y&ONzK_JJ_FbP)Q&NI+)w$J8DzhQPWD=W z4G%ym9}_7lFo)Tlr2O~Sdc*HH(Y~QUfuRvp@wYDbr>!8NrX*&xPxg0G1o$nAwk0s9 zs3u|B@0=!wgl0RSR4R3b49&0Y#jc~d3DNy0MDT=7;I83|RPnb(k!^1sbeBK1ua^EF zk1r*eXkciu>CuX-cLeCO-oED^x8LiLeK3GACXf*I)>9f zF#yTn=*;pUUKZH22bY`#+)wf) z{ltC%!VvK#rVT(gzZ;G%bIg7K_|I|M*kNUC$IVrsRH%0Li)IOk{>`Tmme!L4YC3?&beUm|0>ki2c2UCtpdG;F{&Fsz#jmRGc+^-!k_q&TwV2%w6^wT zG%^HDGJFFES^)To1LVQn?P_rLzwBYMIX1rOtkUCyqZ4QG5BhJ~yZMYpgZYf4tzf8!-MVTm+c?^CL4z1Q-y~2f+jeFhpwM` zfQP7^xqye@6T0LlX!q*FbO?mp_|iPw^|Y65{S{C}v$xhl$2ODpy3_qN8Uk|tl=TPB z1_bFv`I!sSGJDNp<|y9oTiWBg_Ny8gxQglDo%Vx;;GX=I_~pg3-qYR#{JNC>iK9H# zW0LOL1)qVtsp^TD#CK`%Cx!Eq!CV_17>3(A_nQTTZr}7{`Bi)G3$piz{&(Bc{PFtW z@LdPc`Q<=_vqvD8e%-b1FV3<4$)-g^v;Wz(;Ormf(&7O6#Za(U%HF>Eu?HBnfBz>l z&OgILn4Y|h>Ie30|N8IU`DNo#w}0pMjRz2W0O9`rdA_CAK3`=%sk>!kH&GADYGP<`vN)mzVL$B1CJ}J^@QwYwK>2umRP6rjjr1)Tl8GW{XPv8-i8L`v zCz}$&7UO;710g>g64tcKTnS!0Is2Z{oUm{;Y3#T1G5AB=!`YgFk=52uw2?b*LC5}3F4(5Cfmo1 z`vlEcWT0!*{-VZtM@z=xf;%h)^x4NfNA6F}STKS}?@=)6Ng_a{yAFwUTeGc4w#eBW zqw?6AcNrV{@9e^1p|VB*xchS*`N@sP8XccGWCbl4NfbV<>ouuX(qrj{lxJmeSh}Cr z>K>i{)kvBIHC(vlEN^l}N7^Z#WLckMhmKd}`eN%cmYC;`KzmIdy9k6d0ayusE3**n zX$a4^aiSp7sjSdFb}&kXiZ*e(0h9P@m@jRdwDB-9aLu%bsw+_gx^nO&48l7JHBcHS zZe@}-DQCsn4MneGkbEC(!)I&tom5nplYI>G158f^kp|T{Vcro0uUC|GZO04Xic(bd0%G;Ys%>K2a>?qEnH^w<7P0 z4?|-@Y14^6kZggjZ41}exA)qi@z!zRlauR+5zq;E?593?LV7u)2Lum6b_LP#B|7M1 zC5Iyp_(TnB&3dijK29KK+Nt9O0wHB#%4-N5{Px5f-X{6&4um4s6r|)tCMy%X1##r% ziE67#QeQpbB=8a1YJz=bp*v~Cxm$fa;&dq-aHvcE@n`Cj^@c!j#oE|%nke0Y7ITs~ z-_$i#8@mO5PD9;d-}&j^szM4}9`Y6~m+UUoV~h3!N86IzBow;aG(y3&xV)jpC4UhO zja{lgW)!P^3Zi`dbuKUM1Yec&L>An+-ZdaK*r{DYUlql=dShmI4@_{SdVTi((Na*&WH ze;INumU3+x-8@+NoN9wR)urZX9n4H5a-wV} zc=FuY@UIDuHdz`g{v^Z+cuL_Bp=2#aMX|(|S74Z~I*L z&5yu%{27?VhES*F^kH#-r}i;_Mr!SieLSa*jdX_|(Z>xm$?|bA8qH5_UR!FKBmekl zMaeb6+qBt-^pn`kSbXhH9(Y^KvKnI+%g^?{!T{U(C^EhcWBI-n&kBu8TIP&g19SyR z9v}y_a@gM*!NvJO?d^moxOk2L+=OW|8WVGI-KY^|;s01=Q5~z|A z|2iHNzgGc=8E8FG zv@)&3xBJ4^FX<~A8yj7dH6I5xzNaka`}Ut&$xYlaV+kYc_qkGZE20s(u&!>`tlMpp z=M>6)>HAY4nf*B^9|4;aZ^x74jRan{L-JM=Sc}S)2{M8WbQ@j_8*{*_SN?ShNsF*z z^vj$hMX`0fgswCl9U*16_Sp&g?@YbZ?+0GJ@F9t+X?bt5IuAL!$mt+eg%GT}W)TZ^ zD5JNEblkNQ$Dx)-El4uy6k%1K=(1M6B6k(79|XEOTBpETzCS_D~)r5FlE#In3wybK+&Ee0D%JRUMYylRdH&gqZS_3TP#Ob@XyUkFR5KPO?Q zybYfQQPZD=v=!`I5gubqfGL_t@VMQ+G2LVyih|IL<6qE+3P!T2kGJE_c^SSF-L|12 ztef!NlP*K6mH^r@JzC82%_D6?0&Vq~V1)T<=Ay$qwC$h*d-I6 zf!w#QVJB~`)fW)k{k=A>1qSl#Z6)JoE5u_&?#~(yM>iwpCh793`(;?Sv%zHKoXHZ3 z%U1FjL+&h+w3p@10{fXcmh(%{BwE;($KypfGI?N6+5m}+HXl0N)G<%g{@BC*fgL~W zX9q3<(Ak3IL2h`ZE^-}9HBclSiOCvMX$*0?bsUb;Oukc zJ3h=3qUW%jH}SKMalMWi=_gj0n-?~(@;>_9l)h-vW!2K z&kmFeZ%&rI95o0s*TNeRb#0h|G)r6GK`DE@cuGrL&2y83B(f~(u{`1v!ZO?ACtTla z3(rQp-+PoPW0m1(txVo#i#{kzjB1Knveh-Z=PN`7m-Rk3arhFNE;*Y6_7j;IeiCzS zNXcVc0jD}4T99#+O>e(=%4z{qCTH|YCcZ_Ud7}!P@PPmHYFWR*OOOnyl5HUBZfLY! zYm!l4;Q2B+-238Cj6u1b{AyyDlYE!9zmQ~u40RQTvF?)K=*Tqhj?Ss47guDJ*K1Ke zvab@m!M!g2i|IJSyj_x>S3b=3-O$rYrqQK>?$tiTu9Dz}&%H4*L0@ujw;aQyL0j+W zK@|2PTh(Z2w5-HySneI{63Q$;vwp7ql@b~?q%>Tuy-`$)Nx{${UNh>~IF~$%PKDIF zIcv?eEXO+#@t2fw6czF7>^QH#7x3P(Ga3pU_!(?p5))82MqSKMB?f>{(`O}<(%1w4 zwnDOGxelapr)0&;u-8(`X(93E^F~Z({p&IA0+hqH)EB{gq5L*k_u&w& z3yAOm*m*!syX&p6S|zuXlP!>jJXp!J+Co>Bi5bZf!&f45*Hr#|Lulcm$|TA`aC^$^ zb5D-lqU)@kR~3OcnU}fdap$eSAa4KXX?I?bft}YS&-yeIq4fU%M?kp0)jQ8&+cm#& z1yNV3ELe#`VM2}vy0DQ=ID$w!q!NvNAvB?rOX&FY^I=%Nq*rUlzaVV7Gv(Z0sQldB zeNHD~-#n3jSlgB8jMcI72pzd5j)*Gyk3$c5` zaJ!|#jX}dZG&Zx6xHMnes~Yny_Xad>6WrSf!=UseDkU0iYF5~sAwfou4ww7Op^t7nngmv7mdXM@7w$G;L4UmD(>wRX&<+B&K;Ub z{0$c86Cvs?Zz)DN=N)PbQtQcwR;wfU5p{zl-a@=2`CkCLdfQ8j;N+2ICX0v37(^)K zFbugzND02Hce?^iC{%ww`)YoOcERUC&7$2|o=IH`sqhI8Sa@ILjy&?_A>o631}q}& zr&InFRMSNfmY{(Cc zbnl9&Gy?V`?U2*=o$WWmo$c%}Hrz&)%yP4l9q}auiz`a_y*fMDuwQVFf_F49GUazk zwQ#;zWJA?O zt9RaU0Ql+^K1>H@?WPsqild1*l^{O7G-juhq4?4a!#YdrH}&h0L6g?DtZ6Ofl^KUL zYiS;gY}exY;615jacY#uJx{7hGc57NzG5GaQaE{x z2+=xhRDPS{lSGv@NU+akZ`JV1w}sr)UzFIn4x$=+>tYS!s(4u!?gHKj3GGu2_i-_x8BKR5tO0ApA1z= z@3{Hcw5?gvLv&n6tOh~LP(66}VwJg{H|t&~A&7m@9@s2*SpS4ZW`#o|KuxL|&lu7^D%OwXwrST&AoM+Bb3QY_qIBH2Hpw-0Lkup>a!}&2!%_J{ z#YBO&lvJ&@`7#=BkAcagt@ph!_gU$O=4sJ)tZP*tzox_x4k?^;A%8)U$ zcR0vDo~ve*`t>_sOF(wvXfUj+Ra_p*7Moj$E(#s zl!8h{Z>OW_Z6IZeO~NucASQVptNcV7v=lzfR^)Jw)RH+LY=;Kv#d7i7`8s!V`b|Qe z3bX_OG)?O%-K;q8&((?DDsNWn=r(CTtKmK5<$1e-blmrqg&fWjJsVG)B8(T-D}oe zqZ3OPPBoIs*&qD$8(MnG9b+xt_gnIiaNzy?)pG@LeM~hbmaHXj`5tMBG_a3s#ov-e z3)_8F|D2sKQ8G-)R*91eM%;`TSv#jpl=ZV=P$g;%fn2qH6@QPSHG7NXMIM#ftD0&W z53AMwea$U)AqFpwN7hb-Yp$PG@;g4dRUz$o@uhVU@{R_>K`N`oL$Aq&ztI}FZTP=X zW+)u3aeTA8S51SF%6KsdTX}KB5lo9!D<4w&^a>G)celjXXWmGWJg;DbPyC`}PzZ=3fG#jk%~R z6?Pr0Xs{3EB8MQGXEijJg`W8OC3d_RT=nO!hRK=ZuLozagi zZiggCQbds3*TQWCsx5@#m&C}DW2CR_Gv~O=!ePc@`f~;?Y58vN`^|uk`#tSbGRR88 zAl!JZ!zHAlGU1Hbv>BQPP{n_)_h-4?Hzy!G_Sb4!oft@aN(vT!X@Mp)@_c4S?+=FI z5gUqaUCt}5P^DI7Df`$mS=sC|-<=S2^0R)kq?O{aldXv~VKDjdC>{yJK8wot>4$-u z+xQ6fc7-6mfQ}?#xr85lPhtBB{3PZ^7RROT%+3{JZ0Mk&0ux8=N%~KOK|v!)^4LAm zsQ|syLVk;$7LHnJ9;!_YF770uZpA25K2rEY7iwe^ALN>+QMn5vE9f`uEqL)X!Hi=LB#QS0kJBMIncxpU2SAm8QXEs)W=jk z)>$GhlD1F!KpUL2(8=YDxF1NDvU8vmiRg_cH9;g9>PHIXky^F;PbT>9F|QUkS~+#0 z(G$nnzLv;0g;$adv%IWs&wosOKtdS6x?7+yQwx*umhP~tg&cWLHj#cS(tdqeLLw1h zzTWHJLJ(>L4<};YS%dNJvs|)N0gr#7{Aa=~TjI?~L+G}e&`JGj9Ol{;d8Ou`rMU4S z$bF)9KRf77H)*qhnJcwl>e)NUBCd4=CR2=4R+O+;%xV``$wUIpH0x+#7Dp-{N?@0( zu)h^d5`Rb-*(2>vn{%BszPDjO`^do6z-ZG{sRT0xv;X-Ff=sxK$^|WV3_n#NwLZ(} zmH)kZ)K7h!bzY1)V}nXa<1XG}@2!!+lnj`??K350LDfeN%UEJ`Bxul8%umFZIZ>>6 z9t9{JPI1FWrY>BHE2eDQ@i&$YUItPl+wmaG;$G_9l*XeFwQ|ev^<9WbGIUHVLn+Cr zi%q%nI4HI~ZCo5SsI^y=BQy&GS}L*-4)vN0wROmXG8W&KVIznckfhJP+2GKrpy=g& zM^fJ`h0oBc)(J1$KQB6BXo~h2hg%b${r*0v4QY{D0;Z4Q{RS6=G;0m38nVQBb^mN; zA0#`=ncDkW)dLi!Vd$rjCo6n>iyl0JcH#pXG*K8vdJ*QI~{z!Xe6=)yL28u#UZov~$=R3h^TEUpHq<)_yMV zJksJSjvw*72~MMYL}(w6&hbO}4jVV;fk@3fDxyBhtvT>L-lhF6U_HlTzmK@c_44@( zLsl>O6N~96#a(VT2DIUwpx0*DLw!8$4QsZ4&Q$nzw~0POymEC^hdf=hWUVc!DxcC3 zRz9o88kOON&zp>7v-)BQ0jzd8n(R0te^ki&M$#9E+UiZ5WwJFrbTPIJ0N0oW`=Lk@ zPB*97XS4*hLXM?%Uc5s37Y*hNcSj?D`)caaI58Ql!eXLI8@;--x90^qp#eTE!F9*> zIy_$9clHrFMY1|#sS4EYaM|kN8q1>m4^_K)j;VdNdHe*S_|y0NcZ{Wbna>C5sYKi~ zpt1-}E6Mw}Cb=atekPC5PWWh}m@=U@rutdbcn0;zU7ojVo_&&a$d~2iAtyRN19aa7 zBZIbJTe+te=kbzrGZVL|V?v|tp*Hl5^?KZ>*gkb~E`2m_5TZCSsFXyaV$!&hniaea zG_!MS*=uQIh{_M1ZaE=eY@ZA*rDVp=Bh;bXj9t8a_^~f15qOawUttUkt6FakU7VIa z9@vgY!#P2EyydAmA~)bwNY-XYeJXqrpCi`$-qdBCeK*|$x?`!0Ah2B{8YL0pF?)Nm zg~v1jO=BF66F;}w6BXd%>rR;&ZG#^I@bBHGiT9bq|=49=SrSZOZB#Jqgk`)7mQ{#v*^VOYlIlk zIz$RFSjG0xKDrfFhYUP%b6+)*tt*OM*d$Z_A2$Gz_(y8-ZYL4z9o>1YeZ$@i>GCD= z`jeSsgr^%kwp{XT_w}e{($pQVyAzwvLzi;Sx{0sR0r(3q{OQkz_D3t*^l?PzW>z(l z4_I;3H0TQq*Q%NAlXR7Ui@EODf~VCYy}0YOH+Cd@o2{K=GCRb+qO_)%!c6uOqp^@F zbvkU=#?sDqu;iehHrQ+9dAAE%;Wf+Rfv)h)qBO_NZ7jUdr9Ix0J zi-=lzG3Ti;qX=GpFnqTu&$MFGp&Kh4hxpR#aR9pC2AOp3z%f!3_}?$?4nkG&Xc+Lb zOgO>EL#C#fQoRk6DYybCY%uSZ!R^eU<*-hDJR&?Lqz@{AiGs6oCbQw{+lh&vA$!8M z-R;h!lA`ZDpJ?oQ5TT;SQSwyyhDnyticPYz#uw~B%qN`Sd~o>ss7JehiYh>a2z8l# zf0ZAA>FQ?p1w(i&xKZrWDypz~Zc*R+C0J&qo7Mdu!q4ljjTjwYOV~Z-*99OIJp}a4 z5m2Z_r(`&S*W8VC4kT?t$+&xGAVm%_oP{sHmKO^-!^dh}AMAS7ba6V(sF3_XdE9ieSg;BjE=$0_k4Xfl~;5jppp(6MI@J)ldq*iolp7k*0M!RVRLp)97>ZnhByp0->kjdwDpjMl4lpcYof$Va&;5n`94mg zwllcyWz@tu*qQ}*A=K1OzA*BP5;?|%588TCzK&&@F8^wYV z8cuvtki2fpL38UQhl*%NRO04($RYwud6)A|f}me%eceFm%Zu<>jv1wl5XU{V-lz`N zS+u=v`MAHR{c?b5lc z2Wv$2PP?Yvi|gXFAzBw>lH5Ad6-F2Jg0+72 zY{A%f2#qDgu93bBt**1^3x;jF5qn3uAYZn<7`wqG45JO1+2-ppdTPXBhN#?li>p?S zFs}y*c;yb6Wjb6$>(2m`PSlCe=_O0Eoa)eewyn2*Up=8YY@!5I99ZteQ-)cC%9hq% z?0iPPmNHxJMIt&e@Z+@yBR9aZn|jJ2&eb1eHge}$K^-4X2&CR{3#uwy4|1-8W_=#u zyBF-BEy;ZS21TA@WrsW$=grFKNs@nb&x}Ly*h>W(6Xro24O6sZGVcNX@U?Xhk;hXA z9O0?Wl`j=%-?ix=WD%q#KVNKA`0bkEBr2pd1d3*xzb{20(!WZQl*zOoM!|9v(_adA z#pd{|-I(BlgkMMZ6CnvM_Oa}p0A!H#;*`#WQ;u~|*NO6#?7g9!cKncGQEyK{$;j6e zb>nt@&cJgfd77h}U?CBE9xg%{9ZI_!oaSU}ghVVGp`$fd6o(BZHHQyFm9`|~YusC} z^$nej2-dkPS{KCjCQ8zVr|e^vS92L4 z@*47D<6x3l-FcD*#WqOpp|rA>R}g(7qP}L79b=$Ptk!xdS!>Lxwb9VKO3quroO!kj zl>_;&L_M@lzFRvm60^pvC@w8i9w&G+UZ^K0ou@<6+AMpXo#+*UC{sG1WPnO~4^n{j zY|1enj!m{2&^!>sB8F1$cT5*d7_|Vp!xBWRcwmdw@6HkNcJ$Z@!1~r-T4>auLoK$oe6Q$mssU19CO{;YONSkmfbhgeKSZTLjAO7@;UB5AgWb^Eg?*$QAl(*Y$h%{NDbyH{TB--%52BgVnch zY2KH}bftxS@!scR`kB)vq&zEG_$heOT`ph-*FP9e5MXRv@=ZdYf5&&KP$jGfcAqQd zP;_WEe&b_1f4NUY1#x=)6-DDKjcGy7IRqk8rNH5cRGQ$$)cIDr#-&kpDvVa|%9XI8 zAo)od#?GtWeisOVv56NNL+;@MhYXy2#dHG&o*9Ui7^pnY)QI=^ZICSoks-vO&sb7~ zLLJIV%cPZULHTs7EgX7dR6=DBY_Opis6=eW8(Ak`YM zv2h&B`Ww`BED=PlqP>VHO<|59yH4RsDCxe;_Az#3=on}ToiI7r771;QG$I$Qj;E5H zjYet^=I%<{xK?;5o!gOha0j&)S;$bl6Js!}!N#8A2p->AR{Cr<96x(BTq>STCeNo!-2@Yj2F!AU^i9?gr8tzKfs{%aeP*da&7;#KwrHHcI%L#Cc zdQ1vS-b|QO^4C$M?r|?{e{*A&65`o9wK+|lsY-7ReJ4Cn{ISVVA(_{7_7#Ph#Mk0g zku)y3^RRNOa?@*{h$K#j^VJSM3Zk$`ugLEvA!hsH;k5bK=DrcdLJM5Xw`t3S5799A z-lwJ?gk6ds1bp7I=}lmc#txe#27dENg+8hn6wuEU#J$XMbYm`e=6U03+sfo$(X!1; zr+4CzoGAb4!(j4Eg>vIhE6NMaMrh#-bnK71OucRI;$=E<7-v_7AWJJ<$dGNHYM3)o z5H9$*XD~$tT1lS@P%cIJki(9xOq#Z1>{E&b$HPbXhRQjjoMsSeZ0xO_Y10oWHupcW z=M2gC%LgcI4XFn99Yzw$R_QT(;JqQmxEersNuD{Z?^(jE2#kljFp%;5N?Xb9V z8(g0S8R-9owj+w~)vg6O8fwJ;OMI3rz6sDws)HZ-?zAnF%IhtD-MGUf3+o!{&+BJ( z>r0r|@@d+#Yew2IUT)MG6RDBnzHJqadCaxeq*C;Ec0Vkwqy|_o33Vw*EGR`|Wv?s` z-ewe0xrN@ZsKn@4VrR(qX*SLtw?`rvATR~xDVd~LiPTH5I_N#{ZSxrWNkPTK&yi;z zlsaCok1UrqGYJ!jq1SiV@hR{k)8DwMBn1bMuT5k*i(Myhi4x$`P>C>arW*CP9XmtE zJ$bV+)FJwwY(Y&fq8fQ&ylS|9Dcb$MjAKZbxsXhM15iDN^Y4|SQAyl*4*4P4KqqLA zmb@-bC|{V9U;#TTR8%Jw^0YC!C&Nxkt@jp-&-N*LKOO^`wM({Z2DI#t$tZ>qwcchd zxU5%Eq%nuafg{-X?g!bCmvL7K&?FQ~$9M1ifhW_LIA%BJ3{iLmjkaqZ5?;o;9wmq8 ztH)I!s_fEOCLRu7XPHrmJSDOs!H>w$7N|_iAHY~gYp?lglX2xt9V}0 z5Xc6PH7JSHX@4WLBBEP$qgAPep58@m+QKS4uYKc>=2Y%^Z5+g!`$l`%l#Hbt zo0F9vpHt{RgvOZItUMj_=OBmldY`D@*R2|-Mp;iQ@=q5NpP$(CjqdL{8{|tbHP2jS zD~%s_#?Ng_Q0!7j*RtGYhE)?Wro-KLt*}j2C+Wl1)SA40D}IkYJ|Hyp$qM(~7;X;XvHa2EBo>C2DkbAXiXwd>lcD%ACIEkH-I)e$$L$H z$C!oogPM(xePM=B{GK<+qpx|bT#qsn*?KVQhR16&yx=-v6n;E$hlb%p5Z5T~Z&j!R6u6cpUOx??sdzt<5sv{OG@6^5Pk zNY9-2`u9gBi+=MnW-Fi=Q)~84>XOe}C?LwcQX(0Bn{LAb&|KM{3P0^lNzdWkGg(gl zGHE7lS*&KBI3XFZnX2NC_99qhE^6Xo85&P|uEI30fx6Rd5Pa!p4x^;4X(uIu{W(IX zo2r3eVBefws;_S^tRP19NQlmQCW=;-lF)V;S_yVu&9-WA>u!}7M~Hg3=Ci|xp9j^XsZDec8>84@aBiu^+TkFY`#yfKil;vxXY%Z%JSU zzD@g90XLg|24>81Y$GT{M=_MrQ%o6cELIkqp&uL?o3x3uH+Jy}rF(H9){6}YGlgG^ zz}6`B;|tAJOwt;mtx0RVapq6NE4LzEcFPl3b&E*Pg3l>#s$4Auxf<*=+-?9@b}nJl zUFAwQ0$R=?b6&v(Bx+WB@3sf$=3MHM1`j-iRa+BxIz!jca^9#j;O__pQ)(P8>a<(v z*W?@8Zc)O!%*>8F5m}$qS0`a9K-tZLZgP;#{o_?>zrDs*rJeCSS#ybi#Sq?v>|f`r zs}57;S9aCUp2v9Z3r>3KX(*0OOh%;@h`fYSDA#R&r(btot8!iZ=-WaaLr+XvdK?6U zEjVc!doKS$z(kg!YpA+f_f{bs`KE4i8(^(s=D>i4lTE9VYE-Q*nU4OkB%Lhp5Qyk$Mj{#H9`M< z9GlX&vR5~wm}g$Rv&)$NIRLKGmh%Za>^kRwA3^`za99E_+RCr?x2Z`6q|Hw8^ zUK?AkW-3@8%W0{!lmkqvwj4B5f34)9EAz18B~}n_Qn{VdeVnWNO~xmkeC3U^;T>xO zZfQT}h_s(EF3baQ=2f*L5yUkzEMhKR&!zZR@T zE~s3VO*Rdt!%ObbUO2<6a?Tz>K)Ui8+sdGV%sOx;JeO!w*+OXF$i7 zq2PVFg5p_PEN6GgnnM-x(Tny+r~A9oV@$Z4-bn>IZ|y9_E0=e7KXn&K5kF$%Gt%u| zk}x%H6gN8s`*Ra8(o*nFP&86HSEjt{A}cUxH$r{CKvc9#kIzKMXkQ}g`3>VozI&MnEf`3tgPVHD+&eQkQ^$KVH7ll;IECzk=GJZuYS zd#vP4)l-eloe(j@mk|}zXfnOou35qX$nO*$L<9k!!+xAVi9itr_o7~&-upx@d|Asp z)`DDY`<~RrDMyQotxb8~eCv)E#bk06YFPd5;S0Lbg3JAD7=iKAGU_$&wA1 zSZa~D2ZD3PFF-k}WW(eM&ebq0e}FKL_!9;+)aWBrEW{m;-A9Mlq@_#?zGz-8728;T zv~{lXAFaCS@?jV8w{~$DZe6K{i}fJk-I>+PAR3@{JYE|yH;ci#?~l?_k|f@S zTEm3@RI?rO#c&LEQD*EZ=q71DiG?9=T29L4SRtRpbU&Iqq$wF7 z3L@_%(&o_@yc>^to$~!`7>%-s3Yv;|80I^7P|B$y***S7!r_tt+rZa%2@B@*nJ@t5 z2Yom}Pab!NN8-q0cJa3`8W2v9s?~6^qr^g!ym8k)!3?$coImr@qfb8^*%DOC(M6l% z08gaxE=q8mGXq=`&=4fVU{_kQ@NRy*3v;=ikdX28qoAZCOg#5rdmBz7Dd!N+A2Sx$ zY@^*=9U>oGa-50kTh0bJsr#|)yz#)O6LK4CI4l+hWTM!NZZLYg5A(*Q{B7eihh!FD zWO+Rj?LA2y9FfJVYnip(E_*668<>V(CR+M-x{wtL{pV}C-DT5V;D55wq&uG^RC?EGl%+un`81?0eZn5HJgUVJ*vI0J*u~& z)u>$DjXvI{ua1(N&WM_h2ffo_V%zdiPSKHVL^r?)dD1eF z--%Hfi&M4D8oX#nV@Kw9ZYA|b#CT_GU^1~AjtBUm)q$v!Kb>=D=tHr@!nQZx&oX{$ zmbw5tR@$=nbC$i2ab>D(=rbnf=LIjc-8)a@2t{7_a!^66`xoox@kdMLnSA&ejXrJyO`X1Uu|d;=N)N5T<99$IH*0LJJ$AoWni|y-tT+iJC7d7wt+mG|M zlX`|V&5`vVR*X9)^JEr7Q3|)b*-~h~zk*Wh@|q>0s%dx~iBTarFEMoLWd zAMgTY4Ep6;MT#S-hluMeGdz0j!j==7>(y_z+H2n2__G&&pRZHbZ_S9Go%>kUJ+o0J zAFyMzov+&jb&^GibAc!=F5xaz6(`4C*+bn{bZ*t$`MK%p#N{HOqO}_n36>%~R3E+` z9`Z|ZgvR=O>sKft`=N0M)b188Gf1&|q^JZJj`iK>x|#|J*`uov-DQ@5h97u#l##c~ zmGoBi8;cq^JczWCc9#aZxnrNAf66|)jfUy&;2$V|YSNgXCga}NymLlm$l$2jt3GFig zW9xKgH29xBOuc>&C0REv9K&=dI%`yT`azZe@xqn3d8i~Ilw7Wa zVE0(K6XQ_SZaJCu*%eB0@Q=4An#oK3Cvr3dvi;nt!NZE$tkC#({pAEaAt$nT?J~j< znS5^^vL@J&ngr7#B|xi)!qLK63&)L+r&wBQs07X1kZ)o?s=2wu=3``NC8I1J7DPG( z%QRb;R_VaxOR&)6I;}rz*q0wY!}dHN80W(!!Nc}9IlQ{_iWwd{l&N=hf?u+zB*p0( zI@;hymKFwy3GrFXU2p^nuZTVUHuMKfVxFL2RUY8ExsWJoXMv^Qq< zAxn8JBe=I^g?U1EzAPhkS!#3PFLHHbwB7K^*uE08dK5_-Ehb^Bh=K+@a;Tm4NjGLh+0o47Fl~xR zjKa2)hK4k;pY;p7w)#DK?N(e6x?7k7&uWRFVS47TKUcvj4m6tNV*`&dzdnX-gHUcd z$;Hnz!UlSMC`W2`7vQS2S0UMDypzpQ*bXbcQzPgbmm2e?Q79c5&~GEbU3jOJ5qLqd2VNE#`5C_ud$Gyk9zBuvgmCDtQj-uyfSh8dvN=)uTLdV)o{8%!3s^Z}75g(_lrc(fQebyo?9xbt7Sd(TNS>jnk zwyKSgyRo*UG2G#XjJ@|m@ba}bE!(2rZFB%@<~O-kFOJ2`+L-4v;cn*h7s^_Jcg37N zy4Ok8+Kj}`p~di&g4zke=lM2y_=cAYLblFT`U}nQpj_FNY5`F>wv;7GE>`&(fx*{7 zAs>=e$MoMHCM^cO-H?Yr};`eAE1R)s*A9mi{*yHj2=o2 zHay^!GN)h&eg-Kavf_m4mwc_5^0E{i(DDqXc`_)4Ix0|4Q1Pz4KXZ~ti+Uy$jD`ul z_f)Gp7?Qg|an!Fr%dO_9-$wK?tXy?WQ`Lv-S>68jfsXU}-tC7C`!Q8l;=B{3Zmp0M zn?7uQpx>x#+_lxw_=HIaF{C47jLJ0Zb{Na&C5SPuYMuemgT9n<%NTF>*hbeA(we{e zLP5u6!QE>n%~k9xj#+q3wX~cztsZ3p*`c&Un zU6W_>b(gMIpjz~LR``*whVIAp%U#grsU9SgB=~4Ovq4SaMV>e9UMH(kVzCW&QFuc+ zkP7>qbd>of5IQHrCZr2N9t6YhL~qGrmp-|lj0O;u=bb%p1JRfB$6s#-Ho8W=jzzAM z+i~!$v{BvJGtjhF59IFkV=h6t=U?p%yPv4ZG0Tgaf4Wj;$|!#GbVQJbpmQPQL1_UW z;iQh z_!VydpjNe%uDskSsOInvv&EiME<-=(WRP}m{>%Sk;~bVm0T3kEwr%sZZQHhO+qP}n zwr$(C?e3k!&TW38qM{~-GsijUdZcF@_`dY%%}>aFV}w`-WyGxZ z@)i!zv_Jw_g0JzIQCjl33J1xtl1P7veaadr({SAjboj$L4&7F!k_LW;9O71&$PrjY z8P3o56fly{<6%Afw_Zz{P?I7tcK47HU(*~Vx`a&@*6vtEA!L7r6Zo&xH9^d=gO+7c zdeKJ$l<@>uhWoWa-meHDRcR2C9xAyn#X_~AtVU~!5x?Vbcz`IFV_(HCfiRDDUd!Rs zScS~0qqsU(gW4KOq>cR_3kF(=pQPpVKv6Vs#EBD=mwKCa`lOW+szTkJpc-?skp7@& zZkQe5AyH1a(TK<2%d0mvyoh@h z_pfw;ST`WE9(zFJZ0<~Z^>|R&>QSmaXKKN=54q?4gU!G2@jG{1by2wnH|^LN=`ILK zOM~9uO4d=it+TIl6sdDn9;x^D!X^$s>O=GW(I9m6^pF|^n}U0T(xe+n#=Lx6xW|iA z|0MR$fw`_?yrIL+SCZHq>$stL9piV?B*7&DjG_saslx@=pZEuc#~z@s(@$JOcgmja z#j3rRWy!cM_*+i(rdC$u)=x6fqBsU>cHsS^3M=%8@bvsjAO1JWTkU)71(=p$O5diHbWRg}IxNq{L`YeQMB?BLg@`^WHNBANV!&-qw}fsTo!BMtRY-dgm>ddtoSHOxYRsc; z!W?W#*|uj?3iP0wY5Fg$a%Ebt?P57p?H|l8E(6`+-~1e})=KuZqcXlR>nU8Jvk3(bUU9P8|@R$F7g7&;x zn{F}T`2(ke3;Q9ma1URV*m7UUUn%kFQuEyN4BwPEShergao=gBt^9u3*WpTl@jBhT zdp(@MaeGp>Xk~_{?!WEDUzHgK;-p%r{A!(6SO8hwSz`_{V3j3|Yz7?5w?Z&KScHY| z0&Ld~2W=M14Q0G>#Hi#t)!-`z6Kfm4A)gt{L+i0&g=^Hcx6bd4iA?k>vIOtWQ{#E>9Yw5ZwUh*kzOus;vbcjMbBmi zP)j48fT7R4)Z(Ztb+j67H2X6>Dy#H)3xRx&a1G@vgTl_10Kw=}PXqN1P? zUOd65C7d%Z)I~Z<$t?BE%~XQ1;m~uz&=a;dx{d`PwsslO3^nbDy-SAl_9>D}D;=e+ zB+(+j@jPv-TE!i^MSTK@XCRR^ioh-s%~tQ{yya#!d#>l>3^286xJt`rBz;bPpFWM-4b3Bx{VR{-wCyft5Wjup0Bqi( z7e!x!KqnlaB2(uMs!Z+m?TpH<9>4_z6N~YI^oA+`g|?M#G%(D0G$*%c#3q!szIw!b z!AkSDvdHogve|BF5k~N##N@tB+{IlApv66!-x?~>G!#9_S<@PII{`Di0tM$Gi($AH z?I-lW?XQ4X;2T-w2Ka6X!y{4uba-5dQ^a?_wpZGfPT{MKh_-e8M z#`jQkI1=<3j#okX47O?|}5n`VCb9{7iBPz9NQ>~v~JZy2z|wqj{sGb||lbh>Fu{9(84pKB=H zRHL~SMBnMf$4QRj@bcoWr-GA`V{;g}J~LE7@!F=sC6M_E;WY~lSzzS&ZnK{Be|cPe z08uICx=`ilO=eHY!{b8GJ@|4ST_f{g<3lG-@g9+QX4^C6udd^}UR{ahy6Aa63;c(Y z7Ea=Dpc{yFL&^#8S3G=vueM%l114Dp1!olUSY>cpo4MyUzvS?j&juLAs+HWRv__g` zp5hzJsCE3is@ZA_mw}O&v;j{>vWQWf40LbESUAI;a4gBllf@ZK9l@SdOuV}M*+0Rq zc*y-{q1zt7Xbl9PZEUBRTCYmoqews_=MlOD>x4$uWqlqk?fr?m;~hQ~l&!du#D&L8 z9T$a_vBVjGiywHanjT4eoJHUWy=n^!FEOrhkI19AJxO>$G)$RK^Q@q)BXJeficx5j zabhQ_UF_j?p%!vqWUQD#Q)ibN+TyT}k3|v>O~02P^;&4ogl?Lt#@F2i(?*XP|FSC? zAD^C`W)243J&gRAk1F_}V?kOkIRxQuq4qgS5j@?nT4yJ&IzR?^fL&DCu6WtL8DE&F4Gg(Vkn);1(iNsS8BLpSm76}aO>Z9T z(f5b3@Y+s)Gj?2(Wa{`4G`xOzWnh;8j}&v%@Z^djU%*r)@|&ed^z?1^w2`7z`Qt=G zv=H_5#P|go+RG{^+>}PnE@tNDSdD2^A+au760k;<;k%om=fd8Ss~n~QuwsPw5UcMb z1lmtDViA8(!TaceWaqMS$p=fk6?{&Z5JvsG`$wsEYMB5S>jF7N<=2uIrn|T(!zR|M z4tyz+@lu*JI=Xwx;Y`){5G=BD9-c6rZ7FYrGVooD1+6gR^*RvWIs=rc0&H#@=tb^P z#G?2lB!sVwV*shX46}g@GGw!VMlp}82g0q+Xb%f*_5D*Yly!!MP`RJ~5f#`0l~9-A z(nt*}jwW?k?9itxdWWAbfmRP(#y~&5uo+2P>qAyOF91B!QLSXrrtn^=RA=xme(`$x zAzQz=C0@i)1ObR;+_}pgT7~RBLwRQDl#UoMGwwTUz={$N`nfG6|1e!CK|f`V(cmdVsFy*iZ64Nfh#;Jrbe%V=9u|)c5!u zsAc7f@#NSt%|j3mGd-UDvcAD0f+{3+5d&Db&!X{h3P>c5OcmQF>b0c#HMeh~Ipik5 zsBDpha^NPc$_9Sxe@@xY7UeqVcN~8h;*-ROxy{;#F&$EpBLX3kAw`rC3=zUhm1_)} zM*3F68jdPAx?4}a{qy^byYfy%U9@L*BR=2(`y*Y_(bCOs)h^HjHvZ;K{?|J#vh~EA z(WpsLk@j#AcbhFq6+7UOh`Z5Rjka)W3chGm477%_1frMFjiAQ83D2b%H5TGYMzlB@ ztsHE6$&@rQdRPQP)^rr=$TM!{89)>r*k5;*$*ALf)|a{VEicBM5?jU+y{N~ZrD;rk zI>Z+CG(kR=;C~5?#xY@~+nbir!|oftYN3|ooxYuAvbc(GC4O4DekKEkkeKD_(&!WS zGTIh_?H@FAVxXU43P&}wa`7zTEO@JeDWUs?A$-{rk_eRi(tHFb z>b^s~|FDrrO7yTduJL>I)>_P5&09Ig|LZLi(pBiu3Ip%wojhli92fo?LmK6PN(&{gZp|ZG<#+)VW-VGw3LQc z<%?`uroP4?{H%50tttoi!9X)MmtQMR$akP^RXF(Uc=_9G=G@iCgR{69D^0nqZ;Qbk zHO75co^7U(pcN{87iUMfADJN0KVXmBqK0|ILEyrwT0LZ==Ft}8S|DYUtceFP&wegI zNNO)Mz_sA3GZKm7u$3zNi$La%||0pb*t=cnD2s3No0JkeSskjD=3fntA&E zy2VN3V6OHsl%{hm@Z*}3FF&^8KGT)yl2G*MaNKIBNKq(JyeJ;N)VO>2G}I1o5a z$^fPWqcUt7d1Kaw+lww%-T;g8i9$X41<(Nm^z&ry1O+XYa;I#qcPFPJkEv(eC!-2G zYe=2|l3*t2Ru{SweBtZ>Xh4_0#(sSYdxdw-f#nu9mPA{X{?Owp{+oAL}pk3w_^1kN~4*BnZW`*WaU%0I8(yG&;H z)}|%#vHzUpp`p!^6=A9_2v0g-s@T+l4>fsBux-_Um)gpjl8;a5^Ugw?PrC|;-VSwT zd3%02@}_aH*}1@12eL^DlG|Ay&Khr}(2k)MFZusxiW@Prh z&p6z!mivG%O=JU%k9=FVeB`=>J@ESMNnw1IJl+E-HEHoi`jAIa64C)C=cREpOn}?I9_VUOnwy6~6`b17Pw;OO`fs)K5y_yez$B zbDhb>w!n5347S!mDIdr0*RzU8r=L)WFxBF;GUKv_s6vIR4R5mS0!cP&fs0B&>|X!s ziNmv&X`tE0bvXJ*j2YP<@gXxZ;X3@p^y5B-KK*o=k-euc&D6J1zkbsEi)oiEnJ(h) zL|rFUExz`K?gbrfYvDX`1vt+iif_HP3StN&oaxK?l$A!}x$!MjEQGCfM0K8o!^0PP7_Dc{N9#qkNcKp7tXtjy6y>KUjypyWRo zi%5KhLi2q9gX|B`c8sB_G&Zq02C=Rj!~(h|0SW$xE+)8-f#%~%g`Nc*6ExuTlUIuJ zY}rfqQ9j#IB4Jhm{#Z~jYTeAS3Va1&Q=?vbX=#M=CcwFXfmuzr|LZ= z?AFHo8{)Q07nRFU>hyaW72_wjjPpf77_Q^pwVvifCMqP+JK)I=tP;2faF9EyY~rNjw!zQ6*uFy@oF$ zh!G3hGAW`$7ru~Kw>gkLvtf+WG^!X^#Nh`Ba`rd~W*NY?N2*-z_PjH#t|L*GytOY7 zXK1Wg)%xA2Wz3mIbQjj>B$)2tP0|d?KbaF)wnA-;{0$x>arDQVTuuPnMwjyY*aJ%o z8s-V-D7TXSErAI}==TlHE^tC_b-x&>v0_lNUy0q}K}E5hb@-m$&GJ)6e@ULsNxGr> z^dhqI?Oy8-i8-n0B6_cp2kADqSCWiqE7vePqP}vO&1#W^Nzb&AXj|NF>^67TuEc8a zY_)2;|HZVN1{{em_^y1pMPFNx5fl1l`$Z8pqYBKn=hz@W4v!3?bd~*#i&5nAwGWtH z&q0!FAgt_G9Nc%oQPwb!64X`q7~z3MUX01?B?$OA3qt?$5(cK{8YBMH6LDE^^~6&o z8EEtg;KTo?nDpGI>%r0_YmoMhiIhp_2*fuO--am7QVqTX5JU8aiTu8te{0gx;8S)t#gvZ`Vo2lE) zKs){^R^^mlXZs3+_9hjc+O4{yiuJUeC)-9=G&5^v2|46E0+8!Ty3RT8tvNtyvms zA58aT@nN{x2Obe!E&HT?ik2`duIkf2FjSdMPneEq5WszOS-|j+={13L=~ErcB8PLC z?YaOj-v#rFWPRh*Ub12xd>~8+cm9z@D zPrv7XSXUk*D~arA0QV=f-v(qzrq5I46O)*5sQKs-m=`0~Qf4a^VxG^BqpS*!&CBpR z}1y5s7^R@ai+FG8r|-DcUV{?AwAP9|Kcf5PJVFPb@b zM>?+Ob+?ZhN6&~zr>CD|10&z1+7IFl22Gy@ZyHtL9j>;jjavFNAR3KG$>AgmKE`-a zJZ?pmnx006DY(&-)BHnwqjMR(2Qg1_uFUXn9^zbS;0=^%dT~7;5O7GU$O^Jf(btq5^A@Rt0HrYPG%pN@S#a#tN$QBz|R6&^1a<8cgAII^Rt=3({ps(u>FtEd7yxHVm$TK;80%>w~H*V*N+%gD1 zG>evRC#^)3RFK@kVNf6CVVVk zJ8PisLQzY5zR11z@FeH`g!)3nQ_k zUoxbu$RBzvW~?VX*tLHJt4?~vH&m~jkl4FJyDM7JqrYU~{QS_nW6@1ebV<1hj#`)r zti4~Y556%Mwuz`fUzqoU_)rp%3!XonbmoTZt*%Zj^I{&kBd(fAomf##Pq9e-Uh_{! z$#A3Krx;Bq7nCh1g)A{(u_eH9)v?~% zW8HRbo9+jt0QIVS80PlXadFCMM0<@kj=vbnjT8A}%l5QL%E& zP!%Ti5cURd>3t&L?^BmM2NU9c=v}p1!)*1GS<&_qXTZvun_{#>Ga-Nb(*CErYD=fD zjXmWmcL|(WMawbQ`8MscU^|~Q40nY7-iq8k9m!$vcVo7#^lzcF)DjK@QyzV`3t~Bv z#(lok+y7<1AJ3+tDHMVE!P;5~nXP7vB(@w1zse*+R~jP~Y7((DN&HHoPQGpUZ@~!Z z=Ws`Oyp)~bUSJfimWXqGbMaladBX>BKymGTgeb^ikj-xxt3F!linfo)+xkKvP2eP$ zS=*fP9M}wkt~?HB zHQLvqC~>8}R2%rjzoTZv(9iLqdwDB#rK=@1-%y7TEy%tK^$DB8O)JSfU3_Y~w10yn zM)W+SQVgj-0u6YcX8+byA`%Wmd08~7pwmh56EK_UBFMukO(=ZDRA?HO=}wK*p}Le2 zg=D=I>b_p8s3SEWrfmA4fjoC`NFX_QOWOZM1IsEO;A ze}JF9Rq`5bG>rE&sIfH1B+{A4Zu!1T*VMU^1d=qG_jHY#2ITH)KX(>PQP+|^O$e`s znHRGK76Wl}6UdwSq_&dV(D#J37K&94?p@m$5c){lEr7^|0!V+U(I~i$$Tnm5j!$#E z4>$tjw4R)?-}-$wzT>d3hJI3lyTF$&mR~W!fzDPw9+JGi!oDLPnfjB$tnG{WCw z*9uRfuSdYzNy5r!k8kc0qtyTv4;;E}Pc*~07w-o3uOR1k$~Yj|qM|>gimff1rj<8= zE>n)KUnB}y?W_Re*FrMd6ThUN%NAV7%0$0Gt|tz9^)YRG$t_E~kSiai5OLXD%ATte z`4^Kk<7J>%vBqTNEea$WfTC_8Gc-+5tei3@PU@NHZa8AJQ1x4{@t{Kibr-h$88Y!E zYnI%e1XcH_)LRI3gh}YJ!`0+=(K!@`tDE152`!z70^%s=OFxF)bgZlW`#zlzO=Pro zt^;m+HZlv?`Qr7mnv)cmiDS1`KXmWOXp>$+y+=rvnw1aVzODM5r~3;j9a6RXu-)gx^jy@;WtRM@Xj2k;tdttut@Q34av5 zerfBFg-qZ>^7_z_&JpK;$pdsOJ}QX0NA6s3#S27`Z+?qY=={l^2H!eO@r=dxGOm4} z_4w4`gf#pV;TR)m$kt0Z%3BspW&D#AwVoX@SR`|3wC16_>T&#FJm_XNsyv?+nqtm7 z@oO(Iu;yNNWspn|adQJ4TWRqZSaP7xZ&g?%nIX5F!C+qO z%JLp6c;dqcCBXID8ZJ_56`yy~AV)qsnT)Nmw8#1#yB1B(ia0=RP2z6SDwG*wpDMbG zZ{fgfen1|iKTJrBv^x#cVKRjYEeIq)4vpKx63R6s#-zz}#!SRiGT)jnUhD^lgUlQ@ zO;zuB;SjB2%)D6H*r&;3K+}5WVG1$RR?3)@wOC+p@lv7c7X?Z{Ow}MJ zrq!h^(S~x%#F0yQxL^^cXCmY@z#muVlXNy>*Bz2~{M%*xYOOD5zk=zQ>h2b6^f||C-_O0&w3VA*@FOpfkaLlSsMG8zmIw?6tN@>1a?p^97 zDcfBou4n$Aj;f4CBi~-;#_SYL=&@6BF5h+eZzZ%S@6Th>lmp07^BW&Z6Wce6E{Kf3 ze4n=pn!;Zq8OI5@H*zk=j#EE{InlYo-X1b)e|cCl@4)WUS|nJtQXY@(^(Va*HS6~| zA22=JymM%X>{pG^FC+5HN5j3IRFKdymB~#;>B<)OxYMLaDTmCw#A8(8^Wa*uib^eL zyJx(rSp@tCW}$X2F?G2ma%FK@y6=$7pAOP-yzY>Ev!Rnx8sjv0lGa`I9b&BaSfZ6W ze$K)LIsJ-M>@?fu%Zqi)%bDdGF&+2=C&p{6!zZd)12oKx3>2ue_B$d3^lQ_{nRzUO z2B&iMhpi?XJT6<1!FO~wdC->#8Rw?_wP>=8f5jqx6E$0QU_10>$h@DFn^8} z7By5pbI+aL-kDh)1z`iW^r_dd$GYP<`gncdjW~EqoLfm_;l5|?D5n=&UypAw!cmV* z<1RiLO1RtAR`YWO^J1O%QQ-wfkZ(dZcCG!DMs_EWiEmkXDvSeAD)4Jr4+ph2NLrO< zUKgK6)JG>o{3J`(BTm%HS{})Ns@sEQ%pg}wTf+)LU=MaNa-YX&H9LqxcSNADZ5_Ac zKRifcul1U`o??nd?#to3^r_(PeKKBu^-B{Tq?x>i((eF02;Tn1zQ|!ae;x`RM-?Uo z*tORqr{HdG?{meoeV#0i)<|g?*g9aA#+mW?67BH&EIC_N6_H8S)qFP*eAd8;u%yZr z+1$X}PLH+Ppua(U-MW?j$koGt(DI;UpJRV){3rbxY=9yxY&|7a;31>#V84ul=1$1} zKuiBNkDVu?7PQkK2|VCBUnd{^PJ-h`GWa%Y{;}^|#%cpOjx9T+MMvHgf7OPCGidwa z%A=ANJW0R1F8)JYt(cHFh0mS17Taylq&vNZ4De*a}rU{Z%#2$hRf?u8{G3jKTvZ+(zdF7%zbA@S!>!n zSH6&RedCVAVo15`7UCkGByN!0JtDg;Z2@K>9az@a3-~HbON!)9w_VUeaf0zW@elwY z+N(5aTmcFWg^I}N{fIiDBw?D~9CY`8A9#Xan7K9n!6S;lI6K`` z3S<&|8de^>S#+B4kY>aNgG>rbcIm4gxzOzPV7`UONd|dJle`D2CoE~ z{)E;EamnzpiX1j)h1ujIC&D6aE%Kxi&M`xr#=W!YoRj3;4t2&UZr|_dY#81B`0Hl0 z$$@s-^&PBxichQMhmOd&SFDAKeg%$g+VY%O1Y2*>Fp)7c-{_vU5?y2rR}n$&#*okD zbGbt6UbI?M+~v8|RV_y?hW(_p%$wcu7YAx%QM-)i9~bjBOe>(2k()1={q410Ux;{} zkG;bb+Uz^e&WV{mSnQa5ONm>UxUs`5msnxtA2~5wp!u}V%P=y;q0NSDrC%xwL?#IT zPxI&0TFnTQK(%C;vW_7m)Ggjzz12L;g?8YNjaPk0q+E-HLGK;;?HREbzK{#$+}9sa z3a;cz!%M!kR(BG=sitJvwp(6=<)&kH++^)MrULzPJ*W>gr>dq=N|qBlUR0YxPgjH| ze^peA!M(#)UsbPt)W1fBAFj*9M_b4YS`RZZ@h`FcizKG9`8@%6EdM!Oku^P(Uj)} z$Pt*%P7(ab9J1pv<~2`hW2NOcS912=GgitU8(_DUfaV$TIL5jg_~vO7hfu7y8PZ;N zHpf;%Hp$-kIDYXHmZee|ga?14T<}c6j+GGc$W(VhXBaNm;kl%1-rP2iSh*z>i?=vV z91ZjUYs`<7GM0YpjbqV0JbF1Qi7b4KGQS>ay4w6=b(N$q7E75=LguKRxapC~4rN$86N(1_Rrm0UehrCKPiM_;9zrVFA1#~ z{=kZChC}h(!nUAo+SC!A^q+M2|EGfH>_l6o5U&)*p#T`MVM!pWr)5(WbLUYfgcHAe z^#87JJQabX=j5DFEU^z%G&erRPQe{QF(uYhNH-K_a&b1;&UzPsqEtq)4D z(r~)$gy9g&&-TAIkYB8lR|SDj!J1bfSEwp&BO;9C^WWhQRx9wch&mtMZ((I>!$|Z; zPYwH9gw~^~dC*Z!%0VdX9YFnqJ0VIZXnjxm47ExK9U1O7e^h)Pex>m^UKnc<2Rvd zrbxx6{ZN@8ddG=+qWn+d!r6f53d2rDgsXeA1jrmjX)PuS=vxdO5!YlyMoMmm*)_2$ zuvScJ6gV*gYV?}!C1-mEAKUV%?^$TsIj$Zq+R{BVpdSVH-(*=Hw9#+*!qVvA_YlM! zpQ)crW>M{P@(H+L?bgX0uhvPN9jHm4B%O$*SW)~-MA;aB#+fbXhhGenx%%`S_P5=P znEKg8#Gfd+YlMi{uIkXkWv)A+%{Gcw@Zm2THkp7uRmGO7TcFu|Cvk9Pwn2GX?@^#j zEMJ1RujZlepo|8gVyC61$rSH7c1wD^utqya&JdyK*yz#3vLldRIefSw?f^oSS-!uh z4qZxq$a3aU=8ZKds)^Wsi>!?YjSFS@ch$I@BlfG)IOR*mU@9qkVgS>_DkX$>nganG ziYYu2md zS0hN13{H+&PPi^w{Nv1@*q+}GA*ZtqzYe1)3{YPM zzM|H}v8bFg6RDSQbbPOHbVA*Yg9C?i_+tcz{E@zmwnmqD>3>$PEuVoZVSlUJYEtId z_7>i7=_mj=`!e@8Xr_}a`{{x-k0Q3IW^=BzlxX6$q*D*bOylMW_&u!v_ zO0-BXkj+WXoXJ~vf@W0Op;J5gTneD`;ZILO#cXaKju9gg(Mlo^ey0YGaiZ$ zAc#O9KGw7SLBdf~`~9#2{OYTX&#Vm!0^FD{=!seqQVgqTQat5=fXMW(mZHmToUiE^ zI8Gjo%=LB6ZODG7rgm{`IRRpD*hc>Woo5A-?N&EtY<-LHYRkjkjRWmT!b4QYt1|47 zB44SMlW%#-@L$8RNs)3hb%cvEolAFX|MV>)>$l_P1WR{q?8|4c&QyNz@nxWAm#%Aj zboLhdByk3hxQj61sW9=#jSmh>vhJ= z9`-?>=3-S?pJwY_ibsg_0`B>U zMs;!tfR~&rnbvriQeq2>)KbhgUgEEmT$K%Mlz`2WjT`ztF#S9BU2*rHYnEWJ^#!W3s-rGc13t z+3^e-I8xV*5a`kS$k7M;c!xeqCjkOn2CSnSE-<*#+mbex`BhqhU~2GQBk84ZjQ#V! zYK_5SM+ST{o3GQF7$J6XnTk2i?$C+S$NQYC4qXmnmO^L(Siy~v$LD?fp+|ove+|{k0W|NRYy$`q}cNt^0Bi{-GY4C_=)-~ zafC|6f=6!Vfu5XM~rJgC!mP}=wJ~lH5o7>vSzRB zmS$t`8VE^+o5!;59&9Hx+5X9UBG?|lsh9QjhpbRt(=Pvk`sD@iqFX=Wg)^fy{dy8a zAY+@mOvhloo(~US6`CZ6p)vn!njbphH^K&7h!Un%0b;<4aO{m6?GtX)3Lqg*5amx9 zgrZqix1Ph_0e!(67@>}xy-m8Fdt`Bb9a&NtlK8l zYyAv^?Mz7bChoyt@?r%9Ub3gz9SqXx7z0Am3S_EYEZWC0iPD2XzFn)EjcZT1sUp<5 zqQUKP1kGiw594w544384$-83lJenMqf9iYare^YR>h6T9H*Rdq8m1W-L+UH@vfns@6A1eWy|er2=@ISliaGJ!RZoHG)3i#fg? zI0`IPn8^Xj6UlN>3b0E{x_zz>0Vsi`!!68w=TRh2^cPk|K@6M~EEr4td+e{7#qb`s z=Q`pAU5ok}2k69T)ntJeaL8hEbqM*|i^%UTrn=5Q+AKTs?3`GBzI8?xH><7DX@&*A{q*UamD>ffL3Qx4x536UeIKeo#`nYI z84om$3VkOtYBkHr`xRR(;N;; z3zpIlw3-vDC2;t*(6(swpkqa>q!Pw^wu1QU?M*XIz^F2g`9CW!7CAeAv z2{idnBE{KEo16}(^wQHErcVI`kIrjpwwS*}5`n~fV!#Ex*^Ew3@YVd^()j%6(u? zYB%W_U?bx>qR(4XLkU8{&mcuf6~^wMz=L_&=EhNivQtFM#yWwAa;Y?0I@?BxKAelz zj?eicVFOo=soVugipP!Z;|+}Dq?ViD1=MXuzfs%6cISS>D5!gpm^(yCa^o`|yx3$g zD5+0;;lVab6+H8A8t55(+va;Mfxs#D@sqkhxh@CB^FGsoa%fz|IFM||4%^88&KHwZ z6V9IbULlkI7=FWmy3d2zVjHJA#afosL!D8K4lP$n54038NvaoNbDISSE0{CoN51er z;fpd_0WH_bD3F|;U0+ljdd{<{EqBYfa-F@xA!4}3W<3svmDM1Ft2Z)=e5wa z?58?NA;i$xrJZ9}!Exh`JW^vkc00z3nlJ(=bufGjy|-DL{%rYZ`*5__vk-Y5CrXCr zj`-Bbv{9h*-M;*C)H4IoV1^#xAwC(JY}b@bmjLLZRk)bMKOg~Ss{|S7GW|Q8q*kQl zT`$6BUH&baK`5<`^^N$saDzw#=H#5@w4Szd_QnsZ8tEf^A#I@Na`LVcdJ~}I%m_Sj zM4SfBkQ#s~Af>gT*+qzr8H)04r$JNjP*ZETgxYOq*#09jt)5Mj!^wnxa^mxGPk?6! z_M%^I@aGH8P%TQ`SA5T+YK^+aZ(ZKXeebz1y`xyuA0;^w!d5}wO?9k>PlMXo`mST^ zBpuH{BN#5TNo zDCJwJ_YGr>QViy6W0hBP6rTOu|DAYdMsg{sN=st&zoEN)YFpOR*&(hSdh0M#RKf&L zMdJYOt#yeD5&l;jC9PQiW*ri`n2JgPG#9D| zvWZ*6OPOZ82cZ!G^bvMIO%zA@lrC9lG^nV1x(IooJArIPnjV`?_sS$}1Ggg%DzzHT zGvd3Mg&*mr;aQs&SQR={J+Xn`Z<`*y44+NUV5^hHgJ=q+f}K@wd$7K-W$AvN?)`Y0yl=OV z?0U8$>!-%&kHLI=2^)vpEwU}b1tng@yy{Xu%*#M+vPUVGue;l{vjSy&N`_(l9;M>`F)!?HN3C zgv9kCyG^yW=Tspj6JR6#J{aXAYrQXqcw^3loqINmAe54YAdtrbsKbxty+HpHB}&` z6XG69Q>pd-@G&V;?h|NlmwdzD`NhZD~Omqbvhev!}-s6 zBvCZsW z2N=>f>hg~eYt=|9t94iR`YvT>t5`;pC5vN`Pqf^t^v0D_3gtqA6fe5TmqwH;-; z8^NbGu;fW=k0v%*^G|yq{v*VyCh+n%DOf4=c>|^p(FizOy%vT*K?$6OmWD5M zd^Q)TGohtTEu4>%6k0EcW(LZ6gUO|km^rP_G)MKNfJd9}0F$_nAdi+Y!6jwx%X*l+ zl%<5ok=h(vhd+kXoI)~0uihA5Q5%o(@MWr1ssPw3`I9WqncwUF0(Ys3R4 z?RG3(>XoSsyF$WnkPkW0slA7#+{dgAm>n1Gz%QiwRDiYhvW{4nl#$;aF1q6c8u$D~ zK4LMIE|s`Nr3m_tOF-UA4(exoyK-UulQ$Q0VlJ*27o<-r>%Go4A(us)E-`>CaxEA4 z;3VQ#*nakUVEufpIiPI1OOfen2C}OUqqa=ePvk`0p8ZMzQ#8|I{U+lydn_!Q?IuGs z>e_+CC%WxvXq9_m`Clke+eyPBn?bwd^75BD3D}T_K=BbfKvpZWpjjwj+|l+g4y)?0 ziccO*^N&slTC0PvvmBe<#`Yp7V}};CkZ44S>lU&1gUiH$KVuKVkKfej$5%3)y#%p? z8exuyK{;}T9Ir&vVnodaqscfZ6R!L7+H{eW2oqz=Jzqwmk1Zw+P?;z<-h;&-htd6N zG)F5tuIk(^O5Yd=j>YiT*?xXa%?Vt;%b@^{M@G4mdJzKDAiV5LA-wS=%dE50ef~T| z8J%>oe;Sh465{$W+EVTkZLGSPlAP4q9EP|plU0Nxo7|#oG*{_sJll_m z-O`Y9=2=F9Y8K$p<{G}llEK_UlbPBt(wVJGMnFZ-G>I2K$B*{M@U)n^vOH~Pfq$uR zC$o=kxL-I7BH?`MNvf$?LY}|YBoxaVQo>2*$U@Uk{a3$wpqN}ZzHAXgdB8B&b%5>! zhP8BRqAjxntwV@ZlcCk^@pZrQ+ooYwU7Up*VMiHad)oX{Qk9OxtWo0=cJ*XK(7x`+ zlku5MVg>%9Q7HFPS#lSi9@N@o=?t(i+vJCu-J_9v4>RD0-F zbx$N~DBD8E!N20954~N|=Bqlt#U(z`HT}Yzh&uha$z!D`2H$R556%R~q#SFW(V0EJ;nzeEhjh z!Y!&FIUu!f9B$l%fngyYdeeL%r`z;-StU7eUA*e%`S3spleLRA(Crp$)oXmKO6Q~D zY*McfWK3+{+P3RRki8T4#BJIZn>X zNFhi76+v>zH8ng`(6OTj%V{fE57Qy}l^US{@~IUt5*qeqs7qior;32|iv~s5C@3HQ zZifWwr&lJ@&NkziZXW7Z|I3nFK^QBi!X*n_^x-h{7H(}~nC^E+IEH4dEflY+(iP83 zi8jZ)%NXkd==OJXBOS;LDG`x0E=g;#&jQfp!oNDOfv6_pkX=l=^5 z4ej#IbhOj@3VKI!9BS#U`fv6Yllt)ks%#e1>$Mowp)@y@DaxRQa`8a?HOc8#4=Xu{ zgs1qr(|sX}95*#wv^5#bz@e5{%B8NE-_!bRMqyKW!ULseB4MIRWZ|om+u^c%w}UK+ zEbS0dwm;yvvpXPPN}}yf3F9hAhk+ot7Sv+T{o&CBU8AI4QX=4HJ!>}+bu*>m4)so4 zW3t=vd?_t-BX!uTs&!E|d*SCvofo8jg^iJwpM2UPIl1-}8 zc@@Iiki`-54OOvq)Gt}Ti9V&ue2vNOCZbE)d=kWyxT{59>|_r_mXptqsX6&~h2!{0 zWjo!%irCZKGNXK3&<$(t&0e}bZs1cwX*}p`pg!}4>7C}0WA3@Z-XTeHOrhc+LV{Eo*h29=IwFqFiq(XLBH zJaFZ#3H=43b}P!U>Z)0!-KoW9b02G#krb@YvRM|MQa~0lMo!CNE8#o-Zs?j?o>6Eh z3pt!gp)Ia2hP&eQ9f0-g9(R2Tt<#3fY zdo^q`N)IIE6_`H%XPx%mEIQ&ECH^ea?N=b^(Rl1+`4UNy%8Uz?$qLaNmqIpVvH*D4VaV+7OEALn zf8&dA^KszR;H^I^+JeLoo{8sU3EVH*;)RUkf0HbTV%G3G`Y{7(dJQ)THN2h3S=JSi z?|)SaY=`RdozQ~@j!DmSu6r#>-=$x^qF*J=P-ry-Fd#rYjFB{|0cy}9-$PrlN;xcNKe(9kZHd4kQt&tk!!urRm z130X#d5Q^U*2rk2FvSbq6+7jY)r?)oWVc&cEB$YNw);6-rOZ0$bl{J=X0GxzRNJ)x z^#>QBXV+Z#{_#ww$ZN&J*8h&+tfjub4|T9F6CEYu;lWU3uKD{!a#g9E&Q8E5BD+mQ zX^>c8JPW7rKiyOsn}`8u{en1=%x={=93maBCq8H~{#Ql5tyG8Fm-a(?U9i~vyd8DG zC;$*5>ViY#M!0(t3&resxZQm#3;Dx$bNDe5wtpoFZWHJ+gLTbEBRKM+a*u;489TeB z5*?%IJQ0eIdZ&MOqM=sT1=5ANisMOpBlBID7(BuA>T$q?R=}lUYb{?{hJpxhfeLXy zj)N|b5t40aGMTzzC=;48C9h<~S9}px{B~R#2mQVdNzqOiHw}~xVyxGVWdD4CF;(>Y z?NU$CDaWgK1wJb6%+9{1$6?L2-R1C1?Cv{pAsJA?NU(%C34bj|I)x)?-NGVH(nk{U=5%s_WZ>42TSDowj6q5d_U#b@M1ln#Cf@a zN^=LOB64s)B>=kZXsp1b0@*2x@bvZOi@svn`5%$7_CW zb@J$ppKR2MAIXX8F?GaLaNZQ2_g}utuAOOwx{QprvDI0tNo@*COVj}(IeP8VB%aP` zXc~B|#E02Pz}vNzSXwBP&c;4w`cW<;Vih>fUnG(qyb$P$lv209U(;ndZAg|t zqpo7x{(-ozQBCZy#1uTpR06UupI%;KDWeK$M%w>0;oB>aTTi3>b>-(ExjDzeoL8K8 zNyHE*N0sCHMCyaz~+b#orbQL;+2c((E>)%(X@)iUPS3Ck4$M<;Z z$~@3SB5rLM8A#2b;U_M`TY_QP`UnY|dL4G-oqT>65PPa^UPiD1D2KvRLGc7gRCZgr z7Q)CS&n=Nm{U7%?#jiS$tQXX|u9iYrqd)!U%*{xu-`_;E=`QXg^!9;u(*mQ@{IJlJ z7*H(;?O-VohErliN}&FouNI#N5qkm?-z`qv@&|zgT$rpw*N`SUb3ZGgdbSl`LiGyK zp2gnocyP4P)=o$!LBOnbr&S{uPl2HOaOQJ4m(Ta2@XCWT-5dyw;n8N!I#-^bmN!e-C{VSha*VwQz~TsAO0Mm={87w^ca)gmtfRLyVRjH<@`^tbPimI76X|8)e|CtHw2DepEB-4pA|IZOW!|q@ zKO)8Ig9kS$(-}Ijcx-hr*Uq4bhKUKKk^7a1ZN%TEra*wkPklb zO`wh|geFnvq%U}Z*r}=ouj*B5aYac@BeJ8rY=Ll&VVhAFp}h>rK?;xZb;oKHX+j%K zJI{(%xEWz4biqc&ZY&pgIzGNzh1G12MbeLz%SWHkqg>xG&o|i?_?;&Ot7W`e{hx~f zC1MjhA)|BzB!5C;5ts}JU#`+4_#Zr6^tR04VetDicXQV*w}W;wBCrYy6}W4k1$rr7 zIY&KjHz+@;i$>eh#j3b6ku!%;H#u8G@6P&(yLRY0+d5KeO!|6Ty4$rnb#Qdzr{r`r z$xly(Jkt6zkGlJWhx0779o`Mj8)VI2ehN&MG_kXn?p^Bc`MnkSHs^Q+UNZv=A+emi z7jKZk0~u5EjJmaDwDnQX#JwvnG}$pLHjdcMC!@4oZQ@cv$V?Tc0NHT}hWKF`p&(nO zHYrf!W5``a90&WI-&^D7wDYZ#4terx8v}_@~L0xtd^jAHC}4qB{>f>-jk!+-vJJ-Ag=Dfz<0hy0&YSIgL-QrtG>yo#@Mo# zUsO5Ftw#)`UuC)g%+xwiF4+A^9iguzYSpMc=sKzV0K-!*0KxPv-ls?B6T8FV`J~iR zc_Ue|7XzVEUUd1OrXN+aIm`(9ewu?hA{j{xgAj-(3ow?c#r`-3p-DSKnNo}TFGsAJ z7(@Wk7v$^{!>c0uKR))^1H}xB7Z!XvGsuL1{dX~1sK+Y;7gXZ^DP#OGIvl2HLQ8Wd zJ8SQs{&h&igOCml3Pi%TfprR-S@RcaTv{`Ow8qLI9)YhZ3f5Nal~BUh0LS9!CVRXX zb$O1PcoBW}>_(=5s;0bH3X%MG6D|t;>uJ|ojO1#{bmY4v%45hi!J{KC@nf_E5_=JN zYB=f#$8$jD?biz_uq0k|wR=mm1|}GXtpjj3P^r_b-bdl(S;uLbx#4RVr>-7&Kf%$^ z#|M5X0$&tO&17nKs+&M6867r%2aVPiya`{&nUX_jMiEs`?MnwQPHU#oC~1a58iV;o@e#~@7X-UY;ae`cuuzLbOFbY*fNFGg%(bY(Vma%Ev{3V58gx@Am|x%U0YzF!W{qSI6;YFYXNCO>!E+0m7fTs=Kx>@xC32W zt^bz&H*QoP%mDvQ?IYB}#nJws0stB-5Xgy-nc2g`gUQm(6~yG|V#(xW_YXe}D{EJP zhog%v;N$87v;+Rr7&iy=k9LBrfd5MHcToTe)@DElSK!|uDaU^q?LS)i5%htB{)^a0 zBS3$1+WpHN;0gr(uQFC9uK&a;sHi9a>`km4KtKl*2eS`HkO|1m6=3`i_VEOoQ~!$~ z5Fqa6;_|nK;(wZ4{;SM?QWtalnAvALA3qb1e_t^Z2RB#m|IwTOJ#8~b2UlxXkn6uP z0s$7*cEG>vUH`6`wZlJ{qNtLrl%%=_qr!*q92gZHKf2?<1o8y^6iHg8$mii)btQ+d8~jy%3F67})7&{7jr6c#l17p0}nov!tL)7_DU+*BIW(u6Iq zddT0+PZ0b!N)tc;01a>s1K^pd!T3k}7N_|;SnYQZ$M*ig!N~}9|s*6 zR~5mH0O05V{Q2=MH}r=`SnmitT{9zqFZ8!g1Mz2^=2CADa{mkctN)l+`WFu{!;hWv zYLu4A&9&|UI30inF)bOJ4=*_X{ns@2Ta%icm6MYg!5e?Nr}cT~OZK|zs{1$fw8zsN z;McK8!P&WVev#BzPA^ki2GFPgc*1)!q?cwPOM?vJnY&fbe(^EZXoUtLjJSW+(i_xJA>Yo>&@eWa zFFLzA8{j`|1|9%4Iyk$3N=nSqo!@^V`<3#C_z3>Y@7%*4Ei3>KZ-6NTwN$L>cL5vb z%)!ZTA}KTKd~&z6nF@yxm!pI{^#*fJusc#n*Rj9>yf${B&`ZP*sdU54ooGa=Y|S1)t}_XVaac^eQqWEYSJ?h*9tH+}8hjlfb4Z z2(RJCK`9!w5i3l-r{zD?Z%OP9|I5~KmauLou7HayJ3g_RmSViLi9xt+&%Ys_{?7@y zFc6M3sgRv~DZ4~VK-*v-r3TS$&3}f5Bcn^i%mx+&g*wUE^W5sb;=+OACl?j>+!GP4 zwEE&9Jvv~ry$)zVRb}^Ir~v)ca{fUq>@P$EQ7md`o!mQun+Rx2Kh^KbXYXxWZ3QX{ z;8E+B3by)nhsJ5T059vfBPJvm+0FBagny{}I+nI%2_*`Y1ORue)}u)!k`67JB(jpe zihFZ`C$f!J^;mDku;^AGtWy_4`gk*5gYPx!WvN}T&Fjn>cinrQkS4jo70sFFcXTLe z`RVwGRcbq-n@5POMWSq*?hl9j>q6`A+0D@4^#Y?W(PB7~DnGf?q^5eDOL%o(JYE-F z&JNjlrm7@|MkM%VEh$-7UXk$)L3xl=+^ginJ&4GPmfSbf40pu^NLVaa*kZ)2VW5%4 zz3sZBX@O18SHK>(^wEjWCnhSv!RVifyXtlxUzmLTDA-ow#bcydD-)s6eOTrP(=zj~ z&7JcBpe%t1a5zf#5e4c|Yi;WzcvK4t5>`h+k=#oG6HT>HZf#=`_G-FsbJ*k(YAWNU zTZZLOd*FWqUJ;CSbq2}{48nDi&4!!hf|INPJtcT-OJiWwoqY}=Mk#b8GTAN3jTzzP z#U%Uon#E=vXF3j&BDp$T1Ip&mH~5<-vQ%XF>zfs%;Ks3ulx6SB=OwO4%^S4p1!?YZ z**k_$M~UOJL60X4G&A?}^k6lb67__=vzi8I`{3@}eVB5~73r`w-!#FI?7lcZ-@a)- z)s%A+2PTo^3F3Qb^+o(&H@GstbOgOWO#pVdqAn?BDUY}fvCpE2SnoCx(V=Nw|$*nT^`^XJ`UHa^5Pj8^` zqBxbr_G8?^Yg>zPHcxJ|;gUvishqZ|sfQOrgZ23l5$|4A0cQESnpYfr31N6grb|-N z1bg;H6R%AhjVL@qe+hJHz@1=j8+Gd*nO5)2?nZR7G%HvLCJj4WR%}*lzJV%P&8<~0rqvI zIlw`@xJ!H==dmcNHUYOgPl%j^Y7<2+Zqa1U^1^)o>tyJq#md{ey>^ssNs7-4@S)E* zBaKB$+jh~}I;Z)1;Yr__uvhK=xi_-jR;P#&6rWE0hmw|TyNdr*fI({|RM+l)`!|g@ zR6Y~Tk-uw7R6bpXQRfADs#svEgD6G6nSH;)p7UL_G z!^2URV@m+NR0l;ix7f+q*z4>~-eHN0ASdH^USoVU?}kQ#l}*AYK-a|h)X2aH5pXBi zoi12SkI&$i_zb^Mjbc~pnDqif_q9xdXMO_;1KOm9)ED_!WK3TjzFyCR)J+1r$G8RW z&6~*AM(VxJc6A^#ZCE9e)T`OOfYJ~PzQ|9I*HbA%=V|fCFQ@)lno-G}BoU@@=60l! z)VhK~#n%|ksU}WI1+s6!w$g$u#-Ay;^zDCu5v+Lcz1 z2vspr;^tp%d-+1;%tt=6(bt?`2t$32LQz$_OQggu8Y6E)re~*-&;rp2~JU zY69rUKWO^h6*qPFTWDl0} zty0~7AF4na$wt4zFKw-s11D!LqE$bBtad1`Yo-L?t>(gIvM4Iaf!foTVM9ZvjPr@Y zHg%WZ4H>*c{XUx_gNoq>JX70LUBpgN=9whY$tIsB9Y+fqB5u9>!)y8zl7V$f35W## zJGt9rK!VT~-SH&|$X#Hsx_U2-e4ge!&#_`FV>jK7Nb3~%uekDtTA za?#-O^W!fn4+IZCMwQF!lV9yJeD zM`doi*!xwPOULiL?}AKb3hHRe2d>X+5Kb7sy=e=Q_Hg^>@6?+jrB=X9tyMQ+Szs%q zcaBBVJ7&ZQ`}$9&J6qzS|i**wk^;dw5$g*FwD@;@O`1@Mdc21vDdXt*U+D9Fs{N| z4ol9&P!FA{+!($(mWf-I#a-5D!q+R`^3zX8G*lB?bxRIS?~bE6ubRd``HQC za$UE$8tD4-o^cqd+Z_QmWRs7U@JKQw#(Tv2T4%#I$TX7|U^}U?Yt2O-tD=T(a*(_Z z$E>nvnv2vTAciEro%pAf7|9q#-^#V$aP?zeHsw2c4Y9<{1QhnXZhZJ;#8i@xKVAI3 zp@!{An92qIu+j^;5ii}I#ZSeCzqK730kwR}*LSz1*pPAf_HH8}K ziJGxo8aA3iCR3KRFGEFSuh)PyLADM)TmMBTvWjg8<06qgY(;z$Z~Jnt{G4WQYy5JH zEmT+@%vGHNJa|J4Ai+HxEA}&+udl#}aMO8EEuXD`{3Nl3)ed5Rp1_wI?#LwIhRJy* z?-Wx8lvQ#sS_VoYu$`!&KFmp9ojSrVPO=9txgK1;&_66EW^Z_b^kpfqo^pET8O4Q` z_LXZbRC}xN~pJ#nuC*Zlh=>=I1f?&bT zdqlHI;~yDP&!(b$3)QIJlvU75ec&bb#6~!^q4GHqc6oL85WFg31q4C5Il@5w*fOfQ ze7@Fnv`qs@lW&NaVLo~No*W?+Z=xUCR{iH0SG^gJT6XJzHl@ZEe3ZgUO<{5AXy5-) zeT2q;o?{)Ee8+HY$L1iF7+g++QE1J|v)S9CVD#AB2G76AtDs*_T2KOe>J|4Hh>|ri zHHMMC{;~rF*Y1A?C_B-`RZfHxe#7)0Z=1fBT zXANEY8?kaLw-S)d(XwcXE2d+3ol1=hG>oyieM1P76dl%ca7^n{*=6iTCy4$`C`KMj z0p_du$IQg^b9bMH%mnwD!lr?r_9Rm>cuVLxK>{nwPGyzX9wpj(g!7zeSFp4HrPw;! zRFFJ}&g28txCD{Ta6rrFC(BlQpX674hgipw6`ALOi9yT3k2%}}rJpiuB_~}F%u0Yp z5zo`Tcx{%(O`|g%h%Mi_=-62M*LzQ70ss&Th6?O!J2)oFeH38N@7LenAc$i4jAZO? zC3NNbQr1bI8;mL*%$lqO*UB{R=h*&PV^2TPV}BT2N`7E_TOhjv=2UTjXuOvKH)VAg4~34~07}BHpPUAG zWQI)Sd(>EFRN3G$>hl3TH=ssa|1OT{@<&QosrQ*2l#>2|u)EI7GWB>+F-83$Kd;H= zU{B79#h^!lWiuvuRH*y*&aYZ}@>47N?!V&zn}x$c;wz#4WrB+TPJ(v^d7~G4kHL9( zYsVQG$RdkK16ctZJ2hH*Dui2LQ#4Usg7@GAw?vBi8CHHU4r;-7k?&>%-XlBT6<^i-Epg zD8ZvV^s|fomSEGOaLNl-dfh%RQAwpLH5ja=&Rj2F>!ed12mZT_F8^AME30R{e{&Ua zN>fa0Mz!8nB?X9DWeweW&#w6v!+}X0W?j5n?iB@tKa&R(+w(S6o}<&{Bw%WF1_uE_ zobTGKmwnd3DfR3QN?%`Xt$n5DdW#ub&-7f;m1uF2D%?cLj|(2Q)lhU--c#|yc_kCq zi_z18APJg8&cE&2!WGpMM?RBSrvN{tx%OjQ3fFIxn>a@d>FB?^$%fz8TmQd_9-Oh1 z$rL_I+gAJw%avk!5{|PC!1JL`CgybRTq`qFrh>{Q#vG}_%{Mb@VIihyk1&ASO9)i= zuOB$;DBARKj%4jHQ4`NONpw8t((3B`Px^-(u^^Iu_v*WkpO+ z%`P&pp4Dw7(V|8bUQ@fiU3GEEiBB|ZsD$}yh$-gy1Ny&z9d6h{G<8x2vF>eQ)91gT z2ino`M~-|=Azf6~`>Bc=)NWK!NX6QByNi`{nSyr*ehSH;8Zde!g6QHrW1|hh(K#2? zU-itSks96JZ;B36Q(f9;-Vd`X@v}%XaK!EC{ck+gzoe?u%rUi#%W!gBamsI)1yCCg zWzo3Bhd$BZ%`i8P1#6CK>ROoD#z~l*%_CY@E#@QXdplGxgDhO5$X7Hh_g`3l4o`gxR z*7;onB*kykmC&%c`I$twOz_t8*-jzFQo*G~?F$?hrQG;O1~ZLt`NvsYy?M#OX^LuCXmZ?uH+O!lwZYOpb^Z^)4q;#p^P@; zM^iTy0^LmBe~(u>eWPVHADygO##aMeNY(Vk|5AX|nz)k>BR^H&3fmDd!U4 zWt4k)=Ne#|?ZE>^Sm!Cyjx{s{Py+i2t3ZhqYA0MzhLGlE+UeC&53pci61y5!cGh+D z-uvh&AXWXKo-}dmlEg~Z?Mllm?z`Z_c{sD1M!I70WzECn9@QSJU7(QCEz!befL}DP zC6Yklaff)t%sIq^6XO(U(OT=s{n8iW1N0iNrVED9$~>-EV{LH-l`LzFSQG=!J9qch zC5HlwNu4h}W_&m(;?zE=c4}}=wG@Haq@@qM^b$jnK(V?ceV{~kHhN0Yv^?! zNvHNaoX>y{&*4^;ae?1ZwaDG-Bjv?D`7cuZS6!;C2v8_)rH7UK#+xsbu)sZ+w9}Z~ zB~HVVvfLK=J(?0`uw{Es!;t64qB`m7{Z|D=oh(#0z1MP$?x;iVtqQ0`AE(wz&+GjK zoEgqDXw1zvUE67TsQ5!Pg}j&MwNQ6V)?0<!#C=4hFZmM zdekG2woa77AB~U5aD>kiPmmx-`AMGbm`=w9QNQiZi@sydI|+*5wWE@=RlYf+D2Lqn zzMn;wj6bS+^cu)#*TIUo32vOa9pZS|hgvKw?z>pl-7wFCP)M56@`FhUhF}1{Qf)qEYZX5!5$@rs_ZTnXZ83y@mniYm0}bLig|SmF+$^8ENI|o; za_t*Nf%~(1fPKeWUJrzr^+=XJkU|QX8o0u<&lVEZPTrcCM~M?Xdbs-d$k(MI-SL@$ z8&6N>;m06J1DGlDDuYqS4bX|3%_!KWSN>em0w*=Csy^(=nwa;TTE;#s(Uks_vSVm7 z$)V=`zj=?(q?TPHfkk?By@QktJSIK!y72IShU*pV_adIg%6$~mqsTF4nm$9K8|s`@ zXs!u{u}2J7JYvDp-LJY$Pvt%z??Fdcvjr)$p?Trbh2Ym`{F7#Q?G>m_4GaP~m@IH@3#%8WNv z|1s-bfi|tg=E;R|Db0<>Ns(9ODEU!hSnA4b=poi%BXS-m0(DbJIz=?W zcx={V{yC)L-OLth)~w@C<7xTPW|~s-Lu$VzI-XM7zU~I^`H+Uq#m`;RA0K{kwF-ZL zr{Fi2S@2kaLleIT0~GpIUPulq7FY7lpg69wg=?;?R?T^(d){~yGIukX#EwHd$C%|L z6sO~?YYxn}_v|ftH(_y4PCv6y5w1SN`W>T530k0Vv@ZnIMOu4K#a(_q8UOxNb5X^* z&AKhrJ)w4Z-^i^{}XmqR1% zymxOTwp)+6F!7b9m7=XqHU5l|YJ%{4BWT>ix7p30&aN#s!!}(J>d(T9YMMa{2CGsw zPR+TyGQ4-d8#rBODQbQp_o#qumhLdM$t94YE^6`0t!(c%7-4KMVIrC$@H+D+L2Tr- z^-3@istO#%W!d2@%8MuOj?;|gkaO#uW;M`R;((Dy>DXSOrZj7jg*#kw7WyTsN?&}y zY+0%>AXKEZ>>b#aF5U`7gT*l8CH8q%zJtj|)B7|a6g(eEOr@Z6N;9k%Eh?H(;^|EJ z@nIZsqlC@D6r!|>R3&^Q^*~BN95L+HiYm@PKP0v<$Xn=P7mzwN=nD>;F(AF|8BURu zRw3=uf-rhPy%tQ@oGJXe>Pdf6hfua>O?P$L^6w8 zVP5f4jS7T&R1;A^MTT@b)30C3tq{M`iDv>+39?xR?sSSS^j`E{E2r)QVIoFj8m$SE z!g+DsFLqKC*0RA?`_fKmDARx=T_2@$Xs}!N9a?&BnC9t_ zw%BBtje7yV?|1q#AiAf@@^~RAUQBL}_l$4-JBlZ%@4u(j|NKwqI1dT&8Re)C?ZDWf!Uu{@Ll=$X$7cvH z0OY5_ulTuhRbOim_Mw3nuGJ57`*=`PZr3x4-tAVHMr1f$Po;eF<1p9#hnY^ zeHf4U_4h%cZL>|qz+;fO=K2w057pNtl$cvb4LD*n1l-2O1L_uK*1vT!$`MK^MQ=g` zaMn5WseaO;toI_2lGkg>HpyzDk94bbC?YJM00X6Y?#NV;I(m8us5+Gr zY@H8Yd`H9qLkm3O4vd>s##iCb|8%bCgxd;L1~&_o4jTch%Eh_BVP)~%5@(&p2nfBP zx((~@ouK{Q^%R2vCCdj{XgX!SsyzDO#F!ua&o%c#>D|p@y4&D(Ft;aTLym9IN&GXq ztkQyT!_?tR!tN{8%^}&SavfGMObVo9V>SsPV|C(13~EJiEa}mi#9(KCLQiv?7v>T@ z7>%;n_Y<`sR0CHc5X!BtQ7Z!1iD0X z3_0}1F`eB`^$b2pLnr6_EpLLjleQL>9Vs!__ci*96r^jpCAdbqL(yDn{ zuV4rP<{-5S&7vx?t>L)rk{u|NA@8cVyR#4c(2SQ1lPD&dnMnIDsIw~ZJrZ^A9~Pxp z$E<=CW}1vCLck)JqYxuRR4hv&Wc>6**H#+D+C}s({~XT0dt&!o`@|v^v{CD&sBmeR zWl}Bp5gbdsPoS!b$ZOOsFrO-(ov#@gh;h(uGI}IX*&a}iUTNzwX)Fua0q5c-8CY6i z9Accc%@=+5mK)=qt>zmV90(tObceIr9Wn!q;&BS6Fugr|%Qej9n}-9=f8n-Q2Xc$A zKGBtRDp0Aw*B`nFa8C5rta~_VXE8ygObqW@m3*YBu{6uyL@`Oa>o(yPAwbIdw_yZZ z?i^fCqO+oQuFuAl2~pgN5U6KEJwore)J@j7`4N6N+3zOzk`pe{nM}&Zk<;+uDtRTT zHcB0PWbx<)Qe?F^c35cZAvg6>)j6Hv;tQf=f8&}yX+AbGiz2Ae_{7^#bbxFiY}H*` zO$ak~sKBsCiu8%=LhiaDz`x9&aw`oZfelVBynv$CT zWjKE+33ROY5O=o1P<0hmkqFbVVdK(cUO9Qz4Zar?*oGaJftzLuw|yHkMoo5ZX#xy` z-7jvrKzxc{e;tA z&1MiL{VBqV)=_!RIN*Xk&h2ai9?h!#RPHM*@WFRnRd*qywJi=Z_w<&>S_3kG7O_jk zmGH_FTk5^=X&>)<>PK%}*u`?JM^>GDEe;PxRj>D#r5Nr96+&HIyxx>a&0AXubmL8P zFdGYh&TYww6C$3Sz$C(aMN+VBFKJ)Y4KRdB&?AGQBlv3*y#)|fus~NiJb}$Urq{Ju z6{lLsUie+w-TO@rAm|#m@HuD@9g@Uq>3yC(>D^+~zMZ)Hmz~|sy~B|A(?Nc6mZu9< zg$rlKi6HAKSVQ3E6BCxz=5nh~3!1`k{7yQwM>hX!|D73%kn$8pt|P_s6XKA*!nO3V z%0L-3wIOSYOcQ-_Y=^v~V(^kPXI-X*qz(wLSAb zYlG_g>~`2`Te#ww;_`-7_xoW8aCqy)LD*w+lvmhKm>Tpwp zjn$QjaC^V}A4dykPSrS$p2MWMDf217_RJ=Y5=giiUH5?j7*cP~??-!p8ZhXVeB@== zpbd0o)VRU*?O6;T@=iNm7H03!G=0|9l(K~F=oG;n)adwfGW3l&jKX?nZz)l|jba)BWwUR4!U z_8-@91x#GfFHR8O#XYgg&lm1iF05=Ox^uGg8C!-+pg)TzIR>}Xq zm@UE=dQ~mh;wamWqvq<(g1Q1Q6cQIIC!GP6J9bm;_h0&xZ_S)&D1Goid$%jw6S(a}@g3R4)Kx!jP`=t%P}N-e;kA48?qeFV3N>EZ0Gn3`24X z88#W_cB!d2EPq-<=oCD3Sm4Jnrv3|6EqETe=!l>p^5rQY30!qoEZfRzRl)ct`Kr-x;02EtP_Mh`v8K5fU5kXf~g5A zZZr{1uBJ=RbTLiAqOZCwwAijRaK4MZQCXNJ7sUepXZ#0tK57?0Rn@#^BC7Qk_UPsi zFT&J5W57eLT)H0R00Sr1r$%8AH@G&kP{hUt9ythlm$Hc7?;otx^}9u~y&{og0+gUd zEDpE%3h|K;Mue<3)oJ3p*fLCqr@(bJ(t1Q>2!O5aH}Yzdd9D$<=hS*Iydf(|0ENxm zN%M3Y*wR0`W-KlW{UD-uXP#4xJptPd$b?!S{Pxn#BwT^?U196U@&rRcv=y~6S5}v3 zU2wqJm!cHc{64NG>j9t+^}FIktC^U4hPNPt7nDPS^dqOC`IcFYUP!k0^^bEjE&*PY z^eb3=hokR?U;p1^y_?>-a+n4t$mEb8MDzRCQ6|FE86YzVHt;LdlbQz;ZEeCKG{YE!9>ANhq*d{{`PJrXVGP#d#4-%C(@i$wW;~L?>CI zYGrtoe>Iz>$YP&2?e(gP6uS;@uwyj~=0l;u`xV?#I3 zy}6Z|pdC<{N4u{s-1FJ7_S;X#459`ZB}~b0eNu}nv=fG;cdzm{jO0b2JMHScC7>pu zc^T@IT+aDA;32!YhUkPuJVkX-Q4`|ZQV$7@0Z^Sa)UHLJ5=OxhA9~<#8(2H;m>gOZ zcOj|lPlNNeBDk`6Wz)B=cHFA`9CkF&x*jd{Kce&4Jklo(tNC%w$gHj5hi3`4DC=A+ zz^f#sU3pQjqstk0W|P6bQ9_$(aki^Sf@I9vEYF*-lf{b8CsnH~%Txcyj330{jd=5F z(4%M7Cva4t0{9nfQ~wW8^?Io}=U*Q9d5y0ww+`Z)vAY-%pE>V$oX8clcSZP$Y*P-5WT zk_ER#2cBZzF4`rSWjVma_>SOphwd8MjL}C)52I+?uAuyq-|a1xvp1LGl^0`@)Eq$xRB>|{*>ljG zCp@P6jf=EGB6|()&l*qnb{Z3Jpcp?-aF68Bi*=L|Q%j<;*L)>HWOQLWWkidf%C-KK zPJ=7}klxwcFO)$AFG#6yQ!nJxjtbrCWdF5JN?IoxbYOK4+@OZ6J{KWj+e62zt?l#! zEH}SN?IqT7kiFH$^9_BwDk`l(^S)I$tv=)%JBgwm8A6>{0=~)|*hRck@6RfH$bY3j z0I6eOAwZjl_}Mbxpn<-v-eU2<(Qv8HL6!elAur5cppX>UW8qYBLqfraln-7l?#Sj{ zM}K?hNj<)t2aGi}T#0_NZFqXv12`Q*aLxy>X?%wBZ!N14PIeH-+ zS$osWqF;=mf_Rk5 z58kqjyj8YE#X|s}`2&Re()H_y)nu=Xo+D;JfIe!cwFQNZL(=nYuQVDkT3dHZ7VW%4 z-c_0bua?rW74!5~ly-NfVv@^iOPxm~*!)Xkd}(2D5Z0cDuT<}5^n%{efeFcJL*Wcz zLqNb?XL01=W&p&mL?BQDK)44CQ2qV**#_di!6H91MUfI$ugtb%WVmcMsqg7t7u@y3tfbZs zjr$Tusf!qK2GW?{^NMtA3(dPiy0n*pS$dT0AFLmA55=BrNwJh2u%*%5FlX5ndwoP!=Bwc5GRrNhBX0FAGf^}PlAJ3ax0m^m#(9R4p+f=1i6MRV*b>1|;o zS_I!zJtA>9O<1cb&rlj^RiK1%2tBNHSHoYkX1-O(!(EXQ(2*!|f>hp4=kD`QimKpW zCH2-pRb*ffmm{nx9#b`S;EYB{mmM2Uq7zxbDNsln!4JyV5n9Mx%kUT)lo;kb za(W?+4Ngvz(p&AaEGec=10bMi!AcZMxkvME2!HGoJ=QPl-7kSDDe_p6Wlx;{Qo?F> z@c`6$lR3}S-*}I5@Rp^5z}6u#sHugc^|7%Drh5itQ&af0tqBB0%lk~h0EyY1iZFTN z#5QG;X={-cg}O&K-#ZHg^GR%yTlz&l^Fat(R6n{3D-}~+7ad8C0p3Glv5SzqAHpy~ zTQT|w)SgXkqb|+Ls_h{&^|AxhT!Eo66Ve%cPq(yWvD_udKCk%SV{!#s%2vf1Cd#hq zhM!{0xiZb)fw_}2fhSt`IpAXxO32Y5Op*gN9kt0vQ5p(WxzlHOhmyZ2tFH{G_^wuW zMf~e7NgK61ETWvC9$S|!s2{Uj)`zP5^eXCtKqJkX*SG$(8~)iyfCZ}f^K>ei_6)VE zKwoBpcg=DyvYe8Z1XZ(+pz5*dZ+MwxLap4}Tn+-;&4C?T_4bAHtvS1~S|!f-3k#ZK zFZYH)P+Y#2B4uH;^?J~~I#KOc8b-}##f>B?`h5mBEF4M?%uwoXV^4O8$;uoIJ%)^z zunrr8GrIopm>4~_Udb(q7Ejg76)4gFl-&v65fZHVrV(@0h6u*jEn2L9DmS9A*1ohp z98_dEdMoE-nd6WY%|%YpZ`GsZhLzm@ngR}0SL12LE_xeFAIc3r8By}vUA3+DaW|C& z#^5#!V$*k@7g%M>5!Ij~H5fNC|7AA{@1IoNVZ~XeAjixcjrq>znTVSsPsS!Wt_Pya z#77#c)}~G;p@MtKV$e%&*Gk%o!6281G?O$_an;!=jS?vJ^BCy_V5x&4P5Pt zPsf7d3ccA`?80|24QPmA48Ol8`|(->f0zpE{SI6#VJ;RbiPLB+@wr-ykblf;JCsRj zw2M5@H?f@eEH~TqeD`2C1oPWpsf=aVqZ&#m;QkvNL=oG}h_NE=Ed$gCeQ*E*9%IT* z6AO3InM&?_c(Gh?Q~$O~v)MluMLn;@?ZNBC<)8hvZ0z>UKz*=?H!;+Cf!g!$yi3V> zlC9!3b7f^OMz~U7GZ&dGX+;J6^))DGp`OtUeGhnwZtjZQAyTX=Xr&0 z=z}fLMiVRM9N$d++Ja$z`=wsla=fKm5(-OjStqfY7>OOI8tgTEe6U!iYm+`{LY?Z* z4mI)q3L-mMdKTlTW#P<I9L<6a=Kf>mrP3il!L#? zJ)x0UM8&+Yw1)|@XBR`mXiLU{FLrBE;i z)nH75)I(>%lpmxHa?F8HYp$SCDs~icd>tGRoWcXqX*Ws?w;l8uWpi{L|v%Z z`MB@cl4oYKOWms!zUq8vZVnfCIy}f<{`r8!?Zz>Tv0i* z5rs(Z74zMUjh}AakIVO9Eq21`jq+>oRS#dX0aM)BKnuZ)`Ss&28VV3Sc&(>^f0RD7 z!v6Th2~J(Pj)?Bg-C{3cAbX+zh8iM1@&As5MHeheBjl}I@l+E|Kor9 znnn#LcEVv4N1Fd(W}0mW;^^iRc$jJ0B@}XQp8KIA6FK4iixE>-h%a`j|NY~1Gz2FO zu_rkE!bg8H!bmrPB47QutnY|M`18i3eE!lyusM|F-OKgIY12C>sXF;*)R)j&9n zu{R|^3hv6~me;U-aJgs4uo&Q>Vx{YjbrDA_7IiQas9WOnc1x2m#jX08`sA6=P)p0g z(uQeArlz%F&2<@^=C@%(RorA8T+_CZhSbkavebl#xk8Zg*zMH`_iu&&I?8`8Y~EaH z*i6k-sCcQ{Je<_X8#a2cG4o;fP~o2ZePIgRsDrG}uiKqF$zDCeI&9CLnf!6h3trVH zIhZRKRd>xUuw8%WU@kCrBz57o>3b@YlPsNP#NVelR7|wK{)U09v}ZyjccN;9`G(vZ z>c2l$^RletG0}j{PhV|;D-9zX10hyVC%E{dwh6_8}!HO`K>!V0B8S1PPG~R5FTZRC39^Ss6zh<2occYOEPEQoB|O zbq$=Pi*fp;I%g-3zE2^=vi|!jsX}>FMDsQg5;zQ5y1pzXI?XqB^Fh5HzTeT6#O|@) z(#&lhQSu}}4Rs5&vIarB>sLh;mTtlp;xHB}HbH;(h$R5iY#si1o*R@r4uLGV>V(U# z>=o5t%^k1rF2=6RjMwjE;AKDNr|5LPE2SJ9xH_N}O=KYXu}vM<_mcMgrmI9T$#mS) z5*c4JT9ihshsNCAmYlI{E}}E@S>@2nAuQeeC^mCX%vq?( zaX`iqZKzBZ`4*mIMd|8+fm`=p^>8oPIk@U198rfDK3+iYs&lr9$G6G9Vi)bUv4`)1 zUYNzAYezsZFI2X8FSMz<<7EWB*^B)PYq-SBy&Qu?&`|)pG{H_l4;E)~dob+v@0q@* zULu%PIT3MuSFtAo%K!0B5~T<(zL+$2EI1HaOYCjwu}$fTPG#vm5TFDlCv$`+>vrGn zV;$lxae%jQy5Xi~K`IU$&L!vr4Y(*qv;TRUdOjyulih=zIXW0K-VD6j3x>^EcpQ;F zihcIMkkm-}Wr?XPdZju$C7x%e;~dE|s@&ev60JB;Cg0PW`A4nfw?q>NimeyYtz(ca zPZM5#ufK)%Zg;1@ zC(28Zfr%5R?h}eFn$*5zciZAVvEH!ZPj)io*-em)Mo4&LC`;I%($up*#4bE7oM?P z2_wTGDwa_;-l-qpT~)a0)LcN=c?rK0>$FcK2dVG64zWjW*cE3vtTplw1tNjw?tiI88eAY!HNxD8bXzRZm*Mw3v@p}~O3bBxnR|pnye3`kxiO-wM4>NSt3i!gd_6KcUU$wLjf-wIwZKd*ekeodeD{TG zn}L0wH%xBaAwJ9v`!#tQ$PbQ$_S_B(7!rgVyoHAO44YX;*Rt64TV?AsiWb!~E{>8> z1$I0D@#1TpVAP#{Q!oo06;sYE_6m4I%(&WUjf31&wU1~UT(tt9x+(^eBemsH!UN-I z`LFtvWb09iVF{or{r!Xig?4!HCM7Rel!>j;He2fnf{JQRIpm$KWeNu1Q({icB;kaI zz+tErUp=5(gEng)bSB`l|1KKp&dBJ8w)s)i9>{64Cl#HTNX%1OX4N|xjDM9|a5?5W zH?WeeUeyostZhW@97O$`SHkZQEFZ8#ozbvCns%h1F530!%-DRbDZ!{_v0}Qa$T7~Z z9ykO9c8sSD1(t zVA_2{LB4d6cgz*{iD~1ceohhz_{KxPu7=X7FU`p-#%g*EA~v`eEI1v;86ZRYL)T`0 zX3Y!~&;UCXa;UOf6t`|7x2(lYW0}@uT7Czg)X;A0WiQUt-h~_U14%3q+SE{j2Zs9> zeE;kz4!7i$b&QIV@+p%gSL3RVKKF*069LVCdR`fnL3fqr38PVD_1Wquc;Ar9ecI*< z$<>%!ywf_aysT=2u|^5d&)YVtjnTG#XSKO-i5U@7e2D*n_HOJ}9?prN5zeN$eiRp6 zD%K~KCkS5qJVRhYD#Z zaG}NJz8_>xqw84K-vx^3rSQ@bo*@f7Of?P;hfszlvp8W$c+DFx`fK4Uzfz{hbbQJH z$*E%ka`!25^hQ=YMz02IFYBBB`R3_U$6^OR8vNT^ki|VPi%q9|p1@=piiCO^c@~Sz zLQOWgcDUd5a6g{-b@2g>VHory5_-^R7u;vhaC~s)gn+a^rFy95o`U5V1r8d*#blV- zi4zqCuzyhm?JB><9PhJ!+sBaE`(`h9da7;;#)@9XMmtdMT)so>L}6e47N{hHf>es@ zP0UD3dJRrG%Y{7O2{*D3ZhufgOCL4mkdoXYe2CfSr|<@qd^9hjhTf#?JVENC(YtM()OYk9=@rt>oA1 ztu-0z=^VIQtHZ8jt&9)DvSh6`TeJL6oO(G=Ub}p1dOdD+U6rakRjWExZ)D`E@}=l( zt*juD>zf=4%*+fA0LI2C)-|x!92Y76rL(GFZ~$mvWMpg{j*bf8`2k5CH8JS{$noI) z7sg=m?2ex>KhOGxhoQva|A1`#;qt_c{0q1R(9VtX4{gDX!Q?T&ndR$UoD11m?7vxn zZKP*x{Fn0x3tL@p;Z93UW&O_dOUnMZm-fsN!TQyJiKVpE#HF1417aBWse@>M1!h-Y zkXgXxH#Pxa0!+bF$pEyF{h{Q6^DAj;i(%rG6V+7YOHJRi)i*X~XIBsU+QX@;$|V;8 zhl5X)hXDmxx%`VO8z=Xwpyp4L`Y3S)WKWiU+2_iR_UttrPz_LJKpjj+`>X&MfVKVM zVo&{+e!^9UJJsA>u-d;4BQUnvSiHY7w!dC$Vl8ZOCvoyI zzZnRS^mHx2zecz3F0<4B0Y5;%zc&A1vchuG65`4#^fI5qv!R!@{j81+gOi)nKi+@m z6Bd)<0dO&K0GPSi08F1E6}K@KwY9eXWbFv|w|rvepLKGwb#Q0=e{^kWW9w?;`Tyc3 z<~GJAe=Be7Y{#f-V{Y#ZlotCh?@tKse=$>_6MzW-vmY!LsFRC2Zs)&^GpVKaA>Fn0qQE0{YOnf2d8ZvQ9+ zz{p@@>+~4{uyc0u2AJ47!2R7#b`}7miLJB4U-TbB@#i`Ie_Uh@oE*&Ew3$Ao#QceW ze*XKH&c7uf|KBe7KhYv0wr-yEY+S4WdKPXL05dzs=Y^St&HLYCjsA74{3GMfV*mH} zZ%+e&KsTTf+|sL_N!+qMv$DIe81F_fc%wY*)5jSf?=oPa6NYwC^bxrJ-Hvc0oD<4=OeFn%F(p>+c zr(jx_QNsc(_U00pP`TKT;VN(^Tdgu)#w#$p!|$hh+ExVLjY|2?3(sIrQx(l*-g5ze zATT?^qCw}Al2{dS(qZ&mvXU!j5Gj=p_Gt#%St+Cpx~aAQj0ZenX0emDs9jjzh=DQJ zOo%O2%J2L_TpPztbT|?|1|OH&uf+n17p=^NKIPS5on7WnT@zj@rPI z$()wF19Sl7*=PxozQH)r{gKiUq*z(!vuH~%lGjRqbjiPpe}{sKRv~K=;dWi$Q-(xK zjC0VTrMkq*rR6IZWkb#esO8heVWgsn`JyY~7GiV?$!wtevHFT!yd&hRbG%aRP^JJ- z4N_4u9cvWT5AJisoXo#KJsdlInT1eEbJ($$ZKMb{(U*Yfr0WN&XR$fV^pOAbXj>CK zIgT9`x)f2jNUvdtb+6en-tIb z?GH${Z3yvh;?@}*Q6}!=1ibhCC2bBbuO+22)ktCk_wr=h3&?c0c3UATJt=(?4zVeY zwqzKaYC31~UKAsv4fsk9g@IX>&HZh>Gzfg}mBrLYtww1=F~pXm1dQzZt<}Rz|Feev z80E6RuTcatQvD9Au-0Q~QMvISj4!OUHZ?j^AmOkmB?Kol+;zCt=;U}ly0WC@?67iH z@YU7VLXCV}`&Ub_4x2vTU!_~fScv=z*lysc4~9Q5Gs%drv-n`oJ@%mLAM;v|c7I-w zn|Q9!x)%+3&N_%_DR*ElP%SL7S8@>sa^5s56u*+mWbmrEy|fU=xj#*`fjedFCB(5t zDgrz~&o~+c35jyRFNxk1F~kOTrOeS4(Z4p8ZgB_TVVaq-6}2?mY+h35dHh;**_t~+ zU9W?G$R8!@A+)DCn=O@k>_%ESuZ)R1K|g5KTcL};6(jz$4S%p%9MeKcXa0~cmaxV@ zW~kED?rey169c-S-Xs8sWp!Sc_@bZzD`YRefDt3q^h}VAcu&!rSk%JjvrCPP7#2~t z3o%+2_g-+y*vjK5J?>fIwFEv{_I`r6SO;}boc_WAh|hitz0$&~av3;(b?;^pmD4u? zkMY2s<>*Wgc|Vhk2WI)SNSGV{LH{~8DP|}BD}g1{17 zaUfkx@CrR7Gd`BrUFVj;paH!}@eeGy@nW`JKC%mZVO)6#3ZD;fq}!Nqhd$auUW{)6 zB!~3R`wJs02#gdFO`RW9DRY>n0=5)bic@)BJ;#%WJULmqml`?(&)Kv`+rbm$enJox7t1{ zIDp?vrX(`PQ^q5d&GO?(osx!nU;Cf+=8)umpH|qQQOCs`L3!{se9gx3b$)EHLBs^ zNC!Oeat!xY7a$csV6V&e1mzC+$weOki_<`}#X8FEy2d7zXgR5oGic>Wym8>%2PMt+ zq72Vo8K-&yKIv4h%F1I_qzE_P9&*%N&6N7pj-R?!=mAK+ea) z)+vV<)&X4iM5tdLt0L-qjL*A_&)?OCvdKDt9Pm%U&nVJcFZJns1+A~%@qaGP`Qd>$ zbvy+R=OJjf;sgyl!2)WWB^3_RuOgD|@c6~$5yiHv$7S$~9W*N|2FqC__K0k~Z+@BG zhH7)eroRu&K(W55*YnkE0MP39_iNGfdSdXOYUqXY{CkpUVF)kfyM)B1MO7+ut9V=f zaC?=(62pDto3q%5&PyQq>5o^3LUZOicB?Wep8eXK0UUsoQa3j+*t7 zx1Wj*`fHynauaOqeoZI4kMV1LWmcBd!tSqms%Ll37ytI(y>XA*Kt zCfEm-pA~o#%Q*f@4XBsAwErj>?s>zdHsq_JGXLQHNmtmkM>q6;Xb_dz3)#s9U3#Gf9=uPmWEOQ^C zo0Y1Z5#C@x&KSxFZYOsZ=E(k_1y%0cbc~y80IR|*@2x<~bh21Ng?ny!m?=|$OzshUCz`ndrPx!ZKhPt z#2+NoG<+rFz^?kK-3D4~l)x_lI0xjCsj0H=e#+{{{Rxy_lR7uvkOcmfr}ni%ye#0zzpCUnYu>BrVTyqEMFA7k4QzTx#u_SGC5V= z7vi5b!4C740?26NBhE?v`+hq_P)!>#(HF9(v1zOvX%;dfVc0&)o z#Rpx+^bWdLS&M-2ZeGYYL8$&h%gQaX3Bii)jJKkR>!-qoxbVnVmBN-6UI+uP2RDt= zkvlnRT6s2Qd6nn>c;1oxb=UwS8XPEJa9PcYKj*_oD{|dXYWvds4aUk%QV8yF*u6NV zx8T~NG+x#O>opi*IA4hRS!82NrY?3M?HBuCItu@tiOo?JGWq-Mo@c)CaO~=gg693g-u3s4fg27d_em->!4t$lxiQ%WDc@yp!Te`Gj+guGSo1 z)S^AySe=_iTy~w>C`Hl;Z5sn;&9m(aBAKsHXluGXwD>WksXsvuQ^#E&%4+Tukt0LA3?Jqk?7-nD!l2CXR? z@PHqz#pj8qkpZurGldFS|+kS%Ay7{<=2w( zpk;B~n^*`ebyrQa@l1novAIsFidTQt-BSgtN^g)k!(RV>m}wY-)k(-?Q35@>Zi%zj z8UAB|QK8SD+|?;;%3E-9ty;4owY_Sa#^G1aP4=@$8%m11;Iv5Ap=8vDYy;TJ=*m2P zVZ$~O{wu_v>XR$w+=xqRx@q02G;FIl#izbg-v)5dRMhJR0cd)buWK$+ilHI?Z%3XK z%z^P#G8&r!aNU5-9-7FOc)Ybq*Jzo_?Bh@|U-sAi;6Zi$R_HR$Y;C;WnkABEb%Fw_*tFhK zLdE(Z3cK(He>E_zq*-vU0Qbz+c*55pVG(H-OoX$yA~}7;qF1T|*eTjg>X9M8T*__V zST>7i*b3LxpKBYb6X6^{`+kWv>^Dz~MJnZv{P4UUv1lg1?sw;j$=V1pcC(DMoG)7w z*H)K;)am0e`6!xx;DJ&ju`rcj;MLDUfQDui6tEqvm?Ohx(Tp=f&iJbDq;N9*iWAjr;D^O)gExC=+ydbEsr-}|jsAlZ3z(S7PQ2GzCp zT<4b%E9xcp0C9{gRlswSm~W69@%j%X{Xw;KQO|^T)rE{C5z*Z&21-X2q|CJ>{x{I! zlR|?_Dg$hlC+U`okrVvq@2O;vwydY?w$vP2>f)eu(W811GKsvKHpUJXVEscx!ee-j zOmGa~^3{64AfN7$y=({{ZZ)Q?;j#oA)!5&Dx-!lS%_6qKY0T|e0_O;~4OCyTw3X<9 zI!blVT8*CbktviQW-hCy58JWV5hch_;U|g3&3k;)rslGbEoJQjeK6l@=F^iWed!H+ z7CT}OF^>i&6S5vgIC@i+}8}DKmc9>?o2`0k-*JC^Ux?>^;4Ht-%Cj95{F1P3K`t z?_=R#5i+cDf5^6h`>vmUnfToUgQ;lnTf*ywOB|0@3m=I%{7xqBS7#ur5_?9@fu}1L zS9obit;sxXVIb~Z#p>^Uc(~Of$$d+IqJYb&=dF~iw3Spk_37CyCeJKI=>Q;Vb|#}> z+Qx*Sp>_y+w7+!O96UC;oHcc%Qi_U|nN4@Hns+Wru|?S_e)Loyjf}O%rI|q5P7Agj zN`0iy?PTWg_puz1W1oulCXSMagYI`lKL4N_t0*d4FfRF;ccWp zrdNR@C>wwz@^fLt$=AnRiC@Uawl2o;d{2ix%S{vYC8&@SnFU|IbHux_sP>D;KjwE} zl4h7i*FLOXct_Gj_|(Qv?w+3L|H;n0d_1C!xWa$`m^*w$=cQjl!#vQ{w1Jrtn+&*; zSD7HdPV3UqRhSlQlVl${6Liki=uJKDw)W@W_+T$zVnDBa)GKbJc@v_<9&#_K9nyJF zEHO~iy}!C3*;Nos+}0XV(;}6rC;^^rd(d z6*p)2`~AXx>7hx>rpn#9;vjX8ZF*=0W>$D^-&YzCRSDCHUylXY(-PN_=dlU}=22s6 zcM;m~@+rDO(gxJXOkHhIOZlz2aH0kKIbmSHu< zyj9mfYh_lr`@$atJM$sXUFtQ@b2|)^t47-3h(&(?$oFxqN+n1V|GvO`hYl`O^%@c z>JcY3Yy_sitYwJ2$BbpA&{c2R1`!8GUWAAI;K4|q&c@=4#;BsPEAi~Z%?sy^vBP~e zZFtCfW65GN?|vB<+)wobmOQpSFj0G1kz1&uWY1*T(Q6&zSOqO`eRjG5`rFeDzcsY! zFiXd7<6ipKcG&S7{#nL2;#Nb;Fk}ZwL9!dKb(=*D)<6+ik#5u?5 zljX3!^>@e8@bGlv#3-JcQOGpQ96zh$MPAEm;O znOI>w1H9D8?km@1{BX9mPar{?mh-rQxD`Ua-=#|58O;)mYwNcwBTjKSSN_o>;Kw?w zP({%jsJe~zBQWZvO{obwAR4Td!HV|Gx;=CuK?b&)9NoEGop6av*xzjSp)eSuDwOM@ zy>Ov{yhnh}?TeTsCs8buwB<+dtEqjOf*z~@{9zJJK3$>UWT<}7t+BW1*YiGH0cqRy zp03;{1S0u|*;k7Gt?UoPr|n%9<9y}bfwKKQO317r9<$g7(`y#CNEGU0y9SXhBv zW&6_7kjz%!21fG2!i@r*>bIw6dALgVK>GW#y+FRiRHY87r@cW1fKH1yeq%;UGlVD% zkCPKa*!Kw9YVYM#Zg>|=6xLt&cI_%gtHzTJ z%gSU4NeL>4-h5mkKi@*4n#)-h>|0_Wd*fviSoCV`TW(3oi-&Qv3}$$In6TE246i3v zDdKh7)EW#|P)m7|CRrQHWSb)@GOmv%7g?vVphVvzrfN-S8no;>lv>E|r^{$F^=Tk z=R=qzuH=FcncCiHEu{0}C??|6XTP0~!E{Xjz@6Dz2{#v(vEh>nnZusLBP4m{wbT7x zgt1+CoENZR2tk`#7D98~r0az*;0&kKWk>asq!k=2h-^7n3eaR>k}yiw_?D`ZRP#Q~ zlO#;nJNVf5+O{dr2Rx1oO%j&y7RS{s9n6bWJ{6+1oR6karogve16P1tQn6eBEf>Yj zbjdJJ&scv?$V$@cLa;_y1DWcry3bVIj=^WIzVvYu(rHzDCmI^K8Ml1n5A-g$!)YR_ zEWyEGr8Q)zNAT*DzDDxcAgM}5bGJT;+d%08R^ zR<un^3C_KGVGT|Lnc&n`ZHl!@gB#F;M=(=EHHJIy_!Mu}74K`OM3dobGV zLs+A!Z%vZY_Jy9AEVLrOIGpy};di&o*QKS9^#WxxXiA0Q$)WPz+jw?S?r^Y7YnbuU zpWFru2IbZ=>YC?=fF@Gpt$HD9k<03zrrI|UGy-aaqCdwHA5~*-IgE7L&DPe#lAp=I z-L+~lj{)rS8)s!pJkUZY{f5>{l8^z1bf})wg`q#!fW|fUaG_OC-$Emm%Rb`x2&{p0!Gb>Vmn1Nth{w@pxVKU6h$a-xy)wC;cz73 zM2Mr?ADY30C+0Zq1}viakC!SNm&$t*)*jplbw#}>@~EnK;dH(g6h+1)njX(OLrvc| zP%7z@BTrc&cYjA5k-tbVQ${l8R=tKfekK$ zMM#|meZ5S8))zMf&PIBEyhmlP*Ik6%fFIIweuzKk85y8Mf!-KJ%w=1;_kTQS=NG8g&E!kGzR$Ud~26X+G~CZdGMPq(Mw_2R1vlX1kG_bCINgN<6SuX{hR(+yi@|uedUV& zt`9?~+9bXNg*_MbI@dr7J{a0R;Af_fNw$2>uBr35e%PSBLf8S1hZJpg`oD_n$PcsI z_H=FLX4g4Y%Cx*Ya4Y3UmR1*XU4M@-vXKfp3< z;6%mA8)hDP%?b+iM8Z&L(E3A{`9iFy?Bt&TqAXdnW(da zNA{HH7z=<_-2Uu{yAGY^s2&(27jBQ2;XCg}ya!tWq z658|%e9OETh(w^}tTrdT0?f7r&r|m$r*8Xi`~Rjc8hW5forufRyJTRA81FV-L!z`0 zXit*w`6^=1?$7#zM?o)0`q!a%7EvQ_jGb%El}6?+WXU6O&v_v;EXO@5r9v~v>1_7x ztwG~GP=;%}oS5S_W>kwiDY=+dpfpBnSxU?JE87^dT|;H@>d+2OsUX14&Ak6jXi2Cf zh#6siZa9OT;QaW_At_dh2bV#u!+LRc4rGd;M76be>BtTe>kW)4yZfUy!w=3>TG;ce zj0(I_a|)AG`1^Ll)d}uqM9UED?sFR)q#F}1IH)G*8xP&WTvEi4DKaOCQXg^y%EsRMFQiKm2goSI#u49Bbe}^fAa;^>U-@-Ep$t z%VxYCmay{N6E{fX8r2P=+SEEzq;$N;rVK*9_AS*{f-g-a4|xHVCf-wJvXmy;w-Z*w zF3?g?vaX%1>ck%}ss)Sgt=~JK=C0e0(BlO_(E<;jF?})*1tKa9SFn*aP>BwYn#a29 zuRWKibL4&vA!eSt^sbv5gxY>-sL^p6fsIj{z^@a|6*7VL=Df6a)BR|D@6gDB_`}fX zwAY(Bxws+L?I1DzjW`+eDmpRAq_!P*Pqh|DE^}HPzeFC@J4mBqz6Ipxz)4Ld9m3Y~ z_4)9Zq{InW!KT^Fa;jydl|-ZG8>4)tRwz+5G?Or)W^(+prjb7z*BZ&I)llmK-xg%> zOTuuYVY<@vF`?${DsWUF7#4%{_;u4JI<c}5J9xbP=}jNjZQQ;lHD9^ z2y^0lg3}TfcZD#lZkXKAhV+ibhy?!WX_#FJG`?EVgu_O!X}Z%vKOF3z??(*?x=(cV za5^sLFjP$lO*;mhTfeER%$K*_r(u?wjp01pkP@27U3P874zKenP1ne zX*bN4OeJ%6mh!DNy%4m)bU?{3u=vV^Di2PZL%j%adnntm!X^ue`0%g`tPaYa>zX|H zQt&0&yNc$R81>SKqc^e0r`}FSH?`mw>5s>vXH#FojTgHe4~l`_>n%`ozC@s%S?fpH z`YD?0)5~k;H!NhdUdAf7X)nq?nGdNfv3_p&{4~Tp2Q)A{(nQ&*=Tzu)8~ay2 zR2mA?auIN+CM~|O)g}b~DQg*{(m0HZX=c{#mjF=`T!!$PAzJ!Yi{8ZSrx7&fXh5T1 zA(9<@8su+E96~XLuA#)~=NP%1I=lf2XICxsbd3|!ZLG=8jm3+nKB7FoF z7Cz-ahK~zDZC&JB{T1;bLU$EK=h5VN_wqJ=4NY>Xn6$6{UZewLJy@^Q%{j6~ntXwl zM88#7YrkIIkE7X%qN zW)r!;VJ1Sx>5Y8F;9UqiE;5TGc&fx7<90x3{spKKc)>?*IcV|(Jn>}^ihrKD`GNK& zDJF6#odW{p4{^J0_{aE%bS5^W${A!nSY8gCuD&ulzojk!hxxvKce8!W_=?j zzW2LC!qstASBc%WHc7XHT~}$ivSt#|jN2%GK+5W)CHl@-*8brjQ3O-xCw_wmPfpK4 z=ilK@95(~HLE^>a219>Q6z&grkEyNR)8yPhSYh3D&*^Y#{j(OoB)n#bfy1({G2=*W zuaMhLu0knM`Mj+b8tp_%NT1RY#akXYVP|%NHN~4_uQGLwP+gDhbvQl}b7Ip!LP!c) zyV^|<676-26pa?|<=~JWZ%38FypO4UkoDzEYN-bg%~s!=`AMPn8(pjjdZw=TEH~O# z*K(`}c5ryb4M_&JZI9*4wrZt*y1uFL6GWhsPvbU6zR4nlvFoH2 zjf+nA!L;fr0vD*23{%kOYLql5Ll)j4X3!EF~!iZfvIYp)y>A>E1GN=7DLIauXdbKjpi3j5|TDis(W(OHzxiSIAl6t zS{=7wq!^qV)n(J0My(+6M|L61KL)O1Gs~5515TP)&p55V-iV_UhLdq~X?rjH%Hn8LKBxzQ<%k=fpuV%Y8$fsp^Ka!G?RHu{rbr}`X^PTn+TRJjg9!sScR$o|s$(X+ z9|2PhyB8aa+N(-ue(_skb=?su4qBYzEqIpsmEpW+!-}bB{Ngpd+pWpbtLhZ9O0r&}z5u*>% z4}8!SG68y4@MsjAZ>0LvXGbObV8n3Y zC)bEH#Gw6&A19kSl!@HDd#_r~C$HHuoX_CScJ?P|(G+;T8$0&nOwNl`)@n`jvu%Pl z7x>{~v_!*y19t~+xVzPVOUX41+ekHPY(V;&LEF@fkzE&7gga^S1_Ax>!b81mPxV(+zZ zLz8<#yAZC)DP{~}g6GM`HFecjlRP6VR#!@OrZhXm=lWAgF^wuBcmLdzee{61yg6;A9hysPRj~!r*n}bzsRa7@Ys?)JEUb z`g)C_m-DLT_%e05nE?KH-)O+vr`lv13sI-z{1vANGk=G+9B}>82uK(=3jMBYw5B#EoAkUR5en0L!$mPK1J+?&|nFxroc$GE=wka*z;`8S! zI=1=`W+V9SNCxJ_3eWf@{E*G`C*lDKZ9mNa2%aiN4JklYclniqhfC|=-A+ZIP|sQT(@S*cn+I{3 zHzuyurY{pQ7zecn`=0>53UNhVnpD)fWbk@StZ<@t-mx++XcN1tAi|H8e z-jNb`NjT5@6Yy+Vo$f_R_t2wY3vJ48qU}TU)B41Y{*Y~F0*T|}wflQYQ}+V(;DmQA zVX;QsICpudqlp-$-wv{<#j$4rV=tm(C;|()%{_-fgE65D)Yn#**!Iwjw>CWw#n<-r zS#jB+2O20wRM_>yY5Lo&BeP5f!ATxVd1uTdgeYgTfqA&|^k$~xV!h>g)0*2JP~=n9 znf%)4D!h(>bw=q$q1UjH@WQz+HbXBWoV{Pd5f*FA_*t8^0cG`2>#d+kN z6x&rfH%iSz3iTcvK=?Zr+HEaFr&Lb#a>i$rvkNpwa1grNwjWP22bkpS2mTF+a&QZr zE>0_B*=5w+l@%~Zk}tDyhE~P)+l?7}!{uc&#jX%U&hRYTRm&_#UJdzomKooRF5QB+ z9eHJCTnaqU*@tmwdg>K92>yuOtlvOzwn75Ge2CH=D=_0+=G}R-9)%6urdsjZv@oyn zC(JR$Hz*!9Ltrez@ZYkau=!!Jk<34h79&-4&v+Puv|=Za_gxVz*k9C~DQ1xBkk#-FkjI{9`$o4bQ9GjD6k&gHC!{+*RKWQO~&->*+@ zsJN(j1^%Jo1>`!)O-eNU5=fN6p3J4=8UkQX7fV+XW$>>t=3 zj45T+4;ZuwWJbK+`c4yl3&tXJnRx^! ztkImy&r@lD>G=-1wJ&5kpP%xjoTUMA0ow}%gXOi%8bowce&X&PSlMLdpsLf6^BM%`OKijAk+K={Xx~rZR6x5o8Ajhcb z_Z~bt*lXv$%`e{B_{#XpK4=aZ!q>v@7(e|qkNJ)PeeGLcGy;L%8P=Voj__NfWhY;( zm6i)vaZPlfUexGq!hFnxJ-!nSWA6g#X(UKs1CeB`6&dY7&q<%yH6zE4&vkHqL}I2C z!XF`=)b4f#K50GMtEWyTX+KS#6z$XwpGLKZ$9{_pyg*b@F3QO7?kJHXpb$%>g>p!= zULQ(8b`Pet2JWM8ghX{B;5BGKn5)h1xI{*nBH=DtEHyQxLb?0uGg9<0<@Ca{~T zTSf)fy_`)#xjSm@&PP{;eXr<&E0iW)n>R1(-XE_}whc2FrxT*r`It9S`!=u|x^0|6 z-=!|4d5`>C#1#P};!~K?(7uR{e31vv(=$4nP6W8ulwQyu$<1WpexAfWMW?W|y3$3m zadmhoe1cY8`J^auda=Nug?ss!^8mClS8|t;Y(889RK6y6^J-2+#Fc>f&-e4|uL0cn zQzW}2{k8=F&X_O53Eapdnw>$JoGxisT)cksoC^lUb6S80iL9DpYIf-&`ki)x>S*#A zG_?_rj0_Iv)xgUy4`W{p7_+GQqNBuWQ0T?{OQL@HVQfeG#+#`1>CShyVUA*6WH*$j zfe(B5&~J{P$d&tG`2IJ(>?*c;JXRMwmqJV>h& znC51x*=x0>pG0p+B!|7Ahr??}JZmi4rWO}I-E^)L0zhXTUYAZBEz^P}o_0J@yo*`lx;Vyy1x72L zP1>B0ap%>(j$_qPM7kESSPouCAuEtXyY#IGVSRPJ*F{ZFgULq*Ldxd3JxH2UIk^$073@}F zOmE$l4w{M4>6`*Qw3lYCOs8tJMk21T&6hiA;P9C=%CAsN)0FUGS_gk6Q`Brg5xp7D z9Xa?cVo1~xk>`x|EJZ#?Gn6WN^nc)Hu~sU%D;;n>b8OSXpy4o&{;z!EgA&RK#p}P9 zVd_9>QePPV+<$oNxu}ta;mGAPm-zSbp3>js6eOKH=bGi@PpF`NU3N@~PR4qNjWsdE zH9Zhv8Owh+K#D#O`c*8husp{3qm8{ed6s+cR1LFgIMM_L5O9~le}HPls(rdv#~1=u zyK_mX-NWF=IM$aV5*qRPXAWW{{5>eMNUpp%CNfAoOFC7zOBY`!=xJgkq~9(MEg^g& zy>=5dSI_UW`c+i z_HBq4@N-ny5t>G#W96LL#o(Y+5~Prq@zwc6^~ltJDyhj`67*I6ywXvkoziQD%HFH# zqTeBn^VI_AON^B-&B+y@uS*GG62zBw`7pKffooT_~d#?2jcxa?}?N00`@Jo9{>8E)<}10evN zlJXT|=FDvPV`8hSAUyPh&XsI#T0`Q*W=*iAGE{WRNdiC;`TWNs$e@P)W(FF^;WM44 z<=ZM(E5hs8$@wD?v~8*)b1|w!m-b@(c49%ELz8@XR3G}M=GbpvBi~fzF1zF{Z@2Ax z?5j!j$o=0*-FSfyV7Xv*lJA${wj;X^yCuO$kbCToHpS#R{(V1U_(oRWO`1-{bC3MI z50#BJ2ssn=$uq^qL5=nBG$f#g9^eXCV;#cnh-$uBVuMt%G*rC1>N>b^7ISA;Y|>;3 zqGn}eDW9LPP^OCWxKUr&YeWBzED~9O5+rPG>O_Ms{*;%4uyP9ds1anW9A6&qHaF^hKpkTO>q1dWzab+35_iL{yxEXa8&|68x zyT`%2J65K83{)T8@|-EBQM(4aNjK|UgC2?^VI!J^Zw<4lCa5g&^g#dl!{topwUQT^ zPqSc-1_@k(-8)sRTEG|lf@2&!c18S{2;L&c2yvtrtnze!muI>k*KVJUCI~Bg{SgY$ zJO3v&75uj{;|K?e&<}@MuM%(Y%GRte-Sy@U1(X-x6C&{8G`JoET%cBo)M`=~+g@EsvD{uH$0ku(`1^=c*vc$@ zPcDG!^w)~j@#_?st>GjZ>bi(MCEraPNPT%t7g zhgXN{p?|osprBxAD~V663Le_mQq~$|7T^31vwq?8sKU*nc)@$bv)pT6Fsxf}6{@Za z4KPTH{Iv|246fp`;JUSaF}VBP)hsT;Jhuvasjx7Q+32!`I+?*o)J zMYyFB+O0flWrI_4hD`$Z(T$nOBa=B?jV(VLp(C@0G1aP?wycLXNx;*hq+zs1e$?pq znZ#zR(a!c*kkVP>{m%QS{uto({!DgTL};Sy1>4hRv}e66iIY~ZzN8kWuqr@mu1>VX z4xzvu90P{#Cwb(+=z5bueQX<`IIH?Az{)3XcRrqG4EFk5CFQWISW3jY*@r=F+M83K z@kE&9)UC{HnzMKUd37jSuQJ+s;~K`nyN@akzX{MxOhChDgQ$0sU=TDu6#zdt`1Kuh z1kunqg-paPuFt2g>`znHdGq$u#6a7|QUO;tlCfuH08~tKfy_^#?-MHPKO#OTzVf9A zhngP9|0V*Eul%U>i)_2MGrM)x$K62UW+o1OYJGL)NsQb2ja<3v1~OqdZb_V6B#Q=B zuG#;_?}FzhC61|wyz1na^7_c;B-LS~Y?q-3npOWkX=FqcN&6h4YDfPh&o&E92&_rP z_`>@AZb}dVJ2c?qTqdwI7ujgy$?6+}C|3e};t~2BvD4Z21cAsP^2cba-2f9D13mn$ zz2B6@c!5SKu1_4JmhmJrQ+#+kktx=;sloH#o$I{?*odUOhzi_T$;vpmg|#xg63vu^{U-Y&IL_bk=&J_llBfr| zBmxpjp+&@l%3ZnOA*_Fw`j+%s;Up{k zM7zkwwpE$yT7F*_;y^BtsX_H9Gx!%G6R^)10~x^N-5a!)6mMc=>X1-$ZOC$4+^AY%2*7I(=ib-}xiNJK(|JSCSKQh5M02oiq z{T)%G9OJF!iscBSBB~9QTQf(o#AN1}Io~w7k4Y>QbH45f%QaUL4zb(NGSB7~4*32qz5ZTLZpe!nE%_P$QccU8F0 z(DJoZKLz%D@g>iW3O|uylT(xre2s>Cb!We6u|Y!{-^>oq8im`OoW%+DI3-0{*=>1F zW4F0z^&4Uyl64!_C~X53=T{@uYj$#O2h-R{>6_5=H8S*(Ln(8`ME@q5y7AC=S6XB6 zhZp*{LWG>MV!HE)J)*5at9x|uGnsghxNhk#N&qrcQSiov-m(>DL$YN|+4!^VtYqA3 zE&%AU@jGaTQFiO~@kL~Z+wr$(C zZQHhO+qP}nw(T=_7Jo96%wm?cco*GCC*7&)r+ijc!lehb-`jj;tEaQBLKb$kr@ju2 z3;fot&jBjz(WC5|QzE~k-*47sNOJkmA$}S$NSkIq#KreGYj^CUk(1yQt9eR;f6FoM z*!StX>ZD`1tbf}!5m8!Y<(;4}M{ip!*F{G2GcM=^7&4oAXua~wO)ZEyG%~kMgan8; zhMx{cKFjb$$$waoNF^CE&;oMrmVMnkm_V{3{Y)7`Y9cmDnD`%}C^ z*1|RQ-c{*t*P6@yw$L_*KFIkQ*}zA0E25&ui2vn(cBs8B$ zHo`U1ii_z6opER?5HzE)a-SQIN0ZTYRHCP=wtVek~#(ZY;^zj^OaB$uckZ5ia%XcO|7c%X?3oAhU zp6l4d%BJ!%hJG#;5CRYqBW7GDf>!QVi$nij-j?wBY(WrBT-87#k6GkAU*7E608ZA0 z5~a_xk?B~gPBsB?K&0|VmdmtKsOOQHSwin~dnv<0ZCCP|T2^&Rx>;?ahdBgg6b-nA z(6@YW3>@4&IzfCP?4c(zpAr&c?k(w2WI}2;9U-X8&~cXo5d7K#5^VT!!0`*Q(zve^ zJc%cqnI@9DQFIn#RXj$WC| zy+*IG>RMCfoa-f%IiDklEtgwaM_dt3dsHa(B}$ZfD|Yh!gR<=K!aI`VUVJFz$9mRb zf`@iLV8!j&-nIeA5su!dki!+JhBuYwZI<^3_5cu08_K0^gCJK#CvZ)7(Jr3mvr5Z% z?$Nbz{;ibqsF-0tvM1}S$YMur(Cx|Fejc=mH*1{ly*L*z9qT+@P$Cc7J8-N6W%s_E z)cYKI2=yGu=e*KYOD~y>y=d+j`$>^#pW)sj3wNI_o@*aa69baA*ZD#vtskj zqe>CF0mXs4Fa^#y3GD{@w)m9SCV<8`aywaIRrBT8dA761dWcicAzG!$Y)z2aS+n3G z_af*J0BK6tDa>Tni)lEE!t;w(k>_Sa^wJoJ@rP59Gb|3JqXx^!Xvzrz^e-4bP5Dt3!vSn`&8FwnZh9^-zUEq0Dt2 zqwFxe?dI6Qm+31zQ@cqLAW0=;)7cU#j>u;+9sa*wQPwjh2;02Zk++AhJN2+`9z4&~ z&}I$>#D4mevE%7s3qw;!5n9MG#OuHzjuBMgm{@}bjb>qZ)c|-qc!7~W_*WIGc>7LD zRA;~=moZD(y^xmSyc@=htqBW4bY?1>E>B=!Gwt2C3Q9Rd&llX}Qd>%#1U94F&I0d0 z$!;ZqTgY}Az0%p`UJrc0i>_S59XE4z)D_eF)xPpRsu_~)OLl9Pj#5h}U}X?|%43?`A$2>=S{O}h%9;)FJ? zEDpxi1;7Z)j|u_ADF8+TV44eHk{?x(T1FHP%y0LJCJ)TjRiB>0DHIxjPlfeglO&@E z()Ia;v60EmV-*-0^i-W>pQ%(m5VG{lP54g6uuCu>7WCln?*EbQ2VenL4=Br;BaCShy z--3D;=O2rjgp3U2zP=@}BVeGkz-hcNni86tng4_y!#Lf1;H_N%zkhL6(h?oq$sZiY zkI<9e1(N>wB+N~ZkIc@W+nBUK?X0e!`_r9CR*=9f>n!c8pBe}dwas-v-!j+VCKJ}Y z9)pPkiU)>;3=JxY_)&m*@&JMxe>E*Gt)IzXB^hlAF~q*v8K^ykJ@7h#3?*!1JO~#% zzcLrmU!&Y4*kKa%`b5&r-*XEN#5LUGZ=Zt{R(!|_-5HHG&PKBUtDR23qvGF0XmGsW zShYVbfGfZ~4Zyj9?aXV`ZJq50%IP~w==%QE4TLL*dgdky$otY9Fe*ee> z%K6pLS$`iQBRF3F;OI8!;(7c~;J2EKM?48S{=FM|;c=hPp9SD&9~>p{ak8erz-ul5 zbUq1V)_>0JBEs|EUv|^qL83!Tn;L@)z~WzB$zPV)rM}VKN8R+_#60NlqGTPAt0R5O zUmgRN83q=7en@6aRL9HS=r2y18$-x*FZ=i;F2qK1pWtR+Y!Xkg9f4ciOkEXp|KP*a zs$YzN=HfOSh_fr0{lhN{NbveSUm(HmUm~>r(!7$Qa+>}pAJ3CoCi6J9wE=J=kp1I5 zkaSMAG*7|?K}`R|5b*t}|8q0w{861OV0}YiKW#h(fS4MZo)-zvt98N#!6P5$e`mrw&EkM?{-7u`a(t8a4bz~o6 zu+9ZrU_I5JD1vLs9>ienC;VYpebvu@GUk8F4=jQ87GENWu2(w{gSC(UW;*|yk^93V z2EbtY0e28c&fsJhj2)^|{Urk*G`_U9H8Qe2GQPdM=YpGH{ffcYoWa+*v$ngm{*g-# zO8Qb;zlZ}cMlqEwJ&mW@T0DuPpYt2^%WL$j6kN{y3v$PgKGpXU$NOa)ocWq-1Bu5vTl5JZUzhd9rqu$ z{`E&VX!YYChuiuABtYH#10;C-Mfj^4z_G@k%h@A<3%Bc=aK~Taj~DtIV;@GjxATej z-1mbOT0MLBLRXw#Kz@||UORPi`X>9lqd5wt23d?^-QU|9gpL9{kHn1#i?uvm)Ea;{k9O8NvA;0}+Q^+cN_g{8)>MY2sMFkh3Dtx_AW- zhH(8WdcYUq`EOmSbA0&5THU@@p~{ayoIf@DdunZ)b!usSxs%s@caMMFOFtnjI@$fM z{5n|B<6qZ!6qBX2vaGmZ*YRSM;KXnrA9sGDe}wrTKdkp-){v0(VQkQB$1sidg0!e` zw&OgY79_ZPelsk8UoV$+V;si?PkP^1J754Ho&YlmXJ-AUb-Np-tih@85@^!Ng9{M# zxT!}G=M=>Kvt@G4i2uY-e)B;hePOJj#`Aw5R1^O(KT1J@_$(HAgjxMHFJ@~$yetap zSPw1BZN#DLXnGTig{g3n3whAyHwPse$Ptj~^@V?ViAo7VWAJ+!>nY~fJkf?aePcX* z^OP7^Oex}KdmSK)5&d^6Q^&Bu6u2I?g}Iuio^WKfn=RtrxsQq^e93pifx3;{$X;bX zu&k&Ts=w^o>GJAdh&(Qi3njQD;@4+ApRyjVn!=p_Xf;wy$9Oq9wh7zOn z8U{>{+7oyrF2UjHN$n5eNrRH%$)!*?ObeN_uJqaux~gyV@yN^JUP4gjVGc-CnloyE zFvRzmTfvDY-X5z77tKbR4h$Kp77Fp0CTb;8_i%b~g;sDga98b|4#de~nJ0nJnN_2U z_&ov}7Fdyl)PR*_p>?@A%{D3JO7uVV45VdH8G~t?3$UuACu}d96q2Zt@bIQ1Tp6r* z{+4O{cSYIqD{5(?7*g4$SN@#Cf4VWWi6)hj7}XHq_JFuP4H0%V3WbpF6GlYktP+%D zU0F^2>eB93J#8YgbCoLs>`F%k{G_Hf3T4E7I-ApFMLeN&O3xsoUk15q%X3suWfzhQnGiEV(PCTzOiJH3{ zt#d0ePK$+a_Y4zrLqhC2Q~C3dFp?ZyB}uZ@iQFCLb=i~g1GxE&A64t#_uAO66dt3e z&?r`0jL#TZ%-6eRXCp$Ovuw}y2(?rspY|%H+gNaQ&T@N7EgF!SYrYeab&zT7NIPA* zDYE$xc@WSvqwACAQ$OtW(VZd?Cq@&ZZ2@R2vU-$$Gy0lhB&{ujFKZ*`SootkgMTTe z)EEd5H)LSsk?&ISBEk}^y#EUbLg!^hXFvDIdXp*LoPNm*xOpLpB#{^d>`OIXy_n-J z9iw+c1o|9xjYwYJ)XFwe~DvZy%PCDDM|ZPxo#fv0v42Nw`fW1Snul>qlQ6-OC)niAM@pcHL$rZ~Ay+pjLKyWnd8xXum8QkD+{aq(!#-*)8rAYiz zZxus%&3J`i;YkppabMX-__Svpum*#JpS<)BC6}~LWK7z}CT6`(i3r@G<9eGlN`^!8M-2sRG(Yix$=l2GaF6yNhC#N>-3jDaE zsy9FpWFm4YVs#@r1_4T|ie35{#BFRUq5v~v{2+~J+R3-O_unTX9lt~t1i58_ipd82 z%uCVnR^DKwE&hY~QmB`^{F)DQUx!}ers`vYsJsOgR9?^&zC;YRW4!@jPln1C=Zwo|z(qm1dVKoCgWTyyOy_A1Dswtq7M$;C#G30>~e4ln;A+N2i8zar59 zA$2Bfw>+4&->97>30`R69ta0>7W@ae+ed@OiAZPU&alu;JlPT=v$n&5dDfH|!LV<@ z=R)w5`@kM6#U=&{BVx8=lzyf6Lyd^o;Mhg{E1Q4nph4e=r3+&Ay#JYP@~JVUG6fis zNP)R8+aX%`WR)$6t>U5nc7IHA`Xq3M6W_yOq(Gp$rYya9#W!2-!)GJLQ64d$ z_{_Wo7gz%4tF~aI28+dU8&71}v5W7e9s!(wrD4h4&6nDgX^-<^V=!%CDQH2a9vKEp z@Axzt1*>^3@rjqqh|{6FTh|o64nSAjq}KDiNs57WoMXzr$Gh01q=f2j#>tSs0uc*m zbmo3CmClrMYyFCqe%`zdYPP#-nf;o;>!a7YQ!=_v6HSqSwd2MTiNAW6VBn6%z)yX` zvPAIWz>HyiX&VHT%%DI`cH(jw`3yzgG!<2DK3#s^p|nMlX)Dw;*mf5h+6Coy%a5BvJ7{g6_~ z7tKiHxIm=5k**1XCTo5D2v82|oc#jlGB1yvy$OBaivGuo{0&*h`z{Mp-5@^gTKY-RHX8{wPDO-Ad~3Gl4FG(d1EMoM z#^BsCA6fyQmeGS;i_e3jLy?Lw>h-NCqgZ?k^w8Sa1~fULWm7!hEsj1D5wMO!ll{Ya zZ8eGGBP^oM*8>r0Z38V@q?RKrdCOMFll@q(OA2;&-i{syUtimSgYS?2Ym&l@oG^f+a%LBszfZgSE<-VG&z z!UxU!0D=5S;vmSJ>~}Zr8Xf_f`OSJoF*i5PgK#A(J+50mgX&HmCBb;k4uZetlyxwb zq9LU<07NTOSypMg+sQD%_=2g3s|T@pJ#o@xHC55Ht;%Uf3&2Yn>UrqeQ-wPl)ZZZI zh3e2wElnO+g3T*vb3(bOXg!-V$659%`BBu*`38kX!4CX_Yf4tFRn%CqI=YdPRv1fc zr?j3w5lB=*rMuGL8?AM?3aa?Tet^g9n_qO7)dBttCcC>-C!JSGhVBR{@JBUxd!zgl z&z3u8=x4}k*11mY63xWi1uSau;xuT0+@iI^*&}4mWZ>s%XSE-_L-FAORnwOIjHnY& z#E+KtC;ZJ`^e6`STVQb3YA<$Lgt?Yed&-Ca^b?WpbW4UWE>8Spj-F%aTgyWvNHHb|R8bhe7YeAgh_MVtrl-x}%kr;S^aV6zU zDrkUtdKmjEvL5Fu7!oNaba8)JyfBQ*sd`vFX8jhNbK84!}JfI3a0k@KDOT>+jcspEw60F%hNNaJGgx2TY@|+4Mroi!AgZdq9|2a z@OVb&E}>Q{XEQN60Y7;jP`Nfqok{)G`U-Yi8W)`seMpisZoD9xGrJ8`kJ^3tL(h&D@TTPYf{ z+B!&5wVLBTN4ceU{js7VMuQGyy;rl zXY)OQ)xD%L0qIotZ>#_6m(k^BO5fRaeZ3C$0ay>>-eu27GricAgwWCvI7yC7&P6-S zkXH12sLJUx6gs&ZN2D8_rOv@~5*2(51c4G}Pi$~#7=i>2^)=vS=g~jKv&B{8N_bgJ zE~%-S`voh-!-Ly-f_K!Y?1zds4P1Nd=%mncT)cC`H5QsdqMJI)H@r>?eoqiBc8d6T zweOSMWi?)GIGKJos~v47-ZYHq;c&l$Dmk~G+`+u8MNNW9G>l%|>kT`DS0OaOt%6#v zn1wXeW4AL%IO(eEzZStSGL36-#Wdi#3`D(y6j&C0HTj&s3W=dYD-`~EFQJ@XMhf%8 z&uY$@$$g+y1z|IJMxip11kGkPH+QwxT0uMGvI@I->UVwz-i$6W+>g+90kD#5vk_sH z4PWPeP5;C^IND`h5v4FhHFHd0zkQ|@J)pVINU0ZcirFrY;Q=}H&lX{6Z#%MN2gE0- zmCq}&4^^-+X7r?lXCi^YdO4V_Y&p_Hu#4hC!-a=YC5r30*Nu{bd*^gJ*1|iGhrNMNg{<=`fsA97 zD~L^bl&Q=blK?UO5~zpvTGgiE|2-VxO^Bj`O?+wH%{0jnSVOJ3kAn_vBfzLWRgGU$ zzA5^W*|uEb@-%HB5U!CotDyyfNB0G9L6({=NPmRwK2G+kzi53vc}d|4)O zlWf_|to=#q&}6O=z0(K{y@Qj35=%Zl>`^LB2E0@~H3BI_H$9og6@d0gQ%3>vI)VR| zE}5n6KHMqJU8njmC`jWCaQ)zx*vf#=V1r$?y%zRTQvL$6Vgy(Ej|CB20|viI*xznqUeaVO%w!|iJf3ucx+?bDNlo!zBnRI|8YZ zDN@tE-w_Oo1evGpCW|iQpsKKLwPd4wKJ`6SlO|JkQ8NvqE{SvOn{4+o20qCMc@|T& z=bZxO7>YTX`7FgX(f0Xgn?c7Q$Q*b~24g%wddqK!;@R>E`}no=QeuASbLrKjA}Z0^ zw8%?m*@5l;oSCtXnMktwDBA5mmWmd9z(vpKcJ9HF5L>Cc%{TpJlN6H;&*5Bz?(Y!e zOhCgt)3=Ij%(xca_LGYm7X0=4$ES}_nzfClTLlXS6QA$nl5%HK&|LK>G#!+CZK?k9 z(4K4U_S{8poUP=FhFw)X)9D$>pY<%-Kbv*;t7H-jaBD%t4h!L5Tza|l@{{A7GA%8c zwe)*{BvK=axGOBisD@Q5>wHz&skPJai5ObG0oycwGsd*SL;2~O1Z1AG%HDbNc0%63 zX>r{pQF&_-mo0taq`Q3*yWQkA++F$E|8mJ#0y|$%QfX7raB##~zjE^-;pL0A^QiiE z?v2_xBWYi9nT+Lj5-E*YPkzeL1KRV7pK9*w_KFrO-uEcNT$VPJb*k+M6o`bBil^U< zCNAw&&Pv<=eAT8RyKQPnxz+%&@{}{MP}YiKx1KnF?`%kRHFvN)i3?Cy)PBfNpMOu~ zeRb|B+IP+?C=jVQ1bTE#N2tagF9)xR&U}axhrVIow7HZXJ|Q^0S!;#OoFMQ8HRYu& zQZ_-|Fv7}ec8{XqlieqU#<@|A_73JxQ19fMc_|_6qvSQV_OBMtsD-H?hJ!%Zl=`ZT zyy2!t{#h?zk`oKs&F5E|643NIeR?RNJPfoknM`vJ@FO}M`FB6_7_@ifb;WUW!Eo4Z zYz8YK6A{7jHJBffUUfxRVGpXpLrpDU6G!JqRuUs&aqueV(oLzz4BAQcB^#Y$GsSJ& z^?1e+b#D=Es~+ptBBkqc32F}acb?hmpQyHLcbkrRP%YYgfM%_sh;|+etk1Bf)dfqMZpBZkbo5 zV}5pZw<93JRMFoGRQoHXu?wYQz{s_Yfz-jVXTePJKP)I?`9X;H`3z@ z2ef~N{u%Vny(5iDsK#G<%f5%s^Uy|(8ubA&ETZ9F1`=c4HUT_cppML4fp^!`WBc_? z0m&|{?pwhGk7?Ng0)MMe2_{S=2!V(aP6Bi_c8jsgY8wbKb$<~2u7Lok#(m^8B5|lz z++DxDi}~~#^2!V{&#&x-xeh1|IDL}5rWYz>6i&S$0kOum^dt@y)SV@pneY3H+h9;> zj2R591Yc6QF)4@QuD?J919SjLV;X<6v|SsZs_?bb&f+Hv-nbc*r?O%_(%oF` z9#V#+1uNIt^sDXph@?LkNXz?zIu81}kC6ow5kFCSGcK3xq_J>7G^y>`T_xQA*j&gC zGToP*$}q2Ui`69PSn^4LM9S&&>0`W0JbytG|FW1GdPJA9ZH#$R-#1HCH6#m8wac&& zlt^?Vp+^@yIZ~UHZbHryYn4nNxUg-Y#d6q@ROig#I;&tZ*@K{XE$dAob8|!Zc{!0f zmK7+p(N6|0g@<+nYVSIFWlh=R5%E5=e4u(Dk|&z9`|s>79K;GNqC>Mnt_-~IMy^)ahqYkP zNmMfJ1&ft~kGzg1roh;mjo}dknKkC*4QE(u`O?Gs;yk%H`o06KF^X55pXA1>w9w5u zrQ9%rrsi2kO(qa${M&$s*m$Rupe~1?jQ7uBDG7QT(iuCXoZAL2(?5waQyoJ4H-hGa zUiGi$j0HMix2zVW;mk*d*xp^UC&dzg=tBntidk^(5==PHUV1$ZG{mowdk6Vzkcm+y z_B_xUPgp71<~=|!E`Ct)eMQ1j!;*un;WVGKGMHnVV}?Ioq!D=mlV$r&UoLQlj2R|o zJZ4yItqM}xycr%7$g=0*xko#!fFtoilS=P{PcKY@3Z1hkdnPv%Evl@6Z}_X)n?lQ( zAYae@t)b6piHsTjfpGWMT`9XLG#%1xED0SZJcK0jGSc3+=I_DC$GFU}9+3^sF!MlI zkz?}NvSa@OEfZG^+D|8NJBgNhUa}2}oz#C@Soebv-d-doM zME9Sxoid0@zD9YQVLtWi58kqeuc$+3+Gy zLO23P*(lnzoKT++>6rqEU*h}sI!-HpniEqExocSpJ$H79Sn6awcY-7$P1^^07{>Hh zqyL3iHCVRD%JuyAja4~w_JRAA((&1mHF0%&SG{Jif$~+<4S<&5CnZ@ML62qB>@JvhfKQj^J(>W2FM@RHu)V(J>bkuQp zN*=pR)#F;@`0Le;{blWIXv+vafAa|=B!!H~JJr5d-zV}%UlX;$5g$c+Y3y7yI}9#~ zt=arjzuNI_=CdDqOhu=#lU_+uepZPP2NLBEdTd*5MY`Rth zEn&)74#HadjjT86Osh8jifd)R1J9n{!K%N9lyOmkG+HcIgM+}nQ=3cUAV$H@s>4JO zPnBIuPGdFdI3-~Rp8%H{|1&|l@9L`;C8@f8Y&uDxIFPv%h)p5%N%g*z@pu*wDiODJ zsGi}CryFosp}}N`(bl=(*-%F?Nljy&O~E}~5a=|PkX4@^Imxtd19pi8ccx3GWsQ0e zwtdn!Pj|C1rIMGOW1hHRI3jUiD!#w#rq)ZYQVle&xQfb=^%=P%E) zjum^-y<{_NTU(|Wm(1s(PVhP^mKPj^-YtGS1Z^QXo?10zoCTP0k+k)lm2~X0F9#aw z>7S%(pYtWDkW$>fCw`J5*$NjgO||M8aAa;Wd#}cYz*ZPHSMOz!8J;|2jNT~iR-3w_ z5i{w9V;1fzU+LYUB-$J;YU$JOni2PAIzrPKwL$vdY1Y#PQuw~g?)Z;m_J^i-2- z2HF(kg07q&Z|yn=iD-1Z#cO$RAk$nm@%J5*#))&aq_|H~DU=T*mkubetDG;{-Q?ml z`CT+9vw`P{K|fx?Y)KwLDtmAVxPqYyETm?;Ly z5HVSZRlh}VzWF&yTj%!EK*7P^L|sJ9UNDeNGGB7bTK88qbeHq`!D1i5@FjBK%#zG# ziJ8PU&I>;1u5yQ)fnX>FG#=I@nF8@9l2?{1%ZMY69FJl^u3=0r-Jm!ci5W*)~y(p_pP6m zt3?SdVYK=t6P@saZ}IytBR&nIP(hsnj*3Z4_ypvS(467SypX^-vWd8DX<*_l3tYA; zu$bN?O;1?U!0wDF#!OsGE&l&*AaXeDV+E#x$92#-qVp69YQeote&jNTROWk7@gN9QgZ-QO#?kOv4eh%+G``epJlbC%fB_{}oJ79;_al zh>M?X&iJqQc!`tsdv_59)Bg+-)Q9_o`XQc=ds~^JR)k1MNH;Q|T@PXCER9TX9~$qe z3Q!N?#ek@nV7f;;QOnIi3A#vCTF@}0m z7c;{R$}LKjTBfEKsr~?9-T8|atYnr7H#S$+-s=fW1#n=`UT)DE-_hjhT3Q05wQ7Ud16^n;3&WyTy+66z4G-mxk-d zWipvx*2uPc>-nHP)h(Xp!y;w7;)QqOP2^Tx>g8su=`U+sQXGczxDLP$IAc?$9ac&2 zt|f%e_`Lnr1FxP+STvD@$O-RzZA>gM+u>9e5|4sfu7Pm{_;ew`jkR4s@w3+c_)_hT z#3j{Lr7u+{a9chL(n4b}cHALtzgD$7%MB>BZXd3gnpWD(yl&C1in-T+qy87I$=>s% z8<(kgC>sM4DQyo{ihPg?)uk)Lh+OGP1YaC3d_V5{;+-*gGU!e5Zdk}K3kG}TS%xp4 zeTaP7Wzu9wkin&c)KiQ~?%? zB~Le6<)EB_tK693gYZznV!@HY*hg%VEE_?#+EqegJ1wVlm^x@Pwuct+yzc?w!x7|1 zT_I3Tj^*1%>Tx*K>Ga#c{FS)0k!!fIbfMYD!3>S7{B)}Tpu=kgv6+iix9u<**tA$K za`lpKrD$ER=rYKpXzc#OYMC>M$&&!tt&f&n-1@R>C~k9` zt@evl1K<|ewJ+4ns&sZyc@^3%*8pkxTI}pYo zPj4AlnSf$6OB4=jOVmJ-wY&g@UezwPCUDNWR5S?KGN9;wmGQpjr0D5oJP-SH&eh6` z=M9pYZ5+YF4hKes=@^?7*U+Wgr~GDnq5*Yy2w^te5cGvCJr12qcZAWm=gVqn-rD?!rpnhPcm>(lL!JXQCi>yml=4v2kraq~IjcH>K#Izf~cIxU~7 zpIaWyId7o~mKfD$E%hjsMF^?HmHn>IEF2+5g~fqA5$*g$7RyN#v}sCNyV_cQ61$8HI_dLBWeK6l*NMMdw1(T~ZFIePY}lz-9q1 zceA#&Z4?G@QXmSLM#luXihY9jZBP#9W!qUv#)BtV#8n!^nomF4VQi|;-KvqXaJH4H zS+YAOwO`g;NkD^po7!_;>ykr+p>&%mFPt=u+)G>s`hEeF>ws&7SP6)o(s8%ip3K8N^xbL)V-!FMIO#o z^Ejqi->bGwe`j!Gc2n67dlB>M#?fj{cV=crAo%8G$RLp|Nxn zt!npb8AVjsh1@|F&NEFLo!FG+K* zp8RZqL)PgNt*oU4cEg$}_D9yPp?o174z{_rqBkv(wUq@&TVdky<(RKMbmCLu_zrxW z2gH+fM^d5c={7w$kPdXh!Lli-y^9$fq>D$!Yzj-@S|krXiLRVHT2Bkw!|SEQN~Q}s z^CdGgmS%(eu6Hi93&%&GSjwXIeLSVyyddI=5&c7fz!n+|AM0()pmG%&!Dfqs0~6HnqUY(G zPQY-7&I#VoRThk=_*~CYp)V@^TVilqOR_!3DT}rcbM*L;=diugIQ%W+xd0@2X=tp* zN;%xD#sI}qv8LU|W4^Bg4$D{6L2Je%#Eut_ugSmIqiGFoADKB;-PXpMtyi!bsN6j8 zeP%YB!j;PmMSFqW>Rp(dJE%TqKC}|o$->Snv& z`+xi6Q$g(g#YdXSp3f-VeXjT{sQirII7zY9o6a`q!z>$;gp zuv@nrwgfa<{klfrXJzvmMDP!hTSV*xP^Azb&g$A#SLQUQ->OEF_9?FINqAmV7ebww zI$QQi!vb)gdVO=4LnUrRY1v)FrxBiSoQ_~LuPppHhtwS;Wv@K(=FAF7_fT(H8)J45 z9a;(i{ho@s{Do0j1EdI_gmn+Aw2CKG7#vq*o8tnhxG%m6DXXo@D_D8+sa&PD;pGj6 zt*^8w$tNQ*_GYuW@GXyw<=ei$b|%M)Vp$wDT!L$Pvd_>n;8& z-?lOlc+QE}F8VmL8wt*0ex(GsiAg+}soTDcv#lwW_k2HX2tqA${Jdq*EWU{$ofV-$ zG4C`vNb`OFz+0_!b<1Lq_Y*fJ>~=n;b4^bIDX*B_~vLRLW8&Z(?Y*l+6|EeA0acTw}xD*wWSOrww#+Xp^}2-K(BZaCxD@}E5Z7rfFU0b4RrUsXo>*9Qcep#VVs@nal*t?*vnHqX?Byn8OF+Zv zEYHJRlc%y+UCm{lW%rr)J=|k_xR0R@)$Zi9i_=!+ocz;*k`R7oslQhC6kBu}t*aVn z@5xGmtmWbvgfXR}sj>-xwa#c&xIU8liy@pU5_*XrGwDpE^7g1pEVPUoH?SO8G#t{- zBYvjMo~-y6i7}8Z!K3SIQRQOhyzdDX!DWKDGhiHgL6S9uc0dxfp?L&ZxSK=Ya-;iw zRgA*02XJh*JG~FDa+iDXh!Wj5C}Sm!;LzQKYYaIl;PI7k?Iq17U5W$63{Nwk*o`Co zhiFN9d3rR66J!5CQ%%kly$pI4gc9!9w7|lo@e=C7&Ab!rO&g;Fsq9fgE{rnK$CkOQ z9brMcP*g!OjbZCNq<(K01DLw7t14v2?&=u!!)j>rrF7OsA9MEp8FHd-P?85LJHha} zHK&>Jo?*_+-~@HSPZ>O#M820$^kl{|tqty@TwGgDC!bbyUFDA@<_iB^OPxeDM(k3q z?mJvA`em3QXyG9yD5vxB7}h^0DpaytBiMTYje`0m**jc7j3yhy9XCe@?*Zi3-b#Xp z>t6L0qGz-r0##p<70qmY9H_{BzM3JH%ifM5J;GDKWAKu7_2A<=CUywN??Ys)Jl`U< zQ*3}JJ}udbmoWli(Qzf+JKYw>)t8a^@4sDv#W>a2rzsLdlV!g=RgQ67jcF#~ZFG^vxigIW$8t-EJZV-&}mmgjnQpAHJnXJAq$x*G1wnWMN zQNjhTc_Xm!!5f0dUZohQGGbxKDfSE^*wSiW=0dU3$;IJRp!?+3vX#Xiev1go^V)1l z4>lnx6D40IJFD6AIvBTwl{pFoO!UgjX?OAoSTt;3oIi*JR!myeYw2>PkP_Yb$E*IU zZLmmN8%nUVd~1%>D?@@|vIi;mD^P>(pZoUNx7;_9{g$*o9V~B%dbvl}A+EsrLaY@fJZXJed9IhUx8k#|Uu= zEr4mlqT~=ebvO9jDZ9COr$3!kUfqcU=!3FBm^4{JsJ`my~?-G-grKj|SDZ~v> zlJLPkch)FuE|aO71w7`gUQHkG#z({z=NxLd%-Li$0)U)m*(D`T9Y@u_3+Gwt{JDk z?2#;tlDk${UoalXOj>lSxyS=#8@tEvqg=-s`t)UBMt-QoVTSkU!P9FOFCg;U$RWke z1CU3XFZ%7hbNZEgI33H4yDwD!ofF-@V5khHInBium784-20YEiD79xviKWufr%r(3 zJ_qXQZ|FvOJoLSkm7s2Qf@qifW?n$5DS1BG83g@dF}@0EG0q$tK}5Sg+$iUi<`+w! zB*iDb=cj_wBEwhWV^+zihc6x_=ulW|I>9XE-hxT+@{0{W&C4@p2y#QS*ruZ^WfVnD zuiX=z@KA`J+gj|+3c_3q)7(KM8-hzHtL-ScF*Y+?%^~{&SywDVT2d+GBckV1v7PyT zPNBHJb(h6VaCQcKBNqLG6l@c+GL~0uY2H}la7ZZ0Adk@?d!T2#H8JG{(%;_=N7X_y zm!ylqoixE~vlvvFPR@UAg~W}}ramnA$NsJ1>9rVLNu5FqnBYxC@j^T$x;>v8Ar~f; zly(XsJSZ@cSnJVwkRY%!HF_;}tLnoB{Ui|G=`T3DgW*kI(ONO?V$2!g-38|asFUFI z^{v8S>bpHrHs3l!@zw=wor zoX=>D`B=UA?sGsDg8((S^Dial(k8!3dGLsG%QhKHthi?-!m2jsfb_{l)eZ2~J=|o; zw@v^LcT~Aiutyuv=0=J5>v^9?c~yVQHn`PZfR;rqsce_1>~4j=>%Dn2IC^787BS}D z_z-K9*qjK(fL-d~PJS1u<7m?5AD?%U2G3TcxXBmA7}Q=$v)g^pjmoUEjj<9EhXi+? z646-__3G7f|2^Cbe_AofeGSLLyINZGzOzVS-Is)OhHRrFTrMc=GcJ%)N`}YD<;)PM zIiMxt)&BxjK&rocwuNT{bGm8BZa^7VitM>h7{zo{8=q0uLR+bxeZTFzJ*lJr96Ts? z@)muBYH{7};0SsoHn|7Tg$UZi{vObu*0$I*{w=bQ+Ng;owVcc$oM|DN*5k}l+=Jxws!{Ktc`PnAK8i+6l3A_>&L)c)Q{XEup&^} z`~(_~0_84Q{UwWuT#~)Bsc@)1u$rV9T`bCZn=tY_>13N>X-d zgy1T(h4r*0zk_;>Sv+-Ty=S~)A8nT#FJS_uI(Y4_eUN{K%pxKWE@*pYsq#Hmv&vCU ztMc$J!0RIO66 z(NV3AE%~)p$w05-;bru^ll}R}Ea_(%f5xX7e$<`oWv0E8>cx0wEzm0sH+1;PQks-+3ieOb<5CJXC{iBtQlT-Dsyt$%*mHUyz zN;GApNExrkt5lbCVNr5FsHzz%Mdnh);$^WK4i2&Pmb?(Md;eiVE27U?>}8_S{$ttdWHg*iPWLugdGyeTEBo=X>(o`qqo~D z50$983({P>OY1V8BVXrtHOh$+cAV<#F?FyrE%Du;#Mx?DB2f0RO66>ve^`ORhk9sN zj2HoYk~x3P$JtneW1+pfV?n#eXJMJAEce-?qK7e&u`?c)SAU87Y`Ilg&l#G1y} zRdvyp1Yi56flWF;^6js}H;j{VJ54Cb0m6Iby#B8e=lsp8KK$$jwi#{UF3#O_HP`X8 zOHnjr42Zr%a_$tz%WdGA9Bk^ODj7i```UY`6bcZ3S(%-TeLwX6Tm?Ex9R<6q@J8do`(3=v8^c~_qi0udxI819^vrz;2rmkUz0$_ zUgo_}n|)Rm4Xsr=dleAh<>S`b=Qqsti4n3!iLMVnI9ul%xZSrIBp8i0zso$JmV2l{ zi`Jc0yH2?(1QouU3vZ4I_C;Vm)_Yk}B>n9n ze`!d__wAm*0#RF6tW$F`My|2o#pkAu1vXd1P+Z~mmM9(AYz~2w0KF^2*B>2`G2!3F z(j8i>@P@7g`UJs2X=G1I9AI23cK-xRIL!$uLUw^TlJDo=lSWUq{Cwg!AMs5AANq$P1jR-fWpf5*V{)kB_0?QgHz>v9OMQ zcL-ftCl8z-OX&G?KSJdCaUZ6!qJXxy0}_YOnRTLVs)dy?bpVwo-W_~8Gw~KVrfQt zBf@|~!x7Z5*W$hTBy8wVdmgIW0Ilrpb7`>Z9lWdk8OLvyPg*7!e8-h;YtLECc02rT zjqt;Q9j$h~)$!g^NtIst(9)50xK0H^Rd_EXn6%osW=aUyg1zu4`0 z8?(Nlr;OhHD@FeifEHt#l(kTJOY{cm^C}ILC zQ6^>LM!* zK)$XMAdWoR5<$WKwvIec9Dic-X^8N*dm+O_K+bnhyB&cdEFh`EzlH=F4WQ+jW<|2!y ztRT|1*f_wyjMieiqz?AvTQK%NdqhIVp0j;4w3eM^u*wB2KAq1-6~!)l{5es^l+2?m zW;~?LQ!Rf`SoDCs2dw#{HKR?p&OW4jMv!_zP2IZng}O#d6`4Hw`6n5D$C%?cvnnjl zD|?M`&;Er=xjM=(ll*L$3CgYB@6!C1fgyt;bscgcSE_SekrB8HA_Naz+ySjyQM{iq zEkT@0h+JnIl4#tG&Wm2-zcP{W2onxSo!fm>neO1l{PuDQDm5CU_S4$}e}9>_&_@~< z%@xEj&$K4>YQlD|PU8nac}Mkxw@{|caL5d{yUT73gT5sCK9{9dtQywAa8GZlAXZi_ zp8>xHQw8Mv$DPlJb5ql@mfFalUfX!DzP~o!#8KLXBAhoum%2PlC}~>a41xISb6xssI&{odGQiME-fM>51k$yoXp!ju%7 z%VIn*%KAvFi2AmgjT7laTqogZ;Iv&#XLw7WR!OC>56g?MNhm9o+B5somLp=!EKp4iPAiK-sbbZ4QkRuKvcrJ~MbC6Wq zKQ3@;0XqI*H7uXPQyfI16OmE;<+7-y_K0oJ-$1W7(xIcnsb0m%X_D!wxan`6$*_Fs z`KAr&OOhSCC|ON7ewhtKWGGNnyNWni zHaq6@*?5`dwV!b<<43^{p-ot^?v3|OE$^DUf64U#r=$SdR)eNkD;UT+;u<|iyZP}d!Zt1Uv~s)d{TFy1RC6^mNLQ2 z{dL$(=z^E!3I(5zKOztRFtmD%uU9pNAK^ys9KUZfgQqgR^<2!1SUpCuOxkMmGvNCT zp=?Y>u54W*ijjpc%{Ou-gX)plc_k&Uk`(OECuZgi6RJ!4G3STu1&=IswNq(OyjnDo z@(AX=X%j5t3m8e8meGCCkEz&2YXv{8INP|D`gg`cH9^z7KH4>`S$foNPO@7D9AzUr z5#ak0%~=1|K2|vD-5_lO6H`qC~7@_u_G}%+tMb8Bx6L_B@mNoxk!Xn9R zZ8a7gZHwmn!=t?4={xNjIRJ0%Aj{K}0JBoFyO#)V?0AHnu1zhOo)!(wOx@t1yO@#2 zb_=w7I{115`mZK|eehHnmc88SC8G5BO7Ni7=;a`Pmd_pz^2#xqF9*=T@dV3*`=MV= z#>IwdG53GkT+wJOlGF5zZJO_*(Bq2p*#_=a0Lb$Q!LgxEhtL{TK< zzh8SuVNLaR^0N)`nJiW?zUf9S$o|a&XwBR!?-#~CXq;SPf8MSQmmqrsO8dU|-7e!7 z{ZL%p9^qCFS*vfl!ZU+~AbiZr4;d_|XkXR1MoxLWiw<#K4YCxX% z=%yh3j3hpb)Dd4?+;VjqB_?|}#YoY(<$#W7R(TYF->$nfE`Z6$Qt;E`YPSi=-%ZaA zyVexE3SLXcjTCM#`umdE^kzLovJH2Vo;3D%@zmz(cp%Z}qU6VB#RZ!q*rqjeKcWb* zb-vU1M_2wC%LO3QR=Ca9(GBsuV)1j&D-$NnioMtrdNIk~9{T71D)1VhlHmgiJb4UG z77*JIEW+M@`G($YzJ#FL)$l=CEpMyTI-$Dc#)FXAaS*6 z(c8n^vy;n+C4BiHDex3-xyiN>FCLU;=dbws%9;-eE^$+IH*6yOA#KcB*%&cmSH{rX zY>WJDE`h?e?Vd;KjdO=pXu6@4aO!>Y+993_G?owwM({yX=g;(K@{OK2EO>8(hH!~^ zMn}Y&NPc{*npZU*+Q;{ElO<}#gZ-^_;N7!48Bx*vOh(qOVInW%8n<3pE@LU7uREpg z>17yCh_h?Chl?uV(g>I!@sP*N#TFbBqvrm1^fz*exq$k7_}FKL&u~n}ohzjr&uh{5 zDfqV1fNSMvh8w4j%9_<>q~&=h?4v~>h40NpG}`l7^Ccr7haSba9K!O~vS6(dhLGna z%s-8Ig^$d(onRECEh}8th#H|wIxsJ1_3m*wT=e$7ih?NEMkBRucx|I7q4|SVV%xT* z>yIsUhe!nSvA^#;)G!Y;pdhN%)|;hH#vl+n;TrN^1WgR}h~-B^F1KV@fjRTXJp_iv zm$-7Pq3DbEPhl96rrn5y->r-OE;Pvmw(9u6!!Ay$UkZe})aj}wpxp9Th&OC)5?Yr} z9sUpD=+qPvH;)Ge2?x*DTVxw4B($7A^~tvbehbeVbg*3NbC;S-ToDi-U?%KG#G7q9 z?)5adaXpBR9uqg6*7|)qv|%>>;1Ov==a@LbCR*-*AVx!UiAa`1D&0e|7%N%#%%x#* z(S5o=ZXm))8KP0^rw(t`5PT)Ga6JtDe;?6&3_4wrD61P* z&|V1D^1|u#AL?SLi8Ys<*^S4A>*q_r1sYxO>{f~D1ZVCKt?dOZZkn#OHaTn`&hdQg zAV3`XlcLD9IJW-#`TSJiWO3QAN^az2)jLk;PeuMpn_tMgj_(aSE)wF2MBMHK;NNMF z$>lDi6kDbfk)DI1yfzgvs?Dzc9R9d6lt|6Lo;=ngFhNh{SDQ*Qtt1Ad@N@(LW>91; z#bSuQr*p^9?6aIoV;>N3abOS0;%x87D7xcRp{>_5TzzukBKYefN=pCK$ZhIA@X2ti z9T8=bx*TP|U#kni3;TSt)m3aTi6CbNXAfCG;J08%Xx47sca!Wlsz*%ST9mxfX+jp_ zAlK*fc5Qh8vunbd!3~P&0v5^nz;KpK`PoK2-_(J_D-x_scm`v`4g`KaI?Adije!~j zjlYpQ>%Gzd2fG?&hkoHa<6lD7{RcKOwZ=0*pY!nHE0mKhaimE2t&;8yg4V=Mhc5&b zIhgm>==DckJHy+7R3B2#nL5=4y-2T_0fd{HBLRi;9-ojUR^lXf^>p%&;uT-;oi*mA zqiqt~{e}>Er1(G?=z1j3@BP65tyad291^2&a1!VqtR zgEVCGe4r8OG2SCVq3(Q)&&zj#H158V)**LUlg=Rz#)$(u>7`Swyz5zlb*e#9x1D6; zIwUc2SJsm&<-TzgIuI|gU(YJ=o;ms1a>p zluj=~a?Pc15$c&CljFAErH3A~{TM$jo+&}7u`i(-?(SvMPmTU5bgV+zUk7_n9PUp@ zPu;7mA6eWwP-BgRBf{DeJix}Y4lJ=gUq(hIsZ2)G8>DR1*3{2jZR|{5C|0X(0WxP| zW@q8B`Ajtn}TM_#s1u|TQgo#>)=xjtCvPhD7R>P+erA(Fnff9N1! zntDAnxN93^GozVMQX|qS0@t{E1e_uY&V%Q()MqPj_Sm4_MWZT(>5MN$hj{9?4FRDb zC~4WvWMNaOlE+^K!_phCM>mrTF~IbF9997F73M6MJM+XfK6)9vwM z>|-@}mMdVmYXDY?hApmkV%`W47r4prUC}-=NS^oBw05;IroU3`-1R0Z1t^A zh$@ihMy+0mb=3=7puzMt`iYFPKijkMbm45{w{Fy4U!*iK`0E17!HJqQL(SGUq`I;C zM|temDcsT}{Gy<2DdrDm66*)6mu2Yc%2pCTUTfcb5R>6~!5TjxNLe}!x|)1dMQN|N zmyWAQs{8m$H#stlXkFE>`$s;`iqa|!PuYB`lky<-Pkv=eC{N5t_|SpB>g#nT3T&M0 zQT-yF=1TB>Ng;(IfOZ%c%k>(C1LJvYBg0)>dr=V%iOzKcytgk7Ih7D(?G~J@r6}Va z^SLog>uV2ij)K5ZT6i~4C7^9zv!wNtqG0G`9N2(b%K@V}@CTFR;#B<_jabiw+L1@!jv{?ecE>*<_2WdqA3K+Ffcd;q<}T$YGMqHANLh)zbb*lbT`nwhSe zpv1d0t_ln90n<6qCYSUhqSFgVCj)e@#rhBvzd~FkDUK-0i4;;;KnQ+jCWfAC8Z;sL zc+AU@3kHbF78|Hxzf_31wfvUogk+}Fm8w3eziTWi;?ry-4<+Fma8kIw4ye^FlWrwY z6v&zoxdc5^?bveICc(Ayx^GZMNv$&lzJOiy_oI`Zt*5DxVI!Mqlq=sn%0D_9Hxr2& zK8v4SlUhZ$Pd->*a*4o-X`^en**eNU!RdSfPCkP{I;v1X-HIL2P5BWf88R@ytv}vu znD>s|r|uE0v{-9J)4NVq9H3O3Y!i%3Cg{9w{iG%U?lZ!p%hb=C5-)%krDWE3s8tpN z@uh33OA+eLD28yMMz>74&}`f@$ID<^^zlyC#?KoImGPW*wMYGBVOo?EnEzNcU$^m#uH`JgITBhbl>1R%57h`F@!${qd*xNl|vik-k(okj? z^Y#(DRD0cE&no|`L2VSrv$gUd{EkNw(J5Lp7tr~O}3W5$&|KxND5H2$7F zl-{f=OjEo)1bmahRn7eXsvm3pYT``x1YQU8H|DYst1-!AAHQ5}BN{rw>@EyHozk`#5Tg1argB@J8?{V6=pgV~NqwD2>)Jcd@r>~g&IjLrn5($` zf7-v!N5qM>xUko_D;HQd{)gPBnQ_sh8UP2m!U0N0-mcNK078`!J&Dqkcq7U1ZXx%B z)76nO%B*5VMtpHVg}t=kB3!)%o5iSk0zQkC%J4Jw1d(-Wo_cBfemvH4Fy;FBK@6fKy>4Av0QfJPidhEf%m)G6 z(S(NT2DIjDghjnscibQZN{*J&5!+Pn1{Z|KyE8phf=b zP?}0^ZQN04C;!^&P!gkXi(YyW|L@ z9?(cVpM1vgrxgP0c?%=mgq>)N2I=@p#Sx)1*F6{6wr1= zj7c?+iPuDN!U=oi_N#gsP(lj`s@c4hnY}JE*TwgjvCk}6RG2%nO%$$?#h#VuIk49r zOj%n7k-KhTVE~IO7E>}A(3eg0$vfA@ECw&ZSTu8uFQiI+JDVdB!wmZ@k{YY+F9VxY za)?|2GbuI5+7ZMPxQj{_t!zD3YmMpxV^OiUDR3M55M}&pI;HrCjHRF9B6Q#754B)l zBay6CQ9lUx!EwTn-~}oRi#>M!DrXkS%}*I%(?ZJSMZfp11IaQtDBeC*GeVKVH|7D# zNLVe3N%m4|UR1GEjgm&YlLSWYAJ@#^0Uxj3k#V*6j*g0hBA1_O=yV9u677iU(FJHe ziWRO6lY>01s*YYYt&ArJ*l@Ce*__UwaI4o%33Uk}m}8W@X)iG4&aQaZG;B545ZM3b z-IA4F7{yj;9Zzm2&Df?qqIY^2LHc2>U2s?{{4o@Dk`k%&`~qggh!x`-_b&MJ8D7ZH zoYJiu?E%^m-Zfnn6zZQNovf`&T9>peC$uuwfCblA_)96_+ zuWuIJ`Yfy1@h^W+K$@ZT!0kRuXumqkzNU+<;8wq0pD>Aps{T+d|?HmW)+IQG(rAtC-Jwd{2k_XbpGl;l_Fq^+j3#csm zUrnsvO=zklpmn_@6vMRs_*U%A*IEr3GC=qgGf*Jqb3tfr?t%`NEC8Uhf9f>^sZr*w ziVRNLhO<=f&A?@DxBR>wn91Xoz^_o2id9VE25;nB8-^r|{W~}Q)5gdx2w+gG&1%mW z%K8rWs3k&n_c3536`f(?@*^FyJ`f&4H;`S+fTVozQ9T^k5?Dg;TE7TtEoTTy90Ct$ zg53>OOm|?LB$x${Ar#)#;-(L_6fZjc2v9-B!YBADL_fbV3J43E#kb8-?DoACQcc|7 z#(NhbSw}x#bW={Otg_ZWG@N50QC9ZS)z)7!BiVlX$|Xo>CTe@F2-0w}B>}dLpJakU z0pwAE9cUVNp!-C|5=-Fe!p4kasI`uIxqm%Cgt{K+mR(36MA!<^BQ2na(U>!t^DHQo z(}%TXH@Tx;Q*-DFc1{DmFAfGtVx zlg$M>v2kNd-R<8IL#nh3fuwSt0`@`92A}V}(>Tlm+Ka;>`zHpt#lo~IN zUP#w)ld|3#v2thOs%`axvg4vUBE#%ZAR4pi!5!D3qRc%;`!Y|TI7grNiG z2(eqOhoNECW9M(UtU+qmqS>@a7OGNI^JE!A$CZYZpcc|E%^npse;q^>nsRFP+GCU^ zzdgQ@cOCh9nGw*@r>zH#C7dS z&oWwhD@73;lIJrg?#jk_mSP6r<6XD3^z>Ii%P(V@CXpy|&?999jsc$5poim%_eWuw zsk6kl>6FbJjQ8Nx^BqNe3ZigA^9+Xproef|(m!jn2c>a<@^`qr0^&W4;v->3H!>&O z$V0A}zvh;?mfJ{E=`V~v-Wm;Pt#L9;)S^7AXdZrwCPZ~_Lc+(6kodD1@v<_YEYX3; zLc3hlthA5Skra-R+ipk<0#)Nju!xEmmb@immp@#Aw{?v^tEm8aN;9W;`3ZXfYEQ)k z+e;|RkAUFgR;lI3(-7D$K4vVOgcqHR=`dQ>M^-;PkQ`I?f;L29eb0} zF-I%w?HQt*3vzhPYWft?PM&0{o6`Gb0EqSz+C#3$4AvK=g|wVqnyIvYj*`48qp7edT1@KboP6fw~7CKJr)G zhXgr+%kYr1VWIHZeu%N#U_4ysTGRPMD<+&oqmAFfejy)PJv&0vjs>U1Upa`;+Lgmo zHdj!#DSlpVeJ!@u1}7^wiPPT=iXG&Ir(#{{&@Ge9?zk?*wVW#SN>2PEmD_IgLO@V& zRuuQ?b##%YQ$Q4Aru#C zuZ%ZWJF(k3btq$5k1uLl(c#+>p0Z`$sy(3OsL+|3ji9Snkv-{3tNF$MuL1bA*Tw$w zkLq6h!QIZ#Q7I1&)9LCFx-vVRH<$;Il}VRrj~BwbIEzzPLdruTMl-?wt2;CoL3Xg- zGRDrWQZbxc&`igDm0A!d|FNOgwX?q5`8>XHTm5~myevlj$?59a<>~K|icQ7tqG6lB zIP)4;cXsri6SNS>IH=pT;;qxYL4Y9LDs2dXKvROXvtA;-b<`IOwnY4UUb;jm00;qG zg3R`f;P4~h_Z3)~FdFai&c5XO1KnNN$bB<}+>42cttMDj{WY(io&xxqadOb)?`bx)pqumoSRL0*2Y+ z?~q>bhSyTgrZgF20R1Y_?5GL>%v3!D?~nEgN^wDF?azdDr+D5Z&|g5dq)V zAnh1;K~$V_Mg6eMSIzF^2*_!vc^*s`#TirnoM(wz!X`J|WLZL{|nYsSg- z21ZOQ%AEQYz~@>Y?P2cb7f8@S$_$A>b$ev4cGDG#aLV6^MIT_9PgwGUJ_*3dxQ%RB z_Las1xRU@t_$X^(uYv#bmaqfu^jMKJekVs62Hww%N>U($TJCZO%D+@+5+UMyB)6>T zWv`W!kD_&k6^{tUF78_W&51Crc-dHw^U%BO8G!etdnrvh6F3FO27;xQ77~LVj)!cz zVyC4AP0iMnYMp(}VN_ZP?*SRI2WZ!xLf=lvCqzN#x=g_wekU1fwna)9il8*O5@VTja9?ac6yedOYd)vH z^gB#f0t*i2L$Zj)5Z0l|0iw7CQL1UI2o;+prri-blhVh2J0cYBc{dql;J3&O=iMTP zSNV~uNbvFWrm8!&F}%ctBbif2y3zHads9$TQP6r8ng}b28*J~j5wB&KNysKa|LNOt zmrTV=%W42T>B*3m2a$I>C3axgx!Bb<(dbO_QlGhI*I-m?Ads=u_7#^-T^5Xpti?O0 zwZ+7z7D7t#=CQol<&+w+>mk@>?;_SjzOL_y$Ru9v5{;QoSI?+0X#wAB}-e*@Ype!v@8Bw4JE{GW6sf5_&f8rBPl_Le`Or?v>F?#l}a_$ zvH*#G=*KebP`Hc$69h~kFVkz4+0eONn2~iFE5UW)6(yq91=1bj?-Y|ni4A}vD~IQ6 zlkk2jvbyiIJ;ENQ$=O*+hVvSoITk_;!8o+U+~HU2-t4u*i~9I@R+vmOgjzH27(o09 zJ*7HZBp)>E9P1^=EeusR`@0)0Hw_h@WU07e;LTdcGAmxc9lcybLmrU>!>S3Bw0(vt}-W#4+MO_(u z$eEC?$w$ZHNw1d!MF#CaGLkg2I&TBoCQEr9BR7b7FCK;ZS?6Tv0(ww z7aR!)426iRJ+rB8juY%KxC=XiaTM{Vg)~_ z^hW35y^bJQ9^l!gf{V+z}; z{)}|0F5{Ja)Q%i};VQ@v=`AN`e`}`X#5l09x&0hDT5KL|v|l8rO!i>%_1LHF=nA!T zuCIdKFv)FpuuVV~6V^Kr0!&(^wjFOq|4-jcVja6`%-BEIQNq}_8U2!psLPlUIsgyl zD^Ph{^6sZs+f3W^Z-WN>hcqpnKx#pe1umEG5}w?nu^rSw1$3(SJx7~3)6k&<+FA7+ zNEa92aH%LC3N<*Cs$lA5^a)tY>Q*>~=a(vTKCwvuo{lMcZqQ^Gj@*i`H__=ndN>HG zV+Gd?Rpr)@u9LTaVy+9xg5yuHBnHX+96=~Z4Gj0hTPG54ueMFc{lW~L`-^Qc&RSoO zl6_KJVS&<1MHIlXTWXFhOI?1VZ?J4wgBvOWDlstk3A zGNO6fn!wl%jFj6CcSIoJ{%KdKpq9>KmOsKVi&0tcvc(g2`ZiBx&*+}Lj^Ws!qn63|Zx>LyM zyR?%6N7Yy=ov2*tp&fzw23yj{VH{%w5D6#X!vTj4!aeMRUo$` zO(M%mWrTt?^ISuvH3@^Dbi|2GQHj0t%;|-`M{KOCRwPneml69H5x<+^AD9fyagCw) z93)Um4m1@~PWNW^=iTmE$V3(A=s z=I(mWTv5$~toUIhO2~_r*~k`}+|aA+@(~BC%CA1!c0Fb*Y}uiGp-&%wx+^3L{5#7f zA@u#s@=cEq1+!IwFq4-?bd2`F&89~o#?tu!X=iciM0HA7RMeYFZLF2032*6EvITb; z8yW%0BS5TSZo76@CaZTjy z9Ec(c#O#SH=T=Sb{_gkKE`t4u0J)~}5z%0^|3-b&809QADAJ+tCo|Z1T|7q5Ts_rN z&d2;%kf0DQ2?U9>gRHf@j4=$-pk$idIDZZ;j0EGfJX&Vq1x>sjFo=ZxOR6g@<&sYT zM&E2u*0c?8BISi!k&v&I16(i1xA2s_X7iK5m{s}9F=bA8eF#+W_}e!mYy9e~^UfsL z_`#<;V)dcUl|WQgGw!v=PlGJm3wM0E_}J!O*iEKkO9{GRy+ur>9ceByz*2pu9X9Lf zZAC}UZdrZ&)t@OCKOD4NyMWPgbJEf*qXxPbzu6W`;S%G7YcDTDufH2hayCOAj1+ew zAU`6FN~6AYynuGINYo75E(D^eLFkjvi6b=V1Woc>j-cGQ+CXpW$71BvX=s!fFD|NB z@4iFz*@w^j`qQzs#x7p~O{6#}<@)EKj%7JP*^=Z(X-#6|4M*DJSh%bW`LL>^<=GCY zZYp?Luk#2La9DyDN(DTr=5nCnGPB|8+B7*RE|lm}-%;}YoIS-haiD5AjCBU)X$YA~ z2}nxj)TgJT;lW`vHj#Qh#Ph$nt)Off_#oD>3GR1M$Z%Ky+4+Zd+Nc5*nuKG?Fuve1 z9O_L5(Av}7*Ga61F#I=|2Y{oMjR1gO9F+7>9B)Jh7+y<>wLfXre?h}A-L*RN9^IkA zmTSvCWRh}m`!bu`G#6{}S}yVB$WbS7poaPtuS{W0!_a`i>~JrY+2R!f=wJ0uIIzJb z$es_Ouy^TUgg{8 z?H$dat6hFQq^2^;PYv<#yH_Qd&?iJtJ&`jgOp?6c1RrOg!DkK$=Kb?-fZI+us zT!bsv#n!4#4o4vDg>eb!M8scaV&ecU{zOy=5&Cg;fo95Gn}2x9JNT;GMdls`!@ExvotFe>#TjMHt2B@w)9!Rfe|U_WsAER< zc0#DaJMQ|mDhiFx)yGi|Jv$zRczNn{_tuqWsshP%km4U%R?&%rOnu&(K920}e9xjh%5QjWZ{_iH+>H z326Z*PW==!raTPrN|AO7o=({H#1d;nNwa#?Y^jr|lX%8nL|pLJ$PiY{8Bw;M@5zzu z$%jE+C(&;g9oEQOxUjIW&Ns&9cF~a4x-11RbInBj;LnJ$D2>B3={#}4WEH{b)Jz&^ zADPv^{+i?~q`lMHS16K0O7%u5EOtH$BtnVmWdmz8heABzXMsf% z+ZYUJ zlZl9liqF7#mJddb4^j?*Z`0MyDjgc6Fr~%9hNh!{`mxP3#hJTjUkNm<70;*s1KiA9 z{*dQH@;_lD>|DNATXs?<R?j2zqevx*QaQ+HeCs+oqTEUFQ^P(sl@q@Kx_S=yzYA*3SOv&xjyf+^V`|w^ix} zN;9jtZIpd1s-9FCL!jV<`GVNWVZhr1?NBm;G(uKo#%wcc+OH(bWQ@K?f{IFdG>H{T z3Z9+5`DMS_z9mJ*feby12TX>jIsOWuk`$eiJYP;*@Y>p*CmMCfk3luhj(WP}0YBWk zqq>n{B3&kaX8?Ei6tKf0D@=fD7fv{!LIqe}ANalpD1NqM9r;Tjr=hc99G!s$nf!~8 z&Rj~7Q*PI;FL7J{!MbSciW~bCCIs*~!3F~Ochpm$SBR>!p0KPoXJ{G5B}_O*t}@wl zEgrq|XPEbL1W0_%HX`wKuc>{)G3kCPlFn=J5)F!9(ab`Je}-I{CK+p@!_M&=h}hx< z7Hllq3WQxUq@$!zg`ZmkV7w8y-nff$R?u7!@Im!?r3-?xIP!$kD1Ld5&f5dsW_%E) zLqk4TE?2xoEvW>qTqVV;GR$~OjKI#yweh?LNIY0t^hUziZe0;=DnlcWt8s-c56fNa zze}dt8ZRNj;v6_={+3cx=7ojQ(X2t5V`XYwQy`xz+VEO5-|cAMAg%2GIRXoN zw+KAz5B&pYiZze0I(8W8DN$iWj*Pb2kpPBY4!!W_WlNzX)ZBX_BbShpKz3>lhqh?X zWgXG{ng~qXl`nN#Y$zjClTLNT1=@XZTB$rVafB1%q z*gPBsjRSCG4yp>s42pWUyNC-!TNM7MJSKlZ-P!3Dq%G6tRvfpgDAY7M2E4i-5e5z% zJt-Z#l2K!`-d6nqlQHvH37YWK_6%To5 zjPBp44ldW9AZLSU3;g+!B+xf4;9Ooo)?JtW05e!sj_m;MQQst3pPeko^?m*_bhQfDyL-wE7CqK_l*y z!zJD+8m9Zp|BY;+Uf3@pUq4A9YxmBm-h{obmM?3?TFN}8U`CRLG6jS0TB9cp3az(b z=_x+{Xn~}V-d^JjcL*|qbyxO$jl1%-)y*m)h=$6JW@GM9_ZXR*Pk&hidelMIB7_ zK@e4H#J1B-g5TPodEMB0U$a0P!TQCtt7EO@{^cpzRwLBE{M;xnEXoNOc*m^@>;^YB zR&M95hMAfdQU>0+x3Y)LU%_s5c5_8g7nEB!$!{l- zmevf*rAMTdYd`K=qn*b(i_wGfptBO{CMR@-?M&DFZ*2qF-h9PA<`03P$+lP7pyBl& zy%)Yz;KLULEV8niJ;(KiU~>FYb$hgSU*9hkN(-jB-Nn_&Sm|<%$_sLZCt@o6nqGpk zQcR&PVB3S{KRimm5~ZQ4@z^_|4Z{t3Q;}Wgtz8HnHxO$*a z_7wFNM*a^^;pg_pbAj+5di+(yOL_L*7zA%LYcc|$OHtg()s*6o&~IH}19-iWtwae0l)o44b$OFDaE-!uzf9#%se0!V zK%N>lCDv4@H=^(vnt)xf=jA_MEOd@V3lptgp=UyuewKs$7y6r5{J|~*uGv)V*o@o4 zB97)eT5-t6r(Z3S?iNbxDYwPr>w>Vospf+4L~Acmf5}>h?v;&aTULwSb~Efmc@4J= z7DEESV%B4V<=uWWs>w~x&bB`ps@NJUE*&YT>G*$qZFN4h^A``scjt034UNV*ZHFJQz{i$~Hv2o$CR5#7~ft1Kl#A45aMZ%E-)$Oe=xS6U*DS z0zqeq9L9m&X>PCdpmQdebKi`nz0MDZbZUYYKcn5Mm9<_O0&QnZ1{7c@0UmvURUx`! zsnnxlJLw>VOuAR+4y%fRL6H|LwZ5vYX5mcWfw7=OFa_Ft(96m6L(d%dzK_!LQe@{J z30v=~NNpcSc@8Qb?WED4$jG0<*xrbK9_{{Z6+iyZKKa;t$h@q69fm4=Tb~a8PplhY zWBAh+J;x5Y=P&Hz(448;SaW22q7>##5Re7{mKtVslHkaslj_84mdd@kP`gEMT+5tX z%~@~yy=fXUI@wHt&RB#5^htq%fT$`AT;MOkBRR5Lb7KXE8(u_YkatJ+HL;tSlheO| zMGj4mco%IbyJ3v&HCRE5dy8I99!c7U9|MDYDJop{dN_q4R(Nof=6Y2lS!in%0CPZ$ zzs>FsY`OV0Urf>PC4HPZ>@d$_02#VGX3AnkX=l{yiELgVuZ|f+J6hz2X6l@=x+8vU zWdTqLYzTE*M3>RqeIpjJ?#4HYS~}&#>9EW!cn#LN=9DR^^J`eB*@Rd7S--F`?y;-M z9&~LfngC(FYf@Vb+>aZ2qKNlP!5m*1V|OeEflv?%Vlr!&87JDa{6{U`%nuR1#afe$ z69wjdSAKyM+)g`sc+Y{h=LHo4qo>lseG)oH;SK<;G)z5Lnn9|F4O1@^F@w}zaqe^k z2byEvC>nfxxKB81QcGwN=y!f_#mnsSolQG}sK2fBc4DH2XM0hN(@*>eq`N-*n}jC# z({J$-aHX?XwV5<1vH=*^50y0D1@LmG!xUO$?BOH@SJ)y_OwRdvp*) zPY=uie6+N<{aRD1>+0G@i+h2xddJPu=zZ9h62qTzIRWjMVj88^w&HZ?gQ$Kn-gn=l zyD2bMt6;XHUC2=2ViXIhaLJ}Dg4qp?b7*pmQ54&B_T`VE1biOMp8f8W1Q4B!yb#0kt2Bc2s&bR5ZM4B|?S zuowmsCz}#r&M=Ar?}V<_^PQM;=qAspUSF7GPQVM#gzL)z7mK+kD@^29BWN-5jo}Ed z?l!JJIbNL*p;ti6ibk)8+KIHVCgvy?Q+5N2E#d<07p7sSr}ezeC(sv1w2MsayEs>( zBm4Ja0v;w&6;PjTmRy1y6n;~0|M@-k%dl*oK0qF!3E&s4gC(ypZU{2>6FzKwTkybI z*Kk03%k&7dF8qWiBl$YfSydXuWNq^KeRMXKv!}yJ6S2Iny3`2Bh1v-LHt`!fo5E}4 zRgGxB*Mq+sshatt&a87KT3NK^jeiM~d89)_5G5jZ<$EP7&xtD71Ne10n4Cw-fK{h4 zCb@V^yeUd${OF$}&gOA#_qk_%$tK4^%Up!=xyve6??IJA=D&_t#0lnu&JDVM$k%>c z?|S{rN`441@N3(PmP1d0MS*1hvzqxp3toWmnSU!IJv~&^ z4r;~ zK94bI)HJ@Qn0nsksMM@X@?jJCU>`!VrC5buqwET$6Cef3O(;z;H*uXngsM@F>0STw z>A6qZY}12n7)kR-P+k`AYccDH|A?w$>rCc!K7 z?{a*f?&csw0bVIxR6*c&D#*-4uSyxiTQI^mt`Hp(=pkJ7hJl(pcPasoP|smm>b_Cr zWQ8QG`sQFVG6HVTlFK)FTU_Mus^nSbT>&9xrt;WsMa$->6Bae01}g&sel9tRW5E_( z;|K0B8}esGmkT{(UDgJ_@S-2YXn?x(X@D>#UwF1Fu3D?~F#5PdN}R4DZ_K$u;0(OV zL-LQi1f>&7Sj-DzA-d-{Zat&z@>tc~xw)6g27-~BgSr&sa`BGPB!W)?hRBeq_pn|l zL&@vBZjcT9TC;Z-aa@}1EAPgC?{NP>XRhGO2qPoQlXrz#jiv$uj2qyJ2h-b3jRw$<#`DnwI2@N{|J5E-J=*6kle>ymep6KMvOzn6$WX}pd6P!-U!Y# zmPK&a&h?GMvYo8(g6LJU(~WFJ;A>Ki5*tmr5sq2C(JMA@{hXDrfjoUNdR{)E+KZfg z&z1;(P@M{{<3XN2Z2D9Y09*kkKo(m8W%9^73Tkfv-p!ryDFBKY({!mkOHSV)xgb0b z9x9)F)ASs;=4DwC-5H=ZS~c@M6?LnR$K?q0Z*Oi{D3xM^GC1y&S1w&B_LsXi9qAAH z{U!zNfqazU1tgs9+v#71IK+V)N*uS2`uO~KL}Y+%AYT3BZ8xkhtzmf8jpF!H4rw zN1qv7-AQFHVXIzoT-21BL~Su&Rt)Svz_pEhu^TzMRX!~e(mr2RlrZMd+#~RmHCKpg zW#u-ZsD`F8J(!ok3fQ1#4aN9;)C_0g_SIwh)P9f84XG)5r$Ai5bjqV>ksP@-vdIr0 z$cg)A8fggGE{eGwP1%iJ3@5Y10y}*x9@eF;p|U6aNUU{-!{6zR@bSvLHMi2g2B>Kn zxxN2aApX9dOP@7ft*)y8^S=XOG(iCaB=DMr&(^aDZ;LsUtzC@%=iaZ>Q6V2Hvc~SVEEp}pDd`QWpxacq`mgm=%x_O zcmbv08}{O4z<5%oZLtQMN!=Zc=54jzWr}+8BbI!$G2jCedvPSias6;uP`lz0q zw@2b%{E~LTUQ-IM5j&la;A1jplK+x9C??CXO~qEKd4>{$b-P>VySLhQ`hKwPSCzko z3P$vC()Y^l2>n+8j0$u0rJaun*PNRfqotrBdfSZr#wsBeshqPK;rWr^pVF#`Un=S8 z!>ZN=ag;84j(v}I-ly;uA{?_yq!MT506S4T0yFDYO{HlUqa1I@CHQ#&+FX+DLFbsR_rO`@~EK;*$^Q$}eEcCq`Vv$X1?ZMj} zQZ%*RoHBAk4EU;a6y~H2unH^p%0BHvLPi3E1(7l;a1oAFo9jlU!;XRT#b$c*m>K-` zQ%b=2fKfL_-bV4757`JBuUqwWDYHap?=HT4if(vmoC?sZkiy8O8e%>qI(~33Zw~h| zJ;f6?fAhd{46vz`e@G6Z%H@o%JSrHTYlivbZ)bt&ZfDMu!27?94ooRYKTWz|;im>| zUat)MBU&PG#3cmsrxkyygngwBZpJr6h8?@CYI=$nCLAuOk8!&aF};8xSQ5=>H)E~; z0cNJyv;Gz92N;-9L|LFr7uu|BEU5h+Aj?fWreQlq7R(p3K^e8}h~>=*3{e#R^Tvp$TR^=(!}9l@GF z+$l=DtCfJFoUp90l>|6)vgR5?s|WGP5|mr~XB*F+9sztu+he3%)R)=!hirNcwY!nU zfGP68+y_l=aB}u#mz@_N(FpZqr|35aQBABP*r$u}dI?2Giyy!JpLnL4eC=bg-FLT}tOg(f>rq6#ya9oxV3X8Rn+bcgwjZ{H{|@ za5D)H-F6=x5}LPT8!{4DD7d%k&6P_=ty6MZEKDege?^PdM&!2f2L2p3m@KiE zv@OY2G+^xU{hVUVLGZ;^s!$cLfwW&~3NR;eN_Xhs63bDgDAQV;O_AtRa(5ir`Pf8m zi_=g9EV!>LTtJz7lV*A*TxzJ?ZD--S%m9vyEzRC4F%I}I#~+%VeeeJJcD6wL1#R+zhkh08 z>OB~Z#uM6Z%o}NA#+`_fNZFb^6y*Ky%JexChj!=NtAFBLzRe@54*yY36_7nKQ*3RV z34gpwM9d~w{tmKdA->w0bjl)YtqyJo&u=ZkaD+5c#(5BpsXSAuD@cDYu!T|UoL$3q z*IGhJZ-EGI!HJ1z?-7iZ(bZkIU!M#Iplj99D{GS!up_EpIw+c)!KC709f9x_aw8ab z;tWvYptN|?vgCN)k~@Ab&z*Jss6#u7CvbO!jB%UR5D2zDQ=WMX6}NjCkS8HE20Xyc z(mP@7UH2P4Yl5GrVq)*X`jYV}&ACYLq?1qrUlSc^d{hePdHS{-`JbTf6S-7RM=5?m znK`TA*xro-tw1ImkPK=mPlbByHB+A9;3W1+IjhKI#L*=^F5^hl_KKUMm2)p6`Oemh z&Qe_PtFSXBLLh7rnlneogucRX4}h4?5$2BE$S8cR=g||0QQpl7?|iFapUOyT*f$YP zEQHYl!GZc@*7QF>=2Hf>5hSOxq=z6~)nl!pkK?7L+`MK=h3|+2EeRbAV2IEhnQ*93 z&9Z7;oG9|tS_xXhF&bZ#5f7Nryn~JkS1q+zLG<3Ds6M*3{jDNEwl}E;Rb`T0YQY|A zCFRr0od`sueaL7Y-1q!BknLr=w$Eo2zewDXkM+exglV@PetUL~teF)+S+v8`XpuXO z-xvNR;3BI^rKHvoz%(f}!Y^nAegw-m(jaVA=H(efwpe>BG{3e`TXsO5*NVxBngrNT z)}esLz2U1he&iz5xh0(HJ~oR9!Cfi$gwNdR!YS>tvIS8JlqzGG+YZguF7-uE~aXe)@`fH-SU@M-LTen!rsFhYZ6Ncvev|El3KY*S^P!s1N(7L=#W$Mx#Zj`tlRo&k!DPBY=$(xrer*$ zqQe|MbH;>uMrPREsqC@xalEAr5+Y8@Lg2D6@n#T*OquNgk01iGErIXngU~^|@Kkc? z#QgTQXdXmvkrO;gyM?9(VWVcT)`COr*;tU90t>OZv%^-9>!*cjxhuEEQF4@m^DKFg ziV<+7+AJnh2!z5vq>!H8g@K_NF6zK9V_n;!0%xsz&uOJ@5;5A0&J`}*eIJSH82`Fj z0+b9<2K<0=WCm{6KUjq3Uqv?@U;|e%UrK0Pg}WXF2%h;w4knPYRH}F1%9EOfl8JUB zCOYQ)x5+3)3;jTk)-0^r>NlL{l7b*~E*t!&hRFabfM1W0WE{fTR(Ct|?Sh+448 zuCBY+qkvfhAyYj40839cWq`x%0=U>}_^)5dzvFnr$;8c}s+dy6`hIpBms4AG%cdua zC2Y|jYHR7JPKN8WIfciDRD~v!gzK0&hj8A`HfmTDIU&YALOQQf?WQWcE%-|6-SFDR zXzBbd{?Lci8@)LG@RZ!7yv<9$19bQ)2o&peMJ-WzS6;&`BV*(SyuVnzLusx^=+61(?Ira z=W3MHM0FX}vFVCR^GH3@Ur>S{BT|Xr0qey0=n7QP5YkQ=yhr7BeWZss68c_a*lDZt z!5m-1`{{l+>zKtZ52plN!u%_`T95x-vr+*?c7=*2bS%T#yChI~6?3Vmw+$vBFrN&w z>YvLhLA`1q8nWhB(d`a$a5x-EnwI0uq%96ANq0#jULDY14j*e)+UhlT+Yf`A%6JXi zv{gl$W1JC(MzC2k4zW*Et&$pAB!8v~H$#^n|H0+R0HtDd=Nwdc}qD3Af) zO470d#pif!e)6Zll5UNl((tbcXeCt`nXPtfrc|myB-MA<7--J_9B&VS$Qr;R-7FJK z2sGGB2b>DoAe7MU$0|-`fOqr!FAt3l4bHXv1C!XznHy|(|Z(?c+JUk#TMrmwx zWpW@dMr>hpWkh9TZ)9a4FHB`_XLM*WATl&KG$1cbWo~D5Xfhx&HaIpmATLa1ZfA68 zGaxV^FHB`_XLM*FGB`6ZG9W%a3UhRFWnpa!c$~DlWmKHawl#_dhu{#bad&rjcMsaQ zy9D>(65L$_1b26L3GN;=K(O0+_t_`e`+R@zU^Lxx>0GN;RXsgMlM^ec(hHl~8v`Zm z?Of=Y8JKtga`N)_cJ|Cn^eRAeR~sWI01E>X6B|4^xu_G+$i>p$PRz&!$OGVXu>dHV zxPV+i7EDas@ZdjjNvE=Fpe4nSrAmC-*y$==z8-q^?)qz2lVTiOAs zK`x^94xUby<`yo$W3bZG|Bm$ATZ921V`O4&@9u1E2{5uV1;{YSGXNCq-9bW20F}KR zz!+#@WMc-fHv^~vwE*g>;wq{DNfkwPB~@w$P+(P82M2qn|KTF4s-`YU2M`lhP!k6L zHRu46>Z)qL|EmG*K>W?=019d#{qH;=!{6@m;%dTb+DhWgjK6CDFaz9xPR^FU)BYDX z3J^2EA8H_1Gbel7e-Z$wEL>b1co-So-Q5|?U7cMR?48UR9BlsKr)FX43~;x1vIco*lisJ*Yc&3@#on|5*QyCoCoh8bwYn01FEXfC)6D z;&!H@_O`Yl*3R(1+b3oT>XVDTlPBZncDzCYx0js{r7IN0gWx_ z=&}5LaslX>IXM4I2bxF|YdfH`Gk}Bp9~BVPh5zCUYW;7v07ey6DG4<#+W#{$|Cou} znb@0J+L;4b*f{`3PEJOi@JygNU}0wmcr$}W+7#&V&!7Mp8SLy`KrR3WR~H|EnY|PI z@5SU`2QUi%7X1rx02oF7AWi_I=pV!dU=;gb#K{C;l=y>K0gTdr5Gab=9|Vdb{|AAh zDEvX(07k|CMO@4PM&&<<1yuPT1j?ZL2Z1tZ{6U~7+W(8VLHUgSux0}xmp@8SdgDI` zIs>00D z?XMWW)%LDVe>s3m%>RNQ{uciW*+8ph;pt!j{F@_4X89Kcv9kILg1TY-7X;P#H#<-R zZ2v%JP`Q61vx8jiK(qRn9#o(GA2J|k`~OgbigEa(1`%-pI$7HP)j(!YagKl5^&7bY zo&VV+e{4V%|J4s>PRIx4-IV2bnutdi+TXV(n~WgN&i0*kpHC^3aIAXawBtXH!fM)*;ib8uiDIR^_F9CMNTI9i)Hm@5k zNmWy~rpiLQM!xdCc*x?yV>ucO>VX;Z%l_q^B-F<;Ir-WpY_l1y@vVda^|2FC?)I+7 zXqL%0)azstz4F$zJZSf#dFCbzCmS@6<@X;t@Rp}Lz+K)RF=C^PdWBbL<_z;gvBqTH zIrMbB*P%_xSJOJ__D*fSsu&@hkJ!{B|7>Yv zojPqOxBS%rl5xbloJcjPfnrs;L50BZL6s0$N7U~l*l}Z!AnduG6|pT{S&zTfi8ZU< zG+)>lJE}fawMy3)13$L(kzVDTuB=e2Pu&$9Y@%<4K_7Ceg>mo)T{vf%b#q0d=t;vS zm;7snQ>eO>p63;^gMQUXN;H=d8GZN^i0{0YDZ zz!7gzNZ9#?#0%R{;nv+Gdilt_Jic~RnSO2St!5w+g^D3J3j-`qdB4b(NZcOJ)R18A zY(%*;xfti;Wcp>oLYdnxl_J`x};B>Ax{5HmO5hobZ0Z$7;zJ~FFCua|o%=2BLvi1}jGl`usy6k<={=+p#&Qbmq+ zDxsb+e%6Xh`<&KgPO^dW9R91-qqG;k3|;P8AU>ZFh5@B#ve;$8jFU}1ZYv_Ub<@#n zZuVW1p9qZIMF5pS8=w5FZ8ynRduR>efdy!_I3rc6@)}Z9=ecDX{&(ynpEVm^W_)6> z80t%e_pFvOn9_sLGUEFA*OQ1nW>qXKc}{e8!b`Cs8W!p{0FmP%Yd+7;pU*XJWcDy1 z$x-FBL+nI|Rz>q-*SUpGdD?>aPf+1&co8G4q4jz^9fU4Cj(MQJVye=265HQxy3WEQ zCHT#uQh3+#i{fxdPZojYfu~du+L2u7Xbsgh6k3I%^g5+2T`$PkE9WnH5HQM%!<>l^bcdkHBM-9awNLoF1QR@gdnNZm@GHH_K=SYs2LYgD?9!CS}9^Jz&VYr$Mu zz$PW`HVqYYrq>a@ASF0qcI?di z=Z7C0biyVqg`8c^-)7Uk?VosUr+L}>3#NZtk)wp$ijuaQ%q6(F675z6=%K!!RpM0@ z4Udd^Fh?!JskpYz#^6T9J2Q*TF!3MVKh(czZjwHPrhUtTm|^RAOvxgwa`Z&(s*llS zH-*D>bK(G4u%BE6F_io0S~slB&YIK=p}n?+#R9eY_ei%tJi_I;h8BB6Z_07lWMadz zoHzi9X9~r~rc*KKa*2(gQa*_S28aIAhyH(O(5r0+BBTTGC0s zrX>0c->czIP$XSuGT<-7VD1PszIE3pY<-cQ%*Tt}aUh~ap3rly>L|1erTN+hwczn< z-q{cDi&kkrCVow8If?v!ptgKmXjfhq_E|46Bg(u*IZxC>IfNh}(xjRu1`Z9`Ntm31 zEmK!saNiQ)_L#hau2st(9nUccDHxmK(^9uCc}Nl8r<#p8#Vt{0O6MSXdLpR(QmBaU zzeHHo*6Epp8YRr>pXkLW{RxVnn=y^)yl>QH!(^&x6Ug*NWnM~=@V)ZHNLYrs2;nM{ zuF6QHN$QJw*%Ke!J_eAhO&%pJ!wEMqc~^7PBSHwK~trlm@jC@aSog1w7OrB?1v z7+{H0)bO@O@LlDdXkOR1cjS?yMb1rQYG?1GYs{*D4Mu${YvjIPPb_*cdgJncbQ4O@;B4Z z5tnRtn%ka~ZI!oG#WpNhQk8g?FWFPi{E?p*_muk^kb^Sx-G+Egr5`J96=fepHq}ra zw49E9rqsA#)vC5{;^xF{>avD_3vpl^!)%O#2XqvZgv8z5P${P4M8GyNVEHOyI+vJ# z9GEEd(ZfuU_1(1f`%boS6Zdl+0>5CaI_bhy#o3U$ERjz*)D;7V&Hj4S@ z$(I*1U2Q_Nz4d%BNUKgE^3Z+`Mk-hpb!~JEaDi&+auG@#nkjKD;HZktbYC?QNrwQR z>yYB}#|#2Shf*IdA`!*j8`(#QpO1FUndd{9u&bfA;of8(z6F{idg8c?mH&k2sDRX}i*9 zDeeNZ(y#4OyM$T zFbk$jh7+t6RJ&lDjW#l_m-f4h_@xi&-2o*WAFjRlgjqR8ZT-WOINVc`%g{2JSs%h^ ztGAQm*ESI*rOxfAw=NO|r8T8Ks`LgZ#f|c!5$7?wWa|m1mqggkJIt?1cj^0vBtisU zK^)l2Y?THa%l%Yv{0Z)}9#ZpA#kiSU-9d;q`ZJsW_%Md%_sAL_HvuK(KP?k6_5V1= zV^_I1V00=_1c`;H)ndjVuRA2KrR3y{ehMHSyn#J(5N9iD7h&9qqF;gELO(+I4(n(- z84A)4IS`Coo`ZyMH=V8}fgOb;hup3ignl+JU?&`QH2i9zd%R zq~#s!WQM{Wu^qVQ^nC@=V~UY(P3ieeR^Mb6>%^Et(5!oX1?tz&3h+{zkEC$Wbyh){ zr{ufGVQjghG6C{|3VWlfL4&8UKNYti-)RgeOPI>k!xAjTnT0j&(DGE7L9q3og%5n! zpt>a((>V&Xv+)xwQ2h{yYxL&VGBJGQMwJ+Cixhejz?!V`2>QaC1tvZA} zrN~`LO?_-&qg8xeqYE}b_X?c3J;JhNpY}PWuV-BQT&T?HBeuTi>tIYa|5c^`!`qZ> z^va}@Z_9@G=U7Sy+O4I`0BxTkW&t;+?FySzh?r-mWQfPb~vrF z!|SywSkCWDxm3|j(2W!LEhLW$YiS$mm_10XscL~ui~--B<7JYFISyA$5XYg zFX6iuT~o9Q;lATLCv6-m*k&<9_45x8Q=ii6W(6v;Kj-XHPB4+@!gAy!}Pvew3>d%hDvbm>KAA18w7oq?Sn^vwEecGJbg zTsdardZ?2<>XQPQdyDuDHGSQ(EqBAv!)bb2)21U#C#KA?7h7b9|Wx=dc7EtSeo{@o?y`~|W8$-N4Miy66e?wrMhMMgd!2_{{UvEACh0k1ST+t9QyX9}K< znoV#F>h%JyKHt!><&dQAo|g@Yii~PjYEfO258gHv$SR*k>PStaQBhCd0)jM{t8DC> zqhmhpIvIY-@331%Lm{lF4(#S)(HP@!KrmjJm1OeURI#j=FE2&*man3+K@hVn{{-vM zq^WNnL>sn6o<#eP)#Q~8}gwcWORnV+80 zTf&pSobM6K>sG4D$0YdoKhi%bz-nsW#6g;?_jesTTa@(-+s=xXpN{4)BpKG_;7Hd8 zx%w1lqbcFSAH0v((GjG!8}fdllI#&3x2|D555T>7H*P>d+O>LT2yWr7f0IoEHL%%| zv25;L_jMe?oyiN(D17L*e^24WMSAuH#e&EBNqv_735$@x+O|G^2~YWjqb@zCR;ryU z%W1gTw;HzR7deUu3(q24$f@~OXE|5rn!3`^`{^j@T&ABq_++k1 z+%YtUcSN`C&G6r$ctrD)Zc8q`G|P#vJZu@(Wu|1zT|8B{VKXJDW`w<(O?r(Cs`}0F zwehYljVeT`PHif!Mb~8TmLCEf@tPe-XL9c%zNNzp%!+Or^bbnu*R2WDSc!<|A;Vx` z-55Yov=DvXW9^HRL>5U0Bc8|G@oAwx>3ag0&(4tjGK;~YXUMM@vl*UsY1j=i9zUL* zY^;9HqT;?^s)9Lgf3F2dg4FIh>TShdCtwU=2|3gVe12PWyl9c@c5TKSEL4|1BmEwd zLV8-5_?Wpfw|SXLE=`Fil14f2u6xAcMi>5doYlNk)E-jb(^1dTA!DJti$!uWuOmTp zsq+l!CpCTT`Nu`#Vzi%5lA&xFL@s{lYKO>#b56nQiys52&V{G44o9X-2I*}G--Jrm zDEGQagail@M)qe7qG|Ecngn0Rt3SCi2h)CODlE*gsZh#2L|mw{EZr(q?WTCQcC$8W z@)PbNW|V0;EaU}CS4%Ai4U64j-EsOZ4D!0z(XQE2mg`Wr%0OC~?;&1P594cCKiKU! z(Xhs-`%+4;I?mWs5iaMc)rfdtxia`Sz_5x3^6s3G*bU~!_POzh{hQ!e?T zWI?$|lJAy~W5vx5V{;NH+5Q4Talmb@RcQ^Rt~f)f44$|7bSkvg0`*RjRIRPS@4An6fI)8SGNQnGowj(A9pzebV1V)0pj&U^yhiN$ub9_&-3)DdE)ncOMwL}C{fn!w z;olmRT%D)62T!H87fXST6Az*eV^nBIo0)c(mzcviVu{Z-i=NdRD0|kG!5U>h72agB zQ4~06;-7AO`4B|8nfnj-tVl;&Dl(E$r6}dI97Qn6J-iw#-z?45V!$fy*yg#AW!u36H~5;1Dqune zJ>I>3%PPFadlaaulfw65?{61(BxO-Hv$INn1M)G>IV+~JWkEu%?#e95DcbEV#|B8F z_t9xg?mn2}jvXzDmdG{?e{8ErM{4v&;a|v?=r%wY7%IvLT|W$;5*J&7l9-ya<$nMy zBKTsl-dH0gjk%j06_w{tZss;iJ6bClaT`7Ei-*lUbo*X)}G8(Q?t3J+`Rh9 zLtA~U;q$OJNQK2P5?`J*MlxhAv271(zw&vet3_or6<~PJq@i-ObjB{=6QFB3J+O%I z!K1{wj;&p+XXL9@zr(|_r*!<3g1s#Z^%dTfacp)_M{P&R99yU+l(ck+N>MUNn;nxx zJH-hv7TbjuMAkCskNtmY!f^=23t)i4Kxp4i>kP!!>?b2+OZ(i$U!I1a** zbu78h^xy3aK@V=Np2Gc~VZ_PCKh>=waetf_7qfSdo0tubd)gmMVsPsiLOrX~g69{n0(wZkdP~*hG3yQxgiXU9fLgLri`_|BX(ldo{?Y79`s{R$=5k*NPoGzCelJ71~lJ*i*L! zZ{5x%>A^mVo12&*-W;tEqwW2#DNEk@G0OGfKE=53;Wu!@asFn-YW9f;;k>7D*AG4> z*(?K=4Hj2u&UV8($IR75RTv9&pVzMgGlxPAakI^I*RBpu7%O}{Nohxq$`A14Qa6_% z&Z|U~V;Oe$XRkXuxk=N%OZs)tB84_VC_xch`LyO@lZuOGoN|wwyt$r0{=8g%-KSz? z8Ag}po}j8GPO%oLu`+SY6ceRgO-0ccd!aUVND4-An%Wz>3!Z~?$llZ?`pS?WYoIHGDaKZuJVsQ`ovQp+R z#!$Z+EuFYzE5#)StbY!Tzj-C_2D&?hm9;y@ILaC%-tI7Aw?zFg%tvk+QGbM`8~gP} zA8CE1nN%d-yIXLI{|ky0!a+?dq`{1PpnQ!5?QmRamyOxBBdtW(r5|0@z=35{1uR~;{Iuctrz7U>n&c`epE z3cGC0Y8QT5HJS7yGRYmOMSf2nwz`o?HN@QVV>;L@aGs%Kp5=z!_7bzMacA8QVbcrW z(A zcHmS~=YtrlkWIPpXkt40ofs9*O$(vF#BDva-d4pLCH!v1qEskqsIaax2(!XS40%&p~7^Z==xmJ=AE$cu=LoLM%t)Sw})$)2a{;lvdcW3 zi(fKwMU7T#4Pb(~-7f;NEx$?os0GRo2?FpgUO`$rn6C)l^eXgj*nKMr7B@5@&NQKn#HHo7n*^(*{zEm#pF$q*Lb3!SEAg! zLgDm=gspCeAi&Y$&~pDCOmL}60d+(%7xcwRBu25^>!|F(N<$JLFD3KC7PV*(M@I-n z21G5wgtP~G+A9NB=MxNG&MUvG%Tpa)X+?F^IyhesTOvYe^vW|w2??2B0j_bpYG4o# zeQcamQSHuieT357cL4{WN!N+kcqci)S2_WK!Bk_tpTMNt&J8P4@>&OdPP` z6D$6v;7jW(AB}iaSG`VkX(8!3zZ9v;g(H%k(B_K-_$+*}SGMxW>`LLtV14#jk^D@* z+t-Gy`Kb@x&Dy5q2F?;|QRhPta|_;ZKcehcW_W60m21LzhzSEGauE2}@i?BxCHzqX z!f=Kz!6BKyZFCjjDaBPOxcF#LFLG!$4obH#Kn3)FKPn!lhQ@c)3woqYMJjIlYy)?Z zoWvCSH1!UgjS44Z;_d$%GK4Et-x%fW*`dN;CKO4oX9+D-53^ z`6eqIcWEs0^kpP&dEa|ZL+Yh{A#CTky0b9SX$^GV>U|<)ggaPj5AJF#!kf{&Dt9(@ zK0AMbb@GmR+3iy+czsqZl2SM%L`H{XwE-P%Yac}{#fSa4H1kg~Iw_$@Sy-4_J{E&a$F z)WXY}{Gs??QRT@wB(r@?FKW?gs=9p=uXwwxQgtL@KYq$tj(KQ1dt%D)i{C$CO~d;D zC7o^khJ~MeNA#(rDR3a*-A%E1B68_zT*9TBACe_x_K;DwiZB_bb_|9Cu6=Ui=TOlV z7S12Qw1*cNO?8%1nlGADG3iMG1@V}~%0K(XTNZ2=4;q2i=DV*`T^jwnVHR*ewEUH$ zcX3Oewz=V%0d0{x82S?0#yu-^LD^xoKHN3@^q7MCWTbYvd%tL`D%UZ^0sq&++NVx~?*t%|gEmbd71U zbS;rVadYKpFrzc-!tVVT82pXvtFo--*ag=>F=V8SWkqp{bdwNTBJ29Bf) zBTkBRLzk&a;+0ACZ3uSoDbpyv))uMa-VolFo_xCDucaj^O?q(Ae-B~XQpe6^-bUiOT#j0&k;Lu3 zI4CD}DAFmg zO79iSGRiyjQ7*SHWebH03qQJ`>EHZR{ZEkFCLD4%z@54OU zhSb20=$zBPVCqo#W{pX^Udd}5x+p&Vqxrfvu+NqFIQ_?rO%>BQPU8d1o+i|Oei2b$ z+7Y2w*R-go?;6<8&G?aAYONFSI|Hx<7t4p3?+|qp`Lg>hit=(O6`m7F0)F8i#rUOT zyxzA{3o+N9KNtLPjXiuv4#@oCwmb)%Sbk$t((XM*Z%PSutYn(vu<#z!=Ln7`5#xO~ z%uOKBD5aYmkg-XN<@(lvkT$g%ekmNaOA&_V*3#}D)V-iJ?>9Zo`aZ-+vBq*V+8;U| z^X~XN@u3Tqv-d$jebj6NrvU9&AmhhA6R$bzUs)j|V^A2syfM!pj=G~hD|)$`0^isW z&%l%mFMbw&31m(<^as9}eYmUy@N11A4PG71y^t@DYLCxp;Ukxeh+uK8gAZ9LdEeL4 z2~+%}N39SqT2@eMwqo)=;SQwa5LEf1+Ui<)R~OQKLE%+Px*ifCXUOGc)iERH+r9wi zqlDzZN&6>2TML3`De)aCz8ciFML2_3G;UBdFA{^`VQzu!5Ac47)->GUMRQDbrQ9di zae}xcm=Qh}?m|{)zU0D_9YWKwSZ*)3fn{l!9VNvs*gny@a$5%=2Euh(j16;Wprv|2 zb=5<5Yz6|X)z5rAOK={zkNWShRBIim!)wA&XL_yd{d#v47Ztr-I(NWop$iQLg>35; zv(liE#3pPducH8ywIzlM=B8Wn-;Ai9W+Db-8Y`OMA`oF8g%nZKo0zuisjV=@MUYJd zQI5+xbG12=C7taav)tZq3D7uN#%4t($4>2^`lCw~ng@4gY|fNNKBL;f2`jKY%CtlSAJKhCUyYg;vk3QsOt2=$z=aYa*&1$wRc(Yk3{^>WWlg`T+!QV( zKqSq&sXS4IYZ+c9<7m)*4>X*BksS%Bx+*6tga#CMTVQF2Ic?M75wJX6)dX@|VPA$p zK<%s(wVaNKMMkish4i%cNkdb?`VKI>%CB5V_-!_7ZK9nvXTfAN^e)}9~AjixQfT? zU1K<=vffu;D}bwqh08-Vdg6F^UI=1dnO*I0_)1)lLJFJqsk_svux5)}24&(meN}t-f z(Lu?1>~nk^=k^ zr<=?7iJYq18bcPTQ3A2L#-gq0Rh8Syxt5U(a8}DmT2i(`L(BX3B+ltF>OyH(5EC!R znx5NL8w8Yr;Cq;(3&~_1={UoBM4z)096KdGckdjmGhXDNF2vJ%Qz`8sb2UxKC!J)z z*;dp69%An#A!Z+Ea^YwGA3w1WxTVthnAS+3SLx}m>v%O|Zmno`i3f?#XG@ zWv~NI_Pv{(4i*1HNxgfHjnnfVDAm!SxI3+@BPIG^OQh z(-vac6<;$IsD>&eD5N0iQD&8!210-VJqzlO7^)T)L+&#o(CU@-yF1WhN3#f|_-2qu`PQ*E0xdiBpxTiDt@glXq&^pIQq4b%Aq z3!_YRC;Oi$Yge0o#;e39LCbQ&DaBG#Uf`of3cVs@R*@{RU|##b(fu$PfXO>ZN3m^KEY8n;y#-5$nCtT~o%uSFRnMJPu<+#QcLZLY@bL&aH<;hboX`nJu}>;JexR9L2PhBbS16 z;P`pvg8FwQD&E;AUBE(OtdA=1iF72`oI&#DicpmfQ*^#`e;MGc4g5Ke{iMyUwyk|W}7M`5E{f)#JvfWU2|kN9BzplauYgTfo~Tq)q3wNDBhY8^wZ>uwYBbJ(Rnk@`4{ zlr*FQ*}oPR%GNQG^%ZSgho#Qvj^f*frX0GRS$(O5kQ}WXJreoM2st!?r_**0-R7eao0-q?o~S=?UN? zl^>&TQ#U%U=F8xecfO-Qz49toM@}5YCYOIw{rL>9JAzAJ_JI~UcSGU27XU`rcQQ7Y~q3Zto@2~s^`KGz& z(rQ=w0g)T2!WUDuO*e29B6Pmec8UG48uMFtAckaO|CA|_@L zy^lnkB39qM1%KI+p>;KQ@Yi+!TtsH;)RqM<*{tqc@-l>RuXJb@F8abLE7ax9uHDA1 z5jH$S_Vezh9HAQUAhIyLLd3}j)4K%+MI>H~g?A1h8cDLA+Tp=7W*yE(%4btLYUf zsn&XN@oTk@-~2nHIc1`y@(?J|H|OY2TB!w&^W5+HQ9ij_6nlpf81#nwVX>D_QHGRT z`Hn1_rJ!(cUt*GUF^*YN4N&&c7x|YIHn(~L zK1Klrq4-6AU{WY?=c*KYfa^rj^wt2#jV2oy-#>IkdMd}uULOwz?+n8!f0HSnxT5~n7RtcZ6u_*=22E^wmXAd=XE{oo%OHJ)k*w$=?-$r+c^?Cl7f;^@^=!--DTvFBUtGANdUc{H zSN&z}v54xYm4a^D5)ZUCh2w)DJe#pq7dHV?)nMWO*UW|rDd$NQvak=&Beh{UW+;EsI=kXU}@(G^dM@G8jBocs70 zaTu%!Tnn+9&0`!mDCz!V%hxEn@SK;<2}aF!)@ryQQeti{73IN~2e(fH0=i2J~QYs@U)4YHui@ zJGV-dl+kx@S*V_a*y KxTs0$&j8V)@_?FUntbTSc|fuRyJ{L4m-eit;flbV_Rcl zP9|^pfERO@tGS zreXA<3(=3(+~p2sKk>)K?Codgg&rC&vF#PWu_j=PvhN8yZ7GhP=8<(7y;upSoxbrk zW+dxYphkFjIrgV22PiLnMwB%EL8))3Fb%7zm)zCbpN5`qrDM*e{)2|otKeKQjp5qD z+}5ug#dcRItJ`b(v=nOteR7UW?rHxiE-{_+I0Q3=OuHH1ea0)QD_ja?;pbTZpepYz z5A%i#;&WyeZZ3K<&%b{cL-4c>OOMavIEry~c~q!=Z^Xlpi^w4C8ND5=C)y_#-0cL` zbvS0k)lf0yq-2?9yaYpN16kt72?t(TyKw+N4_qWw3C0)uf`PTug{{aPK%?vAaDl@w z&7xG38PyGuMk?lgWG!7hZ$@yMw-87ZCcI75##_RDYEGEArF^l0WSVgx<^D60{y4(t z4pmN=JpMVg?OTMDy^SwvL~^1mm37i0Db)|hn29c3buv}LyAK|K%i16rY56={dPvr703veXjZPMp8risVF zY35f9Cb-lF?-hPI_D~ZhY^)8FH0edRF~&lgq@y3)1gjGlRwN7b9))};aK69_yq?Pw zUA*C2VOoX}nvmkXDxdq+Bu8th>cN@cgw!1u6Y^jsb{-i%hDR=j$@8IfcD3pYw?_zd zp^S0ovzgFFM`gdhU)6FPTCIf9>fkh3vO{P z1BDNNVkGUx!Vs;}=S75$tt$Jdl1{D_Ig}x=6#L8s_L2o9xmtasRY7_FcVv&^LTxF} zgKHbM)dyepcJ|HRE8}(urA5*D`lucuoT@g9wSb%77_8h*FBg06dMYS;rB40I&(lv7 zcH*1M`9UHNeP&CDkq6m{AFkz#03$7`A#WU{SEn5bLCwUU^{F_0`@wK}c>;MPZPs}F z4Rdd4qA`ScjSVRfmt(;51I1}LeBWxhdfgxpK_K<>^~kO>8}(`>-~BihNg4VT0*SRX zJA{iFhZI48H3GX^)??T=m{CNFn**@3N)dYriW18eBnLhfaf6A?Ws|l1^KCSPn5MR`<}FE#51R{g_wi}%A>VA7_2%kv8$!!HhYiE@%zM<8uqCBHDa z$R&Mc1vf0d{WMs=FA2jQJs)_A=AD4p;eV<)!{ZPtnP}cBl&dBlJG``5oA4M9Ol^zi^LAL4-2r7UueZJ+qim)*rmzdGvlYj3`kx`*1K;=6AR)&~ysg)> z;bvB#mcdaMP%f=8d4>1-?p!GwzoGCajPTqTF@9|zEeO1yMpnX$m8)d9ov=JTU5x46bT19H_`Y8h+=8u7HkvJ>&EhVf!@NLQ6;6G@AQ_MTY! zidUC(7lwVXuDx{D_j^_47@vfM2QV}hUwzuq_NE43@6RnE6n8EOguT)cZWd=B!TLg- zjJ$0NH*w@sda~Fe6x;fFbG)Cptc9MkApo*l1tXKn<)`kgNYg@V=ao?-dc-t<3!9o2 zv#~kiwg9ZQ!DoDn&D@JjDVHDwqH1yDO#O*cHaJFQ4jYFvw9A{mSh!Sr>(stl>iLs7 zX%9`Rwy1P~KA)p@F_-e@onca#K`j^`Tn1h;vHwb=>o=q4WATreB^M>&azP!Mw=ZT6 zE|QP9^9~m|KqUenic~_TAN&ZR26-@qvp`ge70ZkBV^R5Q!lM!6kM+ zuEMUl@5GNRn#a5dd0sJVMrdU0!dJ3l(r?;zdUfr{k?3u`?RCK?G%*K7$yCnHprR)* z0u&@yn-(_p1!jappgOdnzNBFO+w{;b)Q?>s3A%sF)pkcmd3=QUdemTAbnQeju@7zs}?Z zld{?0%Vl>K@KXP{0%x$3=rXe@NnZ$b=$xSb#5e9H{Vr~DaQXfn79=%`n*xQa+uG7a z_G@Dv>7|S#V5?|h;0;b0^Cg4s^GNNQtcD8~C9IGsl_c0fs4mSzfexaUHdoj;?@t^7 z%Oc;MRW4t&9^V#Lj83)-h|`SAuc*JQumYqgTB5gO>T20Fe5j1Z%F#_MH(a<6!(=}QBFKvkase?T0HaXu$mVO&}dejP?MuM-Ec^j}mro&^0vVf3oY+X(CC z3Zpw@%&AU}V(UuzWN>q%`cr_db;ztRJCv0@5m7(HUknl58_+(51U*BTJ5wOVcDH|Z zI*rshB@;5Eew>i`H3X{>n+S^?IPD~qLmkc1O4U1#ak^cDeNu4G9LkPYu_Ge3>PWuF zr_Zl=_9#S->8?R1bJmw1C#Pm>&oGw}SVbQQn-~Z*CKt)&`w|oSDM+Pha zp%iDxmL}Ty<3bUWRnfbah6N@PEeK+3|Al3%76XopjT6z@zEwwgQ4!+xx?6(p{htx> z&Vw51%CQSgH*ok`K!ukY{5G9*idu%Gf>RufK+wj*Q+hZ>h|lF z5{dgaFEb}N5~TYXgrIISy8MWArix7OYYr07>};@{A4_YHU6<>hTn{aPeyCs3cdMVPy6>Ld27a zj-&8R#abZ;i+zJYe~`vDKQlf?n{WHX-w_k9ytE!24#|EX*oh3mrUK{bSY1!BC6>*; z*g?cvRlN{Q8z-_2{&Y~-Mq!&_Kk<&07%PstGC&WzM)4=tJ7WwOsb>QY=v(*6>;^ehYlKus{YSE zd`l}hWm%*xWF%AiW~timk8=`V>@QQlLlp8Zwl^+GJP|Y#;rD5*f2e^nbL(3hZQHi9cOP!mKJ1U^hp)S< zn}WF+x>rJ?^s7^U@a{cWcgk7GA3;@sNG=-vUUEMdN+J_-n`PCX@Eo8D#9u?Bo!YNP zrueaH1m-ZLg|$_4EGFNg#~Tt!+3VPH=l1Vwf`0Vfn<8ZjoAf`>H7b4jOCV|Gz7mV# zgKy;wiq!Llq+t}q5CYTv6|tpshQ5O0`%uva=A1y6QZzQS(S->b7B||A`YAwC%SiN% zmbs;`;y8bF^1l7U@!}cZWg;o;=Zidk`i6V5rgC{ibmiaa3hAA)GD0rnZzONV9FE-A ze1?@4);*6P(03Nse{3JhbAOaX{W!QPqQI@a)IZL>2=m6~JcN3y6u3oTv|_5_3-I_1 z@?rd{h>1W8v?-Lu(OITQNHb}ew8G?f!|P+Ut83`df70B9&>>Q$Qo94dO$ley88d$= zaQ%K9N~m;V|MWC~JBf1;Q~%cGbLDNQ8lR7!=?E5z73s|x7w5;3e>xqoFqskD7ejfD zS;s7z`@bMYK!I&i&|pJ3#uXN>Zu|F6ta~n*Zd>$>8&wCa&}O2AMdfw zWt!pW^p|sv=x8hDb)~F356ybbF$Ec#+uLiPvQ2+A=VhDGqY|XBtjO@GMozz$tAzN} zZrl7!zRrN{rqtz%9_KhJ{HuKC@l%km$FaICXQH))?O)2rJ7`syMCdgS_8i3ZV0xIQ zBFr7AcjJ_+)_ER`tu44B^6hzrm`2->UJN60flc%TQFwG32slRvnA$T=KH)Ryy^?o}Wl52ipA}7COb#0qp?9U}n^e ztd~Q!73?zKoBes3G`uG)97oM+#}8d5R3vJ=O}2mG1fE~BJ8uUiKtEUN5OrMXYy7{# zhvi5$e`BuZ9^e?N^B+yiIpZlGz2d2tpJ0&n9Pp&aqnT!Uy(3q|`ExdiGg4qv{>zQt?xt5~TWj z2y7rD+7dE>TczPOY}i}~IQRuCF>Av8Q}SFW#j%qbgGWJk#cAaxD|i7d**_nQGMdd4 zI)o(LFK_MOU!5bxJV93^=vXhyEd2$^6d2ruuq9NN%XKcRR&QqWJ#{rKcpD@*83xm9 zmEbegOo~90gYord8qPuTwc-Kc6IQrzo5CDaRpUtHBe)?4qhx{n#|!%UG4`w5`q~e3 ztco8-*SMQR621D;FQZ+c-yu|m$t2m@ufV?P{Jwe$65K2jx1|-^c_o_8C1CslR7T2Y zuq9wQYQpM98iaTkSOhvAN-I^^H64b3`WMu3yYT28C`W~MiKxDu(-at(ZoXVmk=I3^HDTdQ z+J1P7Xk_`1vTv?)(oxVp>r<=8fBLiRVssKd2xn7|*4#tE(!931YGG$}h)>*pWmcrX z1s0Nm`KpP9^D2{hW)haau$qiv?9GnxKS37*C-4+y%=y0yfVR|MbcoHLN zNatz{Wm_a~q3Hu`tZ{jRzTmoZ?zUbgvi}6p!%TM@rqwtJk#AsA26E=qpdSWz{#ofLZAxMjp|S8k1eES ztc(g>v;hE<(8QC}Ss9fJSPF4b%x9g0IR8j##($h}@+`i$M`V^QiN!tbcNuFq(-#CV)&kJiJg^Y;~MsehV^}Xk*PmPn{@#8UW$yO|pDg#r}0*oVVa(zii?# z9Eliz7u4(W$TJBRqbgG>n095({6*FG+sLIjdQHM}&c8p9ibYn~qAKiuAE0~7SQ|8{ zZTE0Aq3x)uO-sB8Bv&4gS(}_)rDW%R+2C+*t5gb*GF3ld2xO6%q(MJDZwa$AboXm9 zAYL&l6EBY=g#ry8ros`rHV$@Y(Pi32haV`KJ5!bYg2N6!uw{&RTQuSX+#c^7d3dj* zpg8XC?Y_X+1ouR1lR{l`z?>k;O`b2cPZq(Y#0(K=^Kk51;U|5F!dqrWdSPF~TJ0iE zC|9-HSRZ7idsBM-KdX$qqhwA&k%(fD$=yBk3cpDXBL1)@+h%bwFIWca$tnOp?Q6J< zC6I1xsE<#L0tDEl<<@A*{7Z|cHQru;;IZ7xF9_A9_;?5Nol04)q*=zZIt*_@2jSqe zX7B0HyBd46DlvM5{Vx8?qGSqc6MjCXxZ$)D)=bq6cvzRVcRx* zdjl{=IIIlbzI8a7GbW3rx6oPqDj>goyD-$8Fhi>P_0=qlhsDZ4dMdOgJ;IoIqrwS1 zOFG5=Tt^M)fdp)sWX!+#6L!iJL?I`0_Q;(n;v0(liG(}2$&b4|g1Bt|zHgs{9>yb* zUSlHeo?O7mPGPt=W1-6T+xkdej~2}Ijfom$7ty-d@WHcj06%)D(AOvomtXeeqej=Q zYf8AKB$-%c26Hz=8JCqaU98c-a)XiA=#v}GX6Ztx#Gr*@&mphTV zzzW>vHZpE9l+Hj-3-@ulD%$sVb8~@X5lt_lmVBCGw4<#5kj;5*44^B8%`R;q;7?tO zE1vu^yoJLnaHBVLdF>*qWk8|)Bd}iLAOF^Pu0LrE3kG_%8}oCc`IdBpQ$Ln>hEu^7 zcfc!cE`0=~g{fjAp65bANUc%wa(84aC8xE?c~T`K7jnM|&X%%j-L4o7Kj0!OnUpAc z1T{ULqSSwhL?ya62|no!oIo&h-I{kMT1)Z=u7-`oMRb7)J|{_cF4eGBg*c0xE9Hw{ z-2;Ma1cAn2E2&qjlsghV;Fg7XYOx7NYUg;d^hB=IgqWisbsfCRJjD&rqeAAC*L_E6 z#DsqH@%j8x+bP8hk(SRzxtUKm4}`n%nJC^5xU=vzIUMWu^!aGUFaXJ7{9!s=( z)@>8=6#1GSwj~kL?aK}dCJQ?GO+81ZXSn9{n9De4VgYFk|6BYQBj}QI-q^FMTEh1) zm7Z8T;I$)x|>G?8x_FLG#ZVG8P0a~RcE>@S9W?-%i*5WcS9FR4ai-W4;M5{yx) zgMb(0I}&C_cW6(!ZsIC&3y0|jDI6bq7^8dNT9nM?!l ztO;eJERgNF(0H{0S+U_~PIkDm#BzVUepSw8(YM^g-?0fQ9-5?4F+X*h@B@RE=+Iqn zc;;?Q%jMje;v+BUpRui;s~_2G_ZOYMl&=B^oU;+%cQJ^V>2+WTX?IDZ3BiT?nf`sn zXO;MvuZul{5CGOTU!B!Qg)JAS;FJJe{i^D_bH1=Y0#>d(Nt@J(=Z(efDDQk&au~#x zsgW4w1`vq^z23QNo{rcYWihAcau$&Odqo^4kq2IwD}E{Nwbd*%ObYJDo9seu*?JRL zpCs3apy27<_~fw6(6somIT(ELQP*e3=>frJ9|TO&Nv>$*ze+oc!V% z+M6_^b1uWhiP)Oa+?#7ZSJN%`O14Te`V3De-Bd|k8@5gdmgJ3YQyeC`2qMaeJe3jZ z{6-ZXti)$JlYQg$+JCFA7}-}@+{e~mvmd9_ya7u&;UKLeen8_q?oSF|+A;?Im%4bW zw@_oqUMF+k54hOBgB=gOvE=25)GK-;Lli}OQwjSksN-KSQ&}a+ zW(I&Rcz@=lSX84D8f*4mq@Vr^PPz6;*+K`~{5J7?ufALan5%7dV3LXQZMe`+hM!-w z*uKdlv2kQy@_at-*mQ3(oDM$tt`KN;*h3>3LVXmX3D8Q^bEm79Hw@N6ed60n5l_kb z)rRAS^@AjUd3_k~R3bmC^uuO@9aIo|)==aHL8E8BqR(-@-P!x*0L1l0DJ_GKjF$s) z?>1JMP7zjl|C4*diyzkE0XXjCs}uJha`qWHqEVjLW*yfYke0oy)n^+JwrhF4lb7~x zAwJoDtq{dD#ac<7v#qn0JH`Dp>0kIY&0iQzIbJ%vww)`s zT3UW&GijB^f~h3T*}QH$cC;4ezp?Noj^VGh$aSiJ*GNSZc=ppg!BTH#6kgEuY4!Q> z5nqUfsyExWZ|CC@sOL$dR8%%AzU)(EjI8lcg)p(?!2&ycWUIcd3G8Uj z^4Ikq6GyrB0jZ0~Gb^^-+z#2Q1a=x>CL&pkf*>;2VUoTl9ha}I) zgxUJ&sR$jv33rA^+%W;j@i0WoB(ui)t{d7%zt+x}&jzA{S1z=@PFkcy1MQjSaRUN` zZlmKSZB>QF!fvh{hz<)CG=MQvTkx0466uthiTA|d^xM$D-z%nx9976zzs_q5AwPYNMzH;ad6owZ2 z>UR%+hl%A9xi#LcK(-dO3a!W3G1ArpqTsi6bAwpm+Q`IwGv=Cq(lH$%5*rQ4bnr=N zvkU~vX_k0@U*>8=MmUTSwo%i94Gbp`<2z_~LlAPXwblPF;l;Hw?#c4N|37t8{Mzm` zLOF$iVu06DgPM7Y=(1^F1MS}9CetT)7`5E@aD(pzA!b${CB0i2l83STR>v#V_7 z=Kp}CSSk-xH7NO>7q0`w?DXzXx6A!DA;Hoh5mf-mYFpU;2Dxb3rdvi{q$D=Ad$sXI zf@DM&Dee|_05EUwBV43R^>O^1Xwimi2%KE|ylxPF=3+h6V@*2k8wSzDR&f{{(97zP zPj@GJ&*EYFQFinrsYo3s>7xbrLQIc(;aSc>{FltMgmbGjeRrV`Z8Z|1NOe=4KD7#I zcM1lA#XvONLf@h=X(@_maz-tXF8q+_`YM>;29k}TmYH7QD6RB?R|a(a%CuD)?*%!eYJ#)FHP%U|J&l;o24eP)xJpYLSx#Qu2x}5{uhR5Hk^a=7_fpiN-Ugfeb@-SbSczR zU$mt7Ba(R(fd*myy#<%cvteHXo$Dv45xDEwm%qas3KzJSjj+myAJ>oPc;8}FWK%Og?( zyKyn-pOJT2t3UqH$#E;Zd@}T^L4(xiK=cDf;x`ew%zl-nU^GOR0YJs2LPl$+pVpGj zzLhb?770$9p=0V$KqmhRiG#~I)x|O<)!D%K&#cJP&6i0ZwoA5B0-{{vxoN)vs&Uxz8lq4T}{ zI#L)WR$%ib36&O$KkJ2um9BY9eB`npUB!g!fyOHi3*b@kM&Tz*x6&V&2Dh>JR_mJp zD!L?%?eD+w^6??p!kY2Z2Z_N!N4q1O_V5!2b@{bul*7D1M18l<)1>%qj9Hxoecy)4 zF-fI@CSahw24J-Dd(v%^wt^*qu8B8iTT9d3&7`d*O0DJVFLv3PnZ<;A`d*vrIyc-j z^Sjj1XwfURMr4~HzMT2;$hi>xp3#3`f?*Rj3^7GHOP7VYuXd!*m`&qZaH=8zBC?q9 z{_b2p1u;-L5>T1AV$+#%s{{!3nfNB;T5uPzDQ4eM&omF3c5#l8M_RReU?AD&mSxLoeS^Yg$y zIIa9-q&v+?h5FM%Z+T8xyC-l0q`o4A*gwOzGd%Q7DGGmNw>|uT_ z`vh4{UkT#nmO?F9Ae#6iWB$5Nd1ixlvO3o2On*a!UhmASri}lnVX2PZbkf@ix#VSw z42ib=O6|wD&%*i4K+VlQQrWWsjF}2aF82q^O~?z@DWeW3M=;g7DEZO8lsVaLIZ8x5 zv3P4Ww#PPY`&|)mUt<1>;t9(dCkBIZ3$iM$7y_@)?MlpyJ~8;|hFTGtn2D#b^Ha2k zlx_EEQHhTNu-f`9V;36R!}2+F9LEQl)H~uaIistUenuHUMa2gh5hI?A_4rwue!!Z5+UBz$bm{2BROIL)1ODQFj7yd-+pI*;(HTv*UR(Z37Rs82KGrEm%z{?i zU64s0qv}`Kq%?YGz`z(TurfIG?ye%Xl_2fybwMK-@(c8t7w(Y_oKig$_I`Q!;}&09 zA^UnaO*C!Zr)T4>xod$ZJ4DmKNhw9SDv?ezx0)(Us{+16cRtdBnmkPbkPWH7HGjA$ zt;G&28q0yQ64FG670%I~0%WS(nlx#sJA^TJsjIdJtR;>9L!q2%ngKB-7Kg_F)>M4s zrLr`BmKeSLd)FlbF;Y}f2yH>L0{g%QAu|AP=B-1ITZe)eu!tcYsVos1P_SBv;)iH^`bRdc; zkh*-!x#&^GY>G>!x0v8}rX~kF%@kXL*~$oV35Vk<5FCX`i`p*O`b0u|E_6qKPTc|fqL+pePZ zbw8}`l6%ste=|-CZt^pma4Yh(9qo2?CeDo z^jvhEA_FYPqldJlNObO%mOEJgk?R5fTLAdXtU0|3`hIC)d|3u(x0-I*|6_;jH|a+PNDssbNR zw8t^BQgUt4Cl}A8@H^WbsYnL0O%Te5b(|^=+@Rv&d%apj-Ydq33f`m?oHmI;0v0h# z_uw^Q=KE_k(Z#I*b@PH}vQQnoG^p{Kb91ZRf`dgEJO4LH70!78rBf-rU%P54egE8K zNx|=Lv#&PP$L?N>ul^lRA|4ZI?c-YIIG_i;c47SGuO<_@dtTIHxXV-5 z6FlRcsTK(RsI<;~%}@vN0^GPb9I9k4De4R71s@I3zLxd)n;YK2=218ytEw5Z`X-Y| z+9&{SZ*R#~yT5EDp_pe;AoIGM*B+h{;f&}fWl*MC9PDhz;o597w6b8{OZa8LLdijC zi!QJlv=pX#if`~*kP)I`0ol(pZO*->u8d!>|4?bG0uism>J*$1&ycFGVc;*(u}2B0 zilIcaW41P5&1qZOJ%lN}yFnUl=fCAp%*VJ#cs5q^I?`L2Tc{K9Bwa2J zJFu%uwo7No&NZkM>o%;f&nAJ8#Z+l0_3UFDwtHqY3$(;v`5PQl@i82d@EGJ$rExhz zuGLQWgVs5Bfk)5-zr;DBbpRKBBtaG{nGoGtXlJxa zL+sN`=V7ba0DaAvd!d+fg!^vMZ`(K!88o#C1+u30|p|1?m;0IJGYM zCL9JzeqYN6J7xW44kCUo13zSU5}<+sxxg#m5L5V8Ce85T6*PW^Dey#{8qp`ww9jJf zTAu;*4W;wrrY4kVQ5%LImj%xIY(+K(VjVWmuy|0v?m!Nv7&ea*N=b4cH#!?1pdfVZ z`FE%uPy+n6Q#k8O%U}md(nuuvaZjUN{()fAvGjDEb1mW=fdPtJjxSD= z@0mdm?!eY&2$hy2AI-72-?%=&7KfUu+C2?{b`U)c2H5xO#S*EeTre1YGEAqBoKEK2 zMl2&@a0l09qdc!xy(tykk@r51bnoU3AtqaCwRV+mE=?ix>ko{7W{J#Kn(g&N1RHvL z@CpznQ+Y3$5A2q^bc1yo^M-9FPtepeRVM=n0*D=voe*gFeKBK2s9@}d`;^XscxJ0O z_wOWe^|Gy#lFhfqUo+882i4{-NlzbqVopnyEkO{-yRq3Ao5h1L*Kak$H4a$`S48yn z*jrxM&W$_0(rO%`bG&>6&DV8SJqBY>MoE>=YlzUfIU|!DX7NKgN!s0}6D15iW;gTO zXrKRJDq0wG!odxkuw8cH3Tv`h?&5X}3lPLap_o82ajRII1o4f?{R$EQbreu@7WlHz z)K7ui3$vq7GTAklaWpIcOL)ONRnUY~a<}kQoBm5B&yjPWV=FSAI=Fa}bj)c0)~Nkw z-DW<+Wbe@yqvOqk2>ocD;0m6%>aD$t7vee$(tjAw`T#-_+d&2KEXpydpnuE$0u>La zWuh)@2ZgAJqL1`+u&IGmF%qk;5M!=qXyNS2YcS>4KRThF9-4t_h4jp2^Uyo51Dm{v zBMFzGg zZtPJi1w8<6Awp^ZCz{km2pThUgplS!b75$s`71ISL{>lzUGhoYu}{6ZCPrH$j~Zrj zJ}rT6Obk>(pVG-Fdz-GT$gjn+(3J<`O;!zq!Ptmh3fP>Z^Gkv=9YfX;(}>C9)0f;2 zEn8K)hek!m&)*H*{^+?k6THwl9DcyG%F!>O?3>u$12;H<028uxKL~ zAmEjYXLAkyD^tluxV}uH-H_GG*4fpygSp1-1(luiT9mQswuU+xME5N`@+^JA`uSyK6EQP1L_?R3WkQ9qJw_| z{*cHMHsuhQyiAaa+EKVOBfux_g>>#EZ89g2uQKbyhvYw*lp<2KX-YeS7??;U0G;pY zw+=@)DFqJF5~-)Mp7yMKfb! zOOez7CW<$_`NSgT_00XT^=ddZ(tb3f$tV}3U8`|cy@OmqB*1*K?Z7dE5aQ|%&d#=s z=uQrPSZP;4b{Fv^mEEg`aTq) z|K)*Z3%voYj?{)n%neQEQvyAvmxS2S#!SFS4@(AS@e2eB&t9&}UCZ`Brcw>0rFajA zRp%bb(*ymx$2-nA_IJqPXcrv*UH{;;UzQF@tDKAu1Vs;L>H2-}1K7FssoJ=D-^W-M zia{XhQwEzXeiqsWSs|L0#S~5pzGZJY_fK2o$g0%=thfU7@`yAZs#<3^{$Zuk;)uf- z9*hr2C#)T6c z=p0;)fr}6y!QkG%KA#s0OcZ1$+iIL!@c;>NhuP7gbw#8&UV#&*)$-y574itRwyT}$ zU0{>KcqH4)hO}kPx#_6%f#Am}YhUh|q6gfx&dHT~8N_U;?=X(rCG?+T*|IMa%D#d| zU)hm^an*G~7TyEUOV|> z5|PND;j8%mlTo^%;H#*LI5iZ@*=vX7=8}ygPVwV2nrZCQ8tgK&_aw;_2_(Fiq0<^2 z>p(1!(l`w;0VOj&1g76Lqr7H_GC0=l-a^ozZ%58o-jR2~NlgBT(vOh5e*a$0x*R** z{%>6xq+&Rid+m;z-#ehMy8Dak9U!{q!qvDuiw*9 z{=?l&9~W5zt|&;i!%+RD&Ky1?CKW9+5%N_crR7_XBI1CnrY$K}VEr%|`X%h{ zRraipJ|bHh1^=2lds{yChwKtQqXoCh7~SmS{_%4E9r5)_(d%0oJ7fQDAtg{V9EL$% z2Uk?XpIqDv4*`bd^Fs875Jt(R%x!=pLdMx8hZSL0nv$x(KzbzXEN_xQ+HRv(m6X?& z`(i!7d7B}1{BIBwFnU>W+w31~M`W-IdKzvXHd7wGR}NlC6=B$aM~@;j-_LK(e2fxG z_;iAp(2!ox2mO%tsUv{Q2p0JKn&T`cH{M{2mI4uwU_yud$JO!eP@nFKEWDB*fFU4;RZ zaVnp%AT_ws1ZLgd`=1khBCAGrx1r>O+|?6Ef*~|he}7xKri0qk=;2Y)=MBdR6XfgN zo5hbmDU=P$J#BE8$wF?`u_D)5CeP}L{rUjBZ4KsC%%ArGwndmy-!+oOu`&$||x~?Rz&aZD;)c zrsGfJRKnY_Vww^bL(PrXD4cNIpt=ZONXj$SIib0c7*tp{I+Qerc#33ay+Xtx5YRwG zREokjuhkj>Ur`%|v*8*k2!&2w{Mw6QeT!j;gojXy2rI4}W7$|etm)yVScrKU4LSFJ z_&EMDBR7e=>&6Si1K{}Pm4i-|>+(wrh8lDyOu&T~-Tuv1fsd(y2*rramVGE{lG<0) zDZquHfF*Y|nuL^LOnkhn>{R1yR9*CTw|+;yMQA2kSxtxj{LxrIwal|VbP1eK z>JOM2msi+bUa>QU;$5H-J~pHW5Qf11P^4kba9(v9Up4sTud0eV6`a1xjU9_Qw;uMI z!sx#P{!U~=*W4RJJx+vC+bQo1VlZIb;nR#p$H+IY;6o#q5OekS&wH@2d;9FsUbVP9 zpo?qm&}%My7Q9|rdB_2%)B&$jLY<4t z9a;`SkJ>=Tg6a5#|JU|Xwgi^wLWF16A?Dk>^8%s0l`rj0luHdiL?~&(_Hclb*>8;- zi-VF`+{1i~okNf=Tohf~wr$(CZN0Xw*L-c;w(YNN+qP}n`7=mWlFA_STj!4Vu5;D` zA>dKyc_?8ABEw}2hb#k?n49zEi01yub|Zo;BEKU(E=}o%D07GH?Gx_O@~R~#x)IT? zc4ji+>+8KHucF?n$a;b3Vex^ILHXwkW06{vd1q!TN3tsR{?1&2hnwB6dp_)IV^68g z3rC(n`y3sAfjhHI-wo6GJU4_8Vb)Hwq=H>7Y%9M*1`8A^wYT?(WtsX1H`6m!;oe$P zTQ*D=u@^|!R}nA%c=&PncVyt4%ueXUCX$1a1dRdj?;EAV^`EVQh!#2hg;HuF(Og{d zyP|T)Fzu($XJ=sxP4CyTacue_xTUqB3`8O0oI-(f!zdE3ZvF{I1&jD zIAE}trh_u>xJ2!)9|(S~@E1lbH2=aKb!+N>7ie!1hX;N@?*ot(|1Sd`*Z;QxkD2p- zH{h|dvM~QI?cM)jz+>ZJ{=W=(oz5=Bn;rakRlMh1=K!Anxj{|aVpkOuYln-!*)m_x z+v0ofbHnYm`?&4bn~3Xvy9DupKxm)e9Ff5u|3xZ3H?lIlGc~kyuLv(; zeIc0gV+uP5MD_$vZ>x((HwE;uA^`-AV};`%9Wxm=4<}@90?!1Ld7*;?Y8wqKBgF3C zPzVSl5mFP@_@j`JeNqG{Osc7g`Tli8R#Q{KC;@@GpQM5Y^`}t-QZ^GW5fn`EUm^UG zF9%|gA^Z{a5dirumW8Fgr>&TTUR?b71L{w(2iN4z^tt%b+m_(X1o|a^O}f0cGjdS} z0mPyzDXD{?t&NF^iGxZBDT#yYg^Hsy+51MFk;x6DYkNfxL|FGCB)exY!1V{$XYAY( z=1Vg`&Y4ahav^N{UT{bt*zrqs@xfpp2$JZ|ubFw-{f$;r zMnM6_u*AXq8meifmE#M_Wy#&e3Fz$G?DGu*C*dcG2#M&-)Zw=*?5S7p<=5)b;R1w% zkan@*>2AUO{T6?*)y>KAH;m)gYdw@>1;HF47U8!N3Jfdj56CaT{jJf$e8q3Q_}y$s zJQzKtyhH$EHdA}!L=@i!M^baNxA1rA5Hs-uOI(8k2ox05|Hecrq$UQ;uDV)4i?{DP zL2>L#lu{Dci*w_bzy{U^7t<#0H*S4xYGdpz?nXkPrD3;q#92nOIFK{>C z&-vRfRBSG~p{e=V8}kXFB=}9;#lgx52hfXOhG6>F=CdHE(90?_`7c+I1jkCMJrHR= zGhR(}%;6|k{P(Z(#7|XPcv4A8bPsy`RWI82Et3x|t8A+u#E+&w%^jjD3pTD7tWeLieo{_@dp!A!CL%A!B!0a%-n?g zUT?xhOG_ho7e^q#-0v}nujQSAi(v4tD27lK88t0YvCRAKA)^`EKOh4< z-@&2o7DDos84PY}3JCEd&WZ!1qv8v) z-cN1*zXza}@V`Ong4Q@2TE5}$4zT>&{~1eM-}+aN2=9VI&;5uN z!=Os-TT1)AAS#lQIvak?LfG#q}Kzv0dGKalJnh}JE?{)D<$ zzrcbxykFfnh-E<;m%GCPY)*hBccyR2fw1scKT7W>H@0TTpY6E!pzNZkX+abLYp(w$ z(@nqs_h;?o?zcepjnJo`+3)Rp@~1QVuj}pHH$UL_aq0t>mS_>KG9-T^F>PdkTX@6kUJ&pO9f!Q3{a!{tzkmYAM%(u9{IDNll7%@B2?L!EtmifUA3IX zLTOo$0Xhu~W}SJ3>((~}FNg|VuWPcjy^)rP4O)&fXfMo}EuZx+fwze|DqCVEdv+SO zb*Py{m1A~?*i_!@avWE18{#i$vUrYFpW~mbjI4t{ZBP>SHvxk;@f3F@ zm@G^g&`I%MIlcQk-D6gtJDFuB>}}V+BFltr19okZo~^wki1t&6O&uKuA(f?I%#Y&G zJzO*HwsIlHRNol%0ekw^JZ<4Nrw%9=u@`Z|iv#QhWq^xi&8RE>@U~1L*tI)Wd-Z=I zlV8miVa~jNZ=Z@G!ALJ>BPizFV@t*%eiLJ_kvS-rlNF)1Y;d#5d0l4`la1r50=a=} zWg_k+g>~eTN#l#RB2W3y?Ti-Sb5A#{q@_wS?Bzt`E4(%wZ%^ydMyvzRU>)?ta9)Lv zzeSKwjd1Os-xxX-jo55WY?+g6vWH6_K>q}S3a8DLkj9SCf8b9_%&6zMJr_!xR|(=* zilmj9s=Bdfc7T(W;9Gk3rq}NZ!BW#NZ(@vY*eOsZey0~+bUPU|y~Pxj?2xZ6WY0bqsO3i?<+aw!K(e{V#6Sfq;$dNbQXs->cWlr1#MFd1ZW($@@IDJ zKI4#$@ST;ei0i(uol(f*O;#s>M=FV`A{)@9=hm$@DzFX_sTQ6>A@F6QX8A$xJZarD zM2F7R9LiksZ3|>XvG_t1KJfL=M0o6mC1O?8o`=I6f2=!Czs&P}ZSC#$rIH|E{DUre ze&0{pNjce@z=>4}EqREal-n;Po54;JfFEq;>U z(&jIvVEV1s7nk+#bzA-RSWg}(7i<>bNqt7~M%q~;Kb;uz=St(Nu`p-})bKsgSyM^b zpr=Z@$91m2p|0D_uY!~yY;;>U?MtAZZ7cEkW2cpMO_{R^G%d7Sb(=8=jG}yQRjpwa zX=O3nnM1wGFksqm+9G={9A^R2=>8UVKx;yuBX>GguC0E?1T!D&jduZ_=AfU{KRIfk zQ>0NB#X7ZcA7y4s-oFuu0nliIMMhRkMc_N}70B^8dyE99b;)iZHDk|ucqVl(6$lDp z#2^zK?Oyc8jH7}jLXO{^<3v^P1ZQm^>^`EXSDB?rcr@+__fn~X|9j6(%O77AK!A&9 z=)^s_hwfY{%_5HXC-mFMllQj<*nqb?G_=9pX(Mha>iH-O#Cml8?hf9J*rf!f&vCND zn`6rVrSbgkM%HCG8(X|}b@=+3Rsbw>1&DUJPtk9yq^4QGQMe)nlkQwo10(cxjgO^D*c8AHr2S{DWp2BW6y27G z#emG#p~UL5lFth8mh#tV@V3*`DPP|4GHioe}3&xpkrNy;YosHB~ zVm>abWeSEN$(r*Kz>s3fM)s24#*u?KShI#`xgM2pu05!<;Imn)go@ix;`y#Ic%>R@ zt}NBBm&U~Ncz`V~My5=w+W^8Uhh#UWEAubDGa7t$bacWUo^RJ1vA4aVw0qvF6a)Am zyk~`;DNSKw_EZ+!m|Q{vpPzj@y80gmP(}r*Gx-(Ue3;<(&(pL{ZS_KV+OMS(FoLO+gc_4zO7>;kn}d*!g%$qxCiEM&KOQ&ho<808%l@=@ z#y?kSI5YG82RD1rE{fJI&#DGaFTB-|Z;&ANp~flI;UAV-VlZ)trV13QWYiP62Cmq7 zN0*j_8x+zX2Ya(g))A>oNso&cwHUsSUW#+JUvTX_rQa+-K#qPXi(lo>G4_bHQb4E4 zq2FhD!D4R;*%_2*%Qf%)&fKkYDwn6BVwd_dA!yVnVQY`D3oY{iBcmMSy-jN>vV_#bSFMp2(JgQpapQj2;;XCTP^}*+<32sywI{k{%zgq{Ah7*>>jUGCwG~* zdBR6CRzcbciP1qh4{(!tOQxH^1ga?VPZ%t}Z((b5IQiJ4@6DCFzb=n)u>A|^|IdrS ze$7s-lh`YQ+Ng-n>7M00l&X%=RxKOhSf}f|L`RKMgrhbZPbB4{%s4d@LX!=T_43%D zb$BOhO250@{Rb@Y6|Of9C$8Pn)G4+4nu}-A(DJpmp%yW;w*d=}qC%mV>2J|l*+0O) zq};xnP#(F`p^34ZO;Y1rI~109Muxs_F{;Q>Y%*>LJMhduyON81V_zM= zb3c8xw+%y9J@l2-+2K@c1m_Tqjhh$wizG=1?T*77$lP!o{FPyislfi(AzU|ur2IU8 z=?(mBz5mN-f9|?1yi)rbGOa#j1V0-eRezw9M5lO$4tG*Vybl9r4QBff9S|0VlrAk_!B0r`cfNiNz$tJ zi%2(p=J!&3Hj&*rNj^fV?uiz^B-@2;(Ejsg7Z;LF{$*@E zckw-FU9;g$IvwAtU*c`B#`n8w?di;{aQhNl@R zI7%RGMWxfr@<#_gklN<9-_$!++6)z=0x~J74pvU%v3CA57_I0Vc5w$QeKXrV+N0q7 z1rkJGhx=oXkWvkpDbw}2a=+-Q!Sp?1%&=EB3~vhk{!!60#vWd;Ft2C>Gs#a80;Xi7 z^~L0@ugDEpnyT(+L~I{ z?^l7ED+$|Aj~THbLLSl}+{d;B88ntpL~_i4e+I`MJPU~*Qko0DCEfsx_vSJ4M8`Ml zR?Jg(5_M&<1yQ2rQYM((f|m|x{o8BwwUHiu5vN3$6kz90fc>wrrH}cXNPg#l*?p>B zYJGQ|o)%Yz)=1uBU{Vs${CDmHnRoW)Zk(8JQsp}0=o)Qf!R9jO3Q;Gc|AO3Op^yW< z2884^FAWokm`Ang&Xq=O*lZkyPO#cGz@DsIvle<;#?qY0%^8sE7mU7hCN? zQqSr{28G=-M6@vfLXnctQu}hmC2!o;3yft)ts$en05;*~^0%p>ji=X^ zWgZXq-jm}wxTm`>Plio^V175H(R@Gfye^u+_zem zx+xMGRLzO@CJL*G4=l<60nd`4QRLPWUtZ1bO?vzsaqT=sn10&Nvw-}T}VIDbVWFbbj(-AUw1~~ zx94tJpKyl3=jKj>=sIIK62ewk$I2uR={~ACoD1))hQJs+Is-}V^xrhG=Sl7J6YWtP z+7{s!xAp2#xPz~Dg$sTXXastb7{uO)X5Kwzaox1q)j&A9kS#4kCnQQt-!yG+!m`f^ zSyfMheT;=JAN0p8Soi}R!#p;mvTsS1)JiD>L zAS6rwd}CqNH4$Z8&xFCB7KGfkKf*E<>qi^2Ht9s8ezYy7u4Mb_Zh)SxF>U2nL3$Le z$b-fnaO^U@V<{n1)gvmZbQoaMSD+vmYtUY$#0;WN1r$O_VC|)1-FB;P*Rnan6!F_RezG<9a%zT*M=={DM-b{a04^8CD~A%E6k@9D9Za!C zQ!)_d?;@dl$Hr)k&7y@QTp`P}p5{MbM7dhBDmgo62r*q#WR<~8Emj2p(-#L_m2=X+ ze)3^^O5|(lu7QQfWO3)0ovtHX^NqmV*y z0!B_{i+0GE0rf|tLY0oXtr`SpAG3`~7GAtF3ap~OyQoEaVZvl4(>8aFTvewHXp`-T zd*ZUnk>G4&_>(bgp*-8LrOy!YcDy6XwI19=0lg@s;>&q3@bT7k+1Tf0WT~)t1-Zqr zP0UWuKMOFZ8#Mod+!gawEt??MNf;IONZgCKZWY+VYv=>4Rfkj?zU(^ofkg0@y`_I_ z_s$hcJ~W!M#@w-z88I2$PfU;wBy^89lp*oarSHczITlOvyv`wK&D}4<&bJD&+2d5K z@Q;ssqkIMf+lW~wVj@>jg(n1|kj-JKD!Si!RBuJ`0+42538uAKEIU^a2+&^n|Fy^$ ze_q_UeUYC!x~ms=YvtIHY;h0;2p5xdsB1&cXiGuQk8B_abCiK)>I!K{<)gRa|5*Se zYUP~+yR6_!&!}Sfdvd4@k*sA>x=O+Uv5KRIredv!>=>~&CB3(my#A%x!{#)oUV2;c z-JP{$tP&XPrxmKF@0EjDzAMrh2VMkMs1=TRpV9&t1a}y5vpSJJ!|9z@so9eE!%)qQ zeywt+i*H}R@f8@iat#pVYfFg!GC$fRKcTZE;+JEmZs7JBD1N6&S6=r-*}Rmskrxw5 z`hz)T+b|>N?-%nqC*>+r5x|3{FImHlpTw1*jFRbHpx^g`8_DRguzZLyPDz zgGxQq?`L*eIxq6FKVK|L!`k%-8u)|11#{ubtHmBZqm{!k7IQjf&-|D~qe<=Nef;9X zeReiaqC8i@BWe!e^K;j`iq!JGGe0n2c}(%7JliP_Mb@}^j@8FL(`{WhEngq$IvnQ@ z8#Ozrn`gwq#+y>))hh`6pwBUOaenosA<<<7Piuk}6jdxb6LPk(YCM&G2f4y?$S$GH zYq-~HpA@MzW(r;K7oU@QH<7O@57LS-nngimOGsduOVLg&ZD;~`bhmQHpQ5X*;}A&cXHe*}&4IUST;qPM z)*xgy--h#X2~hA05}3*qy{?7&V^8#bff>1c&s_1V`_wAOqEY1%y86)L`86Fz!D64Y=tI$`PlFgrsLD`;}O}yPXZtmceXQ{VFiQm zLVeel=nH(aJfXqK<^VkM)&UiMn9M~pzc8qEun2G`X-qxCr+gS~$I*R<46E@;LT0%A z6EWT|r?M}2&Sn(WvC)Qwn3oW%MHK&#OcuC&5C*4}C%?z{JG1 z7t0k?U?K_mCxw$!#%Y7BylH1nBpVy#9-~l!8{7Vb^~Q}f_)54IPSWTe!ka4`tTlyg zZO&%35r++XE5fK~%DX;t;)V{(zx!-SVp103`?#`0iFB2`v+JH`X@nrKgGt@+9@yyH zNM5>Uxui=Fz~DkZqonwK8P9;|ZmKPW);+R(@kFXExf}b$ zE|U|lbDwUz`k!J|;@3XV#qm7Uy9rkQHEgdhk$+B;BvCp0A$SH!_EjoQh@WR)|H#;V zP@`3{Den4w1Yh(c&SZ@I_gh=T$HnCYdVV7$^eMO7mHSYpI;!^(T}iTYHMjmOlAr$! z;S7$XXuiI&ocwzL1fKnUqCB@EAAgmUjf7SKvq_Ov5!%T|hcoH5++JkLHygvNgS7OT z;6KF2g<~<+2e23>!sj0NbB>pm>ADgR4tz`tYo9pG`#tRS0imqfGl_Y%#VV)ZK}B&{ zdyc%1wJiMv^~0q}0%-S|qlb?iojlb*>%|qNYTxRK-A-u2#aEB3sl{>g9_DsCNo!CYH5+H8&#K8{~<;@!FjSWe>G<$nTVh)GmWlt80N9!F-M46T?C;T}s2$bG+DZ$z$#%&;wgdwFXww#t z&^VSYP3TGQTFtU{^}U`7EqQ)?AY%fVN2cmqM=Y?RRc^h%?HYY7D>$OX+^t|nSE+e9O{1H$AGlu-q!yWfh$weQ4D%ST5%EbktAE-a@t9#*t&mU#s%4=mGLGzzdU6OcPCmX11F58_b|_vkBLv%mR9uHVS*)HW5#`_L2< z(Qxn33BEoq5crZ-9lm{8 zHqB{K5@dwbFA5Px@zu0dQeNgnXq`!-shT)+tm0WuEM;OuY|z9x&U%-iu+3=Ysm_Aj zP`mzU5&wi4c9MH(czIi0g!GOKjDUgXXUqw3VMgj?NQ3qw^Xh_hVnUcT_*mwzCso$Z}Vu4<$; zS4cx08zxPmS4|-2y{&tN%(8PBmIuA*a&4A-9&UB^svede5+r9`ySBbztj`E_`jZvi*8+ zo;yC(v)tH-`W?3T_RR!uIuB8sU@qc&Ze@U$7vWsqt_n?pAW zLLD2E4%3}^29gY+xAhyy>c(6=I?EcI2cKG!rovip-hl*mHB3SWrFg%D1#Jte->hBR zt`D5j@>)FLMUskIlMA5^TIv3F>d~z|mQST1yp*w0FcpsgHsBVd0ItDk1QEUQhDLe3 zc19!7*x(t2dzamCyaL1jx)P;?1m-)`F>Q36MJ7`9dz_e zv)bTCxqw?}bA?KuFa;`(y8kfbW-wq$J@cm@C|^5I7RWvX(yfOyMLi}3o1bpYzRon3>3KzQi9rVzn$Vj}Br=DARPrvHZYuAR!Ct%;0M zevGKx?52lT3q-uB3_;{{9nOXM7Suhh^M}l)=fmg#AUVlQ^PAC`tbO zESZ}j6gRi93trDw{BBU<`~S`*L$bJo75iPZG^%YRq)&IaLcR@I#k0#_X{h1z87o2zGv9zklK$yn_$T6HIzz;hiZ+r6(&S_Z zyA1`e2FUGUSF(lTIedJ@%)=IF>7zO<4v=~$BxcoHoM&YCsRb+SZHMbAW0}CQrBHM^ z98Pq~$GIXjM%ACWw0(;y^CZr;^9aztV~q;*Fijw(FsQ?U1*g5KKcXLobbvk>qh-?S zUry;LM^!SVChR0?4dnE15`Ibx(}ZNBS>Pnh9x+667%mT6{IyPa2)RA1!R%};3OsnE z<@tVzbU5Io+-9A|5MyDYK})UafSD(gJenKKqMyCS)ABtl3}6stJ|hXhk3Jo2|E)_oXIUVj@HvQ#K# zYNH}C`rYrwMZ+1>7x*l2*rtHYimWGvuIs=0iS94D-!=ahnNqEA@UHn*Yv!fn>MJ_tI4 zDRwEBoj`lU4}l1ao1zs}Kg|5;L>K|^_&|A)|LWJuaD6M>f69|c6YYZlClHXX%;)wX#?qe!+CnoJ zzNxMQ&T83;8h~?!u8icn_^VRgeoV#Od9t&Rs;Q2(eOMz@Es1}{rEp&Zf`t{&)^Ot# zWbr625gQg{45BY-6Zp06$G8hGXLoY*QKRO1{g!IMo6mpS0-nMaS7A!D*&!Jt2km|5 zq8T|o@d{B^Xdc2db(b@g0b)4_ZJFa(45tk58@cQq_)5DZ+FvP~@QekUs+cg_V!HPF z*5VD&X00hf?yFs~fLA zV^ng13S6PY6U65ZvT$EMt8OU0IMF;jmUXTp#iOMq=^4@Iuh2?wj^%qqySpb-*IvQ5 z=nI9gY<+`4kin=R#>)#}_r@n`3HdFw*Rt0brNO&KOW{m}+-{7sG4qFRc;L9K!jWWU zps1>bpd_cgD|`(xVKnMVpgbakWJyU|jplCQ2!7TYDd~6CD>2L=uZt50ckHE9Ew53L zrxhm!*yyPX#jl(0a-T%hd3w`UMRO-q!ldKI}woahX_ zQxN9~ic18X*z#Do`S<$W%c1^5j^2Xc-=3kr&o6yy%pk8Ky(y~1=FPvFLNkmc*^$U* zISsbj{4G<2h$khQf^lwh)@p>AI8tx4oWzoEnN4o~sFLdEKDo8yvqMGN%(=Zg=OR~ z&Ju0)0Ep**uW%*&wU#X!l>i6Nja#2!?<R)?!)yWQ+4R z)1-qE>b_KEmXRwKwHk&{>0s8mTB1WqAD|qs(`kh-qC%6M+tz;Ck&SsX7Aw*)#e{~vX6OD}HCo{q42l_Wc*;<21R4_X`@5CkTK>Lq-(JI0Y-~gk zJfaR(wbos|JMw%0W3~<;5l*OZlPBGxM`PE%7tyAXoKHXH-$Gg%v2(Ou)%Ds#!TdKD z&e`xHiYNIRXjO=0{58F5aw`Adx+&Qo!ZY; zq9l)6oV22}g;;(u34R+3|&*p#bEIeKoblL8$STzxQex9u!;Pg25F%lS& zHUb#u5VQ4$p&Py}zKf0XJke4dD9bu9osZif$(20G{H}7$CIZ2|GFv1s3686%S~Qd` zN+lx4s5ctN({5fHY*AXy10ZMZNmk<^qr?^WYPU((h*ok&}a-F%= zY&o#2>*2*aMDCdXp44E|M&a^)RM@0_6WUkDdq4i-rE?7{amZXpVo^ph(Thfw@~TW3 ztK);f$YwK*hAq(`V3_Sa!|M^%##@8eFcz(jFe~&aOpvs00`KWT;*@}U;qibRDp`fz znZzbhi@W31aF?gx994Zf)G`8tL&Xn&t~276VOt07><>cYP1Sy_lM-8%VHlJ1gQU(E zHfnQgI~~TugNi(M%{KDt;k}o>_4=7KUtckc-G30D53VYM_imaty`a0jj2|RsHVs0i zy4%+3wGKyPn1n)h$jn*Cqy>NUxwngeo@9kpD0UKJWnM9?=pMvScnN%G}UHL`C7CJqZ5-Le4+=%3XK!b!zA+_7^cA z+2=o)_|GtOk+U{X1q?1h`~33^MO(Q?dJlMELq#*p3-dm%e=liJZpFmxfgk zA1lAdnBdP_($a^R&KQ?dR!N5e&rizejQ z+sVhBgaO2Byff(y1l2`GO38w^_q*)p?5orwPY^Ef5%I&E&=cA~1dhRhyHN>XikGVq zW9UFnWXnXRICT-(S!?0cet!igwf|Y0c(lc7$1% zH>FVMuQx)kVj(XvDa(hNeCzNp_xfOLj=h$v|Ne0&&IfuAwzLVOkpSVpd9u|eU_M7@19WOAmGeWo9?AsM%#fmE< zyv=Qv_mWC9&CaS%uj!wh$h6a`;|^(#RaKsRkzv8VKmk8R+mm@4zNNyLpYeHxdW6Ad2DrU< zT$kWWxkLC%%t{8UP}E{#W>`+4L8Cwl{9^QX!>6Mdyxov;tmyWKcPrgd|6oJND8+;3 zY8+NKH&{;nVXGpRCm4Kb<8;>V3{039VRmoqLyhFBHRolzjaV`eoz^VE5>)xFoRydM3z+4e%DF(ysZ3U(*BFeQ2D|D_$%WOaxGiXbHix;d8E_f zCLejY7Ur=bE6?oKt-TtDVgTT6UM}RU)fa&5Zd4?beO2RTDN>SdYU|q1`5j0;4{q4n zX3S{+2e$(dmlwdBnC0CP;GO@)28eS+qn1J(FcYAXR1FGC9#TaakAV(AZ(3Kl-+`rZ z*-{o?`+u$t?Z*zOuppCMP}o*2x$80rIHw|*I$NbRpyYNW%NKN6%zb_hqLai}Os?-%H=cz3p7!EC|EswPVdTiJU)f5Hq=)l} zdq6)TZeC-J-Ws44cLQx9bT9G}8A1pV*EVi%-&-;K{Swk8HjYJcK@oUu8rFDHHrC1d zeh_eR*g`AtcLr;4kj{I=25?j3wXo^zrmFPW)!0zFR9L_`+LJ|Llp<3B{Yus(AfGUG|1ueivV;--YQ7J*JdrIAw*11005>yE;6qmC^AL`S7 z+53*fT=U##yE`wuhX8K|pIer@X@%T0_;>5YTkFBF3Rw^RguU}_sO!vQboVUY0@fHHmN^m0M% zu#*g&V*b?sSCHj>vqYHZs~&1Fc=m3^BRD`)r)t}245cWq)ByX>>Rqdt*?%)5Zttl0rjUb zYF(;&o}Z}H9`-|o1auT3?Vcg4@LpP{;v3ayj2Ol$mU;KFmN1o7tylX8Wx^*ggO)OX z&6C-|R@9ztNRg(RF7`q6;z`1pp#-<|z1_!VYWyu<;4tlfqq zE54Gu+5J~3j-!l9V;6V2l(2)oteX5LRLSe2@KM6OD^gxo073-re%x2)>w9;@?PG3jic3N9I*@6{Z?Ev+XV0HMp;Ct}$ zi9Gon})0=v2KLIZcG)CDfG{x&RO=8@@(>s-~kW?GIUuXW;(0ab^U)&sj zw69ys&b>7kUoC0pIr0fWQswb2k=3Gn^8%YoJt@w3%SZOfkaOJ3WouR5+Pnk)t`zdo zoMIz4YCN=|o*@~GwqnU`U6&*?iv6D5;LFH~7Rk&hKYzXk3)-0@ZZw|wDg(-Jqz0*B z14eC{!GatZbf zTJLy-QYNfx4}8C5VuWuoQ&!r#WC<@9VoX{Yz7PvO2YB>1GFgh=G?iU3O4kZEWZ3rp z$!5rdRR{)LcY*9!q@hT_BI0wLN_fW@s7u8zhbZMQNN=!>(lDGL{} z%zHgRd`+FnxN=qW>~;$Y0uzc?Fwu6;a@bZ=&<~rxUPKYt^)W0LiU;NJ4`8Kw^g>K$ z%e&gU(?Q})!P_Hy+ZQ8V(*i*i#K?PgvOK}te4Tr%=AiTR`KdfRquqD5^dyBvHuENa7YOulbKQ;f(Cngfal!GwW!mqNX0Mk%ygy?wKEIDP^6XjQL}S z9yt3U3vyC2!dOBJ`4T5S6d4se_Ow& zz>pTH`R~tVIS^rv4h|>F^&tZnf*lwNWF>p7}49TbOV=SI#9U=*=s+`?>A*f_M#wN8QYAz=aYat zp5ONj!G3tL8CIe*$>Xu4VM~yE|4(A4k6TiEFyfmBQEH1O)Hi0bBOJ!zb|i^ApSgOh zb?1SZ!T*v)tmKqyP|q?I#M+953kWf+V_su`-XtFQkP5Z4?7BmS?r3*jnN4R=Jn1Zq z6V*=hP?hXmd}F`kYH5QD@!WwIuWKpV5;j01PZ$3%aMyQ_;59D;e0+F?3@@sdXu<)1 z^+I!+iYOx~g!?G}aEO-fmT?OhyXIE>%>xbXl(=lV(3)98L_hrHLRMq~oteQy%QK1P z&hSgbeoJ_pSy>QzMrAR9GyTsVcYE-03X->Ic4jT;=XOgnm;^BW?b~9OsMExaEZ!8| z?7wRugvSVxBiiW6{Ut}`NKt|*Sf5yX!&HBbsnSts1MqB+jb;gokFkdtG-+Suk0_Odv%LNca=wtrGFz1SI_n_Z1KU%s_ck{V z`ior|@kO>_0zNpukC`d#0z@8YeJsdJo>qX)8;6O zgGW&lhj$=M1@u7{>oEwdUuf8#%^aB(TuksZ$KWM}S$C4EVm zlwRCwY%nq|)!+q&BfDMtr3^gG=_LR$hTjYV5Gpt?f?`SDqSDvYf24CnE058VB*y2A zH~EFmzdO#P!4pPA=poxBFZ2(4-j$d?76W7+yNRUz6 z25xrhB-4L_-L3e%zz}ToAUCG=k?)+I!M~%BaG8@VKXJd>Yz@XG6h9A*SBBPBBqygD zWoXA&+W*2;AH)>1zL5L8E*bV()yO6A>d%=)SS(6fZ}j#`%V49kd7Hqz(@wcVRF6$ zSH+St!$l^ZO&w|3n3#u6n>^`2h5rKCLW7LmQSY;Sym=ynoH6dmck&v>3_)wN5yolt zM(};>;7vKWnAjZO9m@!4%vdj-dinSEEkZsut(l?V_;C}Vbrdx4&>LX3v9~=)NX4)ynQGCamSaQKK>$qAw1$X_a9=X5@r?;xOUIP_}Cb3{%@9%@6pmE_t( zAst>TCdxZ(l;yuVtGoa7>dMZnTr1Dy26}?cy<>!| zx;NgARRowcDA$UtoZ`6s)oV>fRV>^Igew?q902W+CGHw4Myne@5bHT|7vn4BodSIh zo}|?`uowof2^Kp1&y<8lFszCaNL4<3MbFoL4Q=>twP%Hg5y#}(7J^V2_F7|GokK!f z=GIh&=Kj9=*CEnsMePv9Pc3{ThDQidIU#dVi>gX-Ui^=gYoXyM6{!Yqtv8Kthd0i9 zra6#$NbA`%-=>mu+&M*T3-QqX#|P;;4!~2(&m2*_z;qG%$Yq_QT}{TwrwUPJAG23L zeA%vOo^wp>xV{Jgk^36KoWrXs@|Jn11QTtwcqx{=A^!VBP+>wdD35&esY*nu%Z;2; zrcl!8ALfllYu%f|?rRd!d&^}5t;>pg5V%wabqhfIn`C@(TI^rk7sHr(oZ~eL#?TP_ zGVhn4>L10+84dlEMSt6vb?@GP36^VY9QmFdtaVTO2Hnbor)5v@#-nS*++AOdYs<%F z+H~uo7g`OzaC=^ka=HoCa0N>Vf6-0tt=(8H8Ve!@KmS^>>bnNw6Pze=#BcOY5_-K% z)mORq457FV4t+^ShV-0_WUPH`3M|Q&r40u>5_z}j^!MZMyZ2`>M7br6wG2ndENAAbG_6pa zao1;A`c`EITBsjgmFbMhZDJ*@vjP10CEo+7w{AwIcV0R@e{-IsUWMx?H|)BAIW58q zaT#=bQrJ#jK6O1)tfl&d>t3^}OrNA)SkY)jnp6dhcMnm{;?t#3MV&PTR!zO_BV>N~`! z+>3OyN%N{p+i%u2#CQJ?>pzoIU=^VZCOR`P@yHu%=Rjz6B7sn5ak!Qg*GKI;m0W9S z0xns9rqvidyDRfqqQ!dm2SM7wVXzX#;kZ`sr<3Ips=M)iRJ9v#U2q55mpwL~fz}0T z@*~`Oz;OUKD}5uOPR<$p8O2v)|)1+?5{*)XsEuXD#P0W0uyq9 zagoEel>fYk-m08Chf#l3zPA9o?eM3o1gI#O5)!$&FlT!{nzc;imQI7UhY}>J5mw0^ z2Nb2&gVArY$^sh}g38w%ZZ9uxF%|k~&%w^2fz#dOrW@x(Y`p<(|kC(%UTX zKD~~{YaBRdDu$a{(Vi)JR5s1k7=^0whTx%p$Zg@o2D63jlk`BThEQNpG2_oCOf`Ab zS{g+2em!3$`q+72yC%vJhv|LZO^1(!tG0jS5}lHQOn^v^{+Y=$W4$@ z-cXlaHQ-#B4JHU@{&cY2Pf}hz{T$g|W*DR)bW!bVZswN!RSsg^B;FC}JB1}|;!wI- zVvD9JK!w9&M#O%5`geQ=@ZvN-Vj3yao4d#I%zbRvpEcJLh7v?C8iublGuIB!KI_kf z=}_1=F5_9b)GV{35692+hyrA{^3YGOfJ@0akzuyzG+*h<8Lw)quFG)Q0LRLuSwbG2 z0jv`Aq2z@&^IlwJX6<)X9Ix`~eHJ^+_v@(y*5SA~;`m1yW63)5Al5C-Z6RE{tCt~+ zXUMB3Qx`dfJ`L(?ogTjR1ObWi*E&WO<&FA)BzYoPG%l1|5nn&Z0-CRhJz+|Q*+IFx zGG6A4uFs{L&O4_*?K-7U_Uo3bj6O+Q#|+BY^hzY1-#~TwFS8dYk;8tXd7p!YkQC-1 z>_bq3o;dX~eE~J=#+-*gt5BaaOJJ5VbLID=qG|HC3%SO^7~ONio_<>ZFS`lj`IH(L zfY15>%T8fa=;r8|*v3b2H5`<^GDjYQQyChqpy5&hSeqbEqtbh$6WMI1;bD{rLrZb` zDjNfSS_~dYP3Wx?N^ll7HL*zqJI`z! z8E;<6G7TBKr(3DN5z+skw}k(4citLmO=B)9!WtXh{^icRV-`h{i(3{|sHusAb8Pkt z7L^$?J#w3t7X&sA3Fi13t9;`mAdsq3 zHlQxQt2tT#ZSxX4`Wqp9Sfil5O?V8$$D)|J26lhTT7>}W~d=f%HgI2 zzU44FyExD^n*88fDoc#zBqIw;y=G>H%4 zozK$anJ)0tG|hVEv0}Nri7z-Oz{4u5{O}v^I zO_x+SDZLQX3-C4fnpVV^;#?_}6b{0}h17Ucn6z-EuW;WXkeW4*heF{!cIXpuWLldP zc^V4?$%Bgact`y_6@#khDdk4%iB7bXEif~F96?7pXN;4Fd?8o+9a|WN@*Oams1vY< zmieoQ)IT;gYLi(n8Qb7^8;q8TpmBpX#D6=C6!f9S8QlU(_@;j&eU!Vo<@cBkZl$c% zyBj>ZhaR9!xo7y3A>%jDGpUK=A`EuX06gB^-D5%hM9@JtEm6CLXrNVu=CEfs`C2&z z=eqn1zYND|+O|PU{hDAtm_P#d=a(dVksenkY_y(X9(gN$)J+az>d{&#W!_9Cs=92K ze>Ri)MVaq)H`b9N|NOq}F&)iY5H~qF8k`L-vaPd+{@Lendo0vNcO%Vnl@m_Ki@1M% z1zI+`=1Xx9SA?5-h&Hd}tpNN$4TwlDz3G76E-jq{-2krt=40XZON5o3gm|dqrAT-T z%3r+bU?B~$MfJ9SPI$C4!$T#g*!_z((eZ1#?>;*QD0FfLT#<)}lsip1v*!YSZ*}(7 zO(O!Dll9^NK6J|w5-7BrNsJs~BTk3%1r+I%BLltal||$>87}Z^#lRy~oDmCc6fopU zZq+pdpu}2?P>7*D$j+E(XoVQ>8^JL*-v8>!UxUs*PGT{g!^EGp1vLAW!Zd7t$BUt1 zHwyUnaR$MH`y&04+6P}QX%EG>t(X6@mDm~y?pc9CdI{(UEXf-M3+_HS>);0BYGE4f;S{B~HqeRc(itEac9DVuDC5fEIH`jC)!)(_z{V z>RAM?api1k5@Rm3cvGz`p>P#Njy8~7kehZ2Qrq(1p=B^4Q1)9nm}f_?(QBZZQR|W6 zKg$v%(QXz_XkOP>(_a!eSCgsjpR`j8DF ziXicV-aA8HSV;;-EJKeedAEJ7Q`UTw=F^{hC1VZk;M9q=;D{lR6nh(Z;Tr7qG1k41 zTbLrIzN+^}wd}26&D=}YmA3NJ?IBxb6W^K;0z0@qkB$Yk_If@O=;vnQjS5mMhy+G( zogij)mLFLa`3msuL|7Q9MA4rjW{CTLsysrpZ_u?*12+>bfj)k0$)w!+7EYE<)y6e5 z8)>O%T?bf<|hkfZCor|Z_|Mfc+7&O@K zm-I&p*R^NK5H)fIimeF?(`#n+*8%*^JZ3FboBSQnOUBQQwtxGh6eEW{_=;7RK=Ztr zQuoq|rA7r)&W=rGeB(FAQJ(7+1UI)fiZpgBHrv~p8eQep^(=#X7A z8jT~B(@+Y93m2}b>|o5VuMB5GSV`(cSia!ms|PJz3X-5Q9o zAhw^-It@jja$C_}fI77r!Cet7vVOiPu zoATmoMN#Xt9q3|d0n^jc9KB~;&QM}PGt%9&+YVzxp-^WVcL?tTp7>H%MBpN}e>%48 z@q*L#y6R1?@1a>czSYP5>UOV|g*BIl{lNQ>k-{gVa8`+OS5zlkxHknf4*f?xGXT8DvNV=)Hm%ufO6#9`L z-cUV8A?lR{k32mFf=q}yZS`-C5!au5kbc}KWVr#*;AU`YH`YFy!6<+SMdqfM(lYUG z`k^ngw)Du`thH(ZhQ9o-jtzdtVPG1f3miNfB%BBPcJ<<^P7j!FWF2$;dT^)mbFC{I zwJI4;AFr06x%MI9NsVAttA5i0g_v{nvA#zLO+IC$pQ@4aDDb8r|yYmQn}9jW!=P12DPd1fs=t zg9%L%1%*=_Tf)~CHa<2q&+~i3G_S>MpaDdu)spm}N0D8B=0V)YowUXia`I+`kkdBC zd9doc+UZ*KW{7N9{K1N-8F3GJK^Y1#4aZG+@&xsYqTEuFO$#gmlz$+_?}7M;mYx$} z>$56yr3SZ`jfXhPfI)OvVpY7%g9&|Smd8V)9P!oe@v2gA3#cmeP-uk3?%GC*8Vc5o zWG5RH>h0)Tsndy8hl-Ov#j&djS}zIjNQK zBDZG=u8d#3^epo;F<*ia7=N_y3*xJpT(~ADCf=- zZF7q+)jX(1Co=fPN1uBJ29w9}7{y};bETNBjgz^;x?eqgN?0xB`ls5bzA^+evt zW0vb&&hz4hR3J`03a!+Vyz;r?=FVL>=+@ZJH9f5~!w`{n)72d|G53;tx^;|n2QJcT z&~Er5B1Tz8dg?;@Z8RV?>o^?VqPMqo7YqZ`^Lyd|_r126eQo_kLwj}@wQ;DO=b8FX zWJyc89ZN=2Czp!nL>QFc=o4 zydinrb2&8Welz8nHME$T5pdevak%W%HhDT5?tuPAx zRF?$qoM?7adAO$}Ne;ibMpzsQ(UA#QnMl#?_|MAT#))j8sVEwc z)3FXic@s2aWz3?ZvSmV9@OnwH(+AoL_Lbr^ex{+83)Si}zT^!{zS)}=Qi*xzn7G-{4|lJ*V?1ygi!|%~DGj@rJfw9s%;YVB0rT@}f(^d06 z)W3_E$}G3nea&$bSZysDBH|9WEBq|*DOQcLl9FmP)gqR%*~nvCDlkV6H93S-rq5FK z*-In^VSzhhD9lp>UpNjJ6J=~Og>4F}&Qp_&xeit$kqc4Bw7jS!Ezzmj1xJgQmBPYD zma(Xuc8)`l#aBbibp9-gZCgyEma3#JF{nZ&I z{FC3x@u#CM7bm@YxP2M-*l4sY+H0n~eB5ODD{&PcOk}{;90HlC{ZvmfQzS!^e=~Te zN}F@*gyNRjrCHEU8O7r=iyoUB9?HCM?t_(x)QvkM<77t#IlTv&M(dZ}N&oyK(sZPi zVtjr~ehbkHpkXFMTC<0yW-X?8ZVQqERjF0~$Jx_&;zL(>p-5CK3aHq&1PE_`F+Sgb z{i8)zud(&-P>og}$y`OS^H6jqte7hC4o8CLou+O2Pi4s54@WXFs$rhRHoq#TmQzAalYh2kxLZ|?K+Xb^wz*0v+M@_Qh zk+1cZ(}!e)haG;2Z3Ni6JJqxkGVrpSb$FrigsfuGZF%~ZdI6TD0@cDA7}=;=rp_0j z0(Bm%4Qp2=1yl`lNec8lGycj&wDcE+BF0qpakHzY5?|YEa?6qwSmZV$$sp;lMmnxd zNn0I9-#XS&jeUv0Kt87%j#NkT7@X1Nq@lS>V|y@I!h-z7hDI$fsrZcU8DO^8o>+$E zY=}xt=qYga8HQsi55G}^b!^Kzl0&MHx>wRN0U(4IE%0+DJH#S&$)Bjp1Yv*iVQ&D4Wa%+ zUwIt@g2uOKB9MD=>A)fu?~QH8+vC&c;n>Ks(s3Z<>U59qKt14R&W9ae_;|kpVk@}v6x5*246RWrt_WAoKNZxTbM^@Q-DU`GvZb078%-@(n@|( z2BvU>nG4C8y5CN#=T3-iHH}hUITcueTldI|Nc*+z|Fow}IqaLCVHHhc<}8rC<@c7R zL6_GaeQ;pmDf`iYxv?(_h+^zSGS<1JpRdz^47he2kwPe5!V6^8j4|KCnd61p9Oq1A z$WB=cy$>E2K3HW?>XCSQW{YcS*_1=gK9yBp_bQF~#k6@DFu9brfNzREj@;*O6d{xh zyS0OM>1`OxTAZT^QS;$ozjR%*S@?HxB$nTlo~K45>Sqr3Dy6P7(?oV(yFg6*}x+C!Dq?gmHJ)Y-Bq zjey#YQsYK}brV-QKz|=lxO#Sx;0w(cnbve27KR(3upkU(#<6eD({UdvX7gDBI>r$` z&aE~VTQ;zJRlUhwR}r&Tt9?nbEv{Upq5wemzTIgFEV9BdM#JFUy5P8_87gWFMknOt z($hCn8OoY`J;nsP&AvY3mA;H*@3*^NU5B1Ra*2C;IKYCM^jip(cEv-rN#2Nt{wJ?z zOWLrT0Eqw2DgfG+Jmf(I_nbO_&)thV8%{Rp9S~vK%8yXvBz*-NLM$waELQey&$Gw< zui(`QxFhv6f!TqaU06-KSQ$&ynaBCf02oSQ%ort8W&K->*x1KmZ0=VRcbSJH02;6g zw;1bBG0O8KR}BjxOSWWq?30e#yt2UY0sfU+Ca;Q86t0*V2RWL&Qj{%F21dd0a``-Z z&=<%oo%xDb1frVYuTGI3LR zHM!Q2hL!eqBQG|%{#v4pyh8!ylcQPMQ)=o~?sqx3_&KXpzv$BrAqaQLcRWZYFX;$Z zm{m6+*OKyXOjrM^22lVPYMqp{n(E@mZ$%oEr?f6ga?s?k`@M0gW`*o|-q8%xpHYMh zz@EL?N)N$nmk|P;*zjqAJCIvKx4{9+qA$6Wb`jZ=x27I5gSotNe@olyF8)p9Tnr%c z)x=1KCv&rNH!D7y9cAG74HJ)qYlSUUk0ME}P|GVj$FMGx_NdIn$~xy&JxOZuNDvH? zrM!X1H&^X<{Uk*=?d@YbLEzKB1bVKyCs8jM41*p;_iAF`?&SfbbMtCb>o4o8S5HV6 z)8GW(jzb48CC0^pG#Mkxy_)yb6w7?$nY&56e8IAYPa&lR^UNTk| zK;5N{Zvsh**Y-aoZ2;;-LhN63lyn1?Gh_lks;Nb{T3WvFOMhu|g5IaTtLJy-UGZfNx8U1U=Z)O={La53=;u-Ent(VJ-^7SM>fy%SlDej;Rsbww0XP z2IWZGpPO$u3d{Zy9><6KJ|$V$f~ob^O?Bs$H0h+3ZiT4uV=eWO8`Ru0>jh@*vANnp^12m>1}sut zDvizO3R5-v0xVP}S9QD`Nn2B%Nr77%u(r=TD!sTL&1m!dRitwH#XsGU@LtHV64yf> zKB0}s&bm{!s4>H#TJyVfDxf8~0?#nxk}_W9pb7V}#O@MRFlS-5wP zHrxh>lOJ>&%iuB8+fdsHCUOG4E^s?%2|yHg`hH^{1uvXp)VwM|0i7n z&i|b*0RcM$0|6f&l(UPIsi7^DN4BeqtbHK^LiZzeSGd$QVT>&jgDMK-BjItK?XpT` zcgdbh0#ZZ(&in1EeHcrnm3N;P-p&*czyH>!$Ii?`hqrU%on90=yUS^hQ{&a?4a(Nm zk*+IWhntT}job6_?HxhxN6kiu;+9#K&m!Al4!Qltg+tAD=(B&K>#eMJaeiEzuWPUQ zVZK?oacvR4C6S=N?MQH`?;rvnO-~po>9^C@oXd5v!mE-LKE(#tWvs>&!W6lnkzYca z{YFQgN+i1lOfs6Qa?E1%$i<67) zZ`31y>*?b89)ZZ1K{s0S=11mE_~`b0^LdR6s>6??W@%BA~tC$ySU z&2RrO{%~yW07{M}LDJ3J%OW=TUC5OVF0MNlO z9blMWSZpN$qkQp%3JwIsy6B*a9*Qbo>(5DUl*CP`7g|EUBbPKmWV&5)0qHd&0QH=B zz+*8=`xsA0`oa!}PF+h$9VYHU)|~VYOmJHJIM;tGC|iy&2o$Je9)~FESAx8&4q)V3 ztd!TIO|Mv-&7V~^6J1n)*vbu&kdLM|mq7)m>l3FCxpmjeAyKwL8FV?P(A3cOkNqJ5@Tlt1;0TS&YfY z4&eT!e)1?-HsgD{3*{Ac2H3Z3ul4$-mD7RTa8_u3^Uz`Rq5|d*NTTJV`aht6jp2WZ z0ya*L|AqqF|2+!GC{>q%SByW+PTt-QPgWn@QdhRKvO-^#y4ni3UsJM7d)-U-PRMR6 z9_O163!R%QYa0iHJ#IR!F2x+PY#(Gdquq16EGq}w-PBiRNiKJ?+H`U!9%gb*f$u2>I z{HC%jtFF&^4cu;0i5)+fLT<>9rrU#;EH?gD-aI*Q za9km<%jUA}owO%Bo|{;?DSy2a^7H4zqmUB+;fabGw?RwwcE8o0P6N_ot*Qwp+_si) zHgxBa&2J%CL0DvdiF3y~Cg`~71$Zqh5T$F!Or$|$d15u94za3o0u@X zm>wPjL40EJ-RR6u7O#i@HwJQenB2G=!+Z?!VQ&SaSkr?TyCgjLm5#`u zZ$M=sttIOHAuSb_*!R0z=ThVKqg=>1o%g(jp3{zuC%lu4>}f9H6&Lkx;GQ&A?Dun; zyVyfKe<;{Xohh9?)S2iGC;yEB_LE26e=v{-ke+qVZMQL(Yz!6 zCk!zD7YwL)I+zmB%NtoKyV(Ax`V0XRBO?a^y_ltqi>cFJYh&nQDq?DEZ}NAh904o) zKQ1R{7XlVe_Wy>F7)@#WEit5?+uE~2*Sevb3P0K)0J-eGJ}6DY#@0~8Ibtsn3znrB zhbxX=K2;Ovp&QDD4vp-ZQ+0h;_XIHT3Z*>-PGSG5g`GwH(rEanwA_SYLBrvPviMcs zVwzg->g?vHRf<~4)!+xd1$`AlqOWEuEph*x1=M(8rErRVzDp}mz?4R$sI?7QRmQQTP^t5g zK>Y+vd1;A3TjuYAmF1KxG(0P~Qa~#6ao~PSfEKeTs!xPQ;WrUkOT()e2b&a^k40m% zQpF{xS3T4mfQvt`R$ZmQ_nip7mAw&GD5&1!^MB^s?NAmKQ9t9mK59Rj^Uv{rc2vSo z_uyA>eSX_--eL`B+|T0s^JouJ{Iqi?ylX44_qK!^^Rvtmv{i@eJK_zgJH>YjKZd#x z!dp(p(?MJ633bpE%{qkg9ECVTAx#iy4kXn!HcwZTceS%(Xz~M2Swm@PZPK_UGt2ix z$@rXG1G+cBWw5G+vR;HLdGpdYvtxGFP1xyq&|pImKB}TPU1X1U(BNum`bYDc@@jT2 zY|yhbj>6PTDm_^{OomMI4f+@-pPy>?r+#YWXmYP}nI@v|q=vTES>?QD&_df^E0u;+ zxnOS9dBG*`Nw?Uv^!wJ!l4+4$Hr%-h@(<6jhfzb{HVlAwM-k5srG&a`j)FIyP#;|% zNnz=@xr8j|U_5N^O}w9gTWGXlB+@5+P*v*U8HKveG|g7>p^ zE>0`WohN^LWIStLlS_FoCMI=;d)Zq={c-q^&i&L;EmeLqy*JkjF}mlaF9`uVz3u4S zg^bIi?fyA`xyj42`*|zbzJgEcy8hIsdlgG z^}QkYfQO~uVxN9g-sV4o5M|DpSyJrqwBP`K^f_w@e~%op6plIgc$ZKzSsgX=x6x8d zFWhz`R|@FiUZTki-puWwY&KrxY4E^nm(k(Y3H@=f;!OGRFAgEUGu8t-)6pzV@=Lx3 zoBw`Vy17Kb@}+DIu^5#cZa-o{HD26i2Y<T zD0i9;{skWX72M7A!pTR+`ZOKZu(kB}{oTS1Gs|*cbQd^t5vENsTh8`AaPH*AEi1h| z?YVwgl`ETk^|8n%|E%7z$GifFMZCkHH+W@{OroP>s}DY%x<-f1w92+2 z$(Vsdn_!&&Uk=2}@$5c6y3#_ffYUVw#;(RBZUO6zNZfr_7?HRJZqY+<4BY3|1EyYN zzG{EyIMjm%tzdB$jY+%$&KZ$-`<^f&@eI5KeuHP=l`CcE1g;{6qnDC&pxRW~3BXJ| z1&F{i@e<4j%fL%K5i9~LJvw`l4P5rUA>ruLOzx%UQjkqNi7+abkV{2A1RxZ9Y5YC= z>R-=j;{ZABKnE{=re4$7j&M&T^%CexqG=G>C|+q0u>!u-AR2g~(Zq3d-c|^5HiOkM zSViJxYY_Q8%E*hfT4RJ9iC3ygWUD^|SJ7NHUPz@22MMFn2vWJ7} zTloext9>pkr-ga4@&;AD$ATu`U!~U-1D4`!S{RQ3S(-Mt&z>?r`=xxpK%R5Y-Ykcf zXPs_3)*L+S$H$hn&l^lXa^BvCWKPBLw@;Dr(-JLXr3`4%dafx%r`RXbLALh1vCYj< zj5C)nCpdP!hsI#Nu$V~>Sc+`#s8pLBnE@yQ9SB25*DI)BxlIFqq2bpi?IH`LV(On} z&5+Q;a8^Px_WX8@R$)4dsz&^ehFtQ#fKo>p8F)iSj`K_T&NOt*5*mF2mTi%7`VH5Q{^>r|KWemClCD=!}c9 zfm#Ho-XG*r-%7Fz7QvSB{VcBeFPE})IQzcNB(6)RKY#vUQ|B(j|4;D>)BiVKVP#|c zf5t1E|8o5kuW+#Z_dnZdEo=MD4$SW!{r)&}e5pr2P7aE>NiV>TY76c?n?XK`*hCM< zqAA4%w_d+lX~hPc$#P@VJB1=5%1oa>UFwNUQVjjki871AjJJ1lGFFkLh70qHOr$iq zaNT8D=32_U4A;cDe+u&!^GZui)U0?=8)x>)GmxKHrZNIbO?M7nHyH8Dd(8zGKl+H` zF>p1T`u>Vp1d*{Ifo6_VkQL_x5ljT^Vi>3-YSBQkWI77KkBxYs5XB&;liZDnkRdbu zN}v_|+ixZ9(t@#GxULb6W7&*GEG|r|!B-;QUyxtErx*bh+Lx@fgp{#>2h7!)wVHMF?c-dP|}43KD-mGqyz_VZMNekO`ZR8rVY}B5SZHS6yU~kzBAVasdjN5nU6P zd3mEL9fXzRl&EwL?_(kh_*WNMyRoF6un3yf{y9H2i3{3q5WSONf;}uN)AHdu0zw^V zN3h>BP@_QQt)U#jprmMzpfE8)CWLbx#RiHNTCE{mK_JwqPzKQ|$A_voy$f%C*g zpW(ZH#*M32X`N0xG#62m{0-~oY~I@bCtS*BQW8+*E zAIFJb)XzN>^mHKro&zaD9>M&fOdB{U^){uOZs^Y5Z$8*TSm6u+`Ju;58C2n)VgLqC z;z>EJNj3oe+@Ws5wk zd&K0+s%9Xjz=B{A!Uq+|93Y(2nAeHwJV*RaeyKS?8VWr?MDhM!Y){65P zq6(vUhF3JBltyn`QdV%oncQM;xz+KyS-WDZt5Vi<(75+1XWT5t<$&z2KF7?)C=|S< z@Yc)JOge9XzR2UFpR6)J`TqKx8sBC)+_u=+WTP>CyeJj=Dc0y>DqmbrU_?6fZf>gwj&?b zzXptZ=3Bn4Xw2OM(G1ogC@x+;E0_766Eko8T7K?hYuW7!zf8AS%q69x-I7s5$3n6U zV~u)?I4PseKAKK{u0}GNaH}YtBt9L>fjJxpV=%T3k9Y$y8l%kdS&w#EmM>0Q)7DK} z`8dq}ec%DklU*-e`zcWS(l&3BbMGiEd)skDIlI0was#Co`?$X%!Bu(TV~^GC{OM?5 zi0i&y@&3y619n?&4Jt#Gl*D{~w(TTX!Hg%nf<*uoKKOemYkzzLpe@*1c8%yGxWR$m?sGYEC+0br3Jje3E0LyQG zc*Av!XA`(XgH@!m>A41-YB+-~JD&A^Q=#QUn<_=24E!oPuR~kWY39mr&z1v})m^22 zdJLr;@dR(&D}}nQwo||E&GrEzELJHzH`1aaI!q{OL(t9=2FKp^cZ*`(&1~jvD`}Rm zhRUkV3u%hJit2VW@t3`(F49=%<*okZGV4AMedd9k9u4~Pal;=kcEIYVzEE5(aCNYw zx}-fibdZ*gUQHEH!yEuP&p~3QH@Ix5SaCw~1W#2Gwrfqs-dw0zqB}}1&%kJ4&*H(BfE*57bV@mLGVuUvx?if+^e+!B z83EH$S^W_E1L$F?i5PQg;BjK-(y)7VNi#id0XTUUBd58h+3u4q5FD^Oea;v2y1R1o#n?_ z(t7@nKku*>l)bD5*ClN<`iT3zKhIaYB)D3)AJ|-*SKyo8ZnTS(dhvLGHite}mgL*U z-OWM=0<8={aER19SKt0ey`Wdx3#h>AK|sZaP&15Ba_pSs%}mhke-O^VVoEPDQQN`U zfUO^2Sm1Psdn7SJ0BvJ$u=nZCb7A~=LzLg`ZHd%fgjWEAov6#n2tO)e zaFLu^L?uAk$po&xk65cigoUvxa6UrPDVu9Q6ZZWyY3x_#d!@|I zUiyd!I2T3$u_N8+mV{V?7x;B19hzh$>@*yzAR|sR(R>Ow9-}SC{zC2~k{FxnQ~4a* z!`_d_+Q(n9c2giXnGv>!;Bvl4n20TOwkT0wG>9gMS8SvHdQNf9uRm!u5tu!+vLkMz z2E(S@)C~=W3GG`G?BKjbRv{GjdUfMfuOr9l-6%xn+K>vw;H!(|@o^3pMHcN+oe z!u)~T5p@E2ZfF1jQU=PJf+RSzzzUF#5*MCGc8GQ2V|d@$Ae}W&Ybpz8BCW*F*!JvY8b04QS8e65b)z1gX}-Sm z=~}eO_x+scq=AsbC({f?CSQl&Kv!uM47n!bP4E%?d@Dl?f%~N-!tiD zO-(Efh3q~4KINa}G%F`79TPj--(Z!Ij)Rln?*$5mPJibUFtPtr!BHZhS2A_BcXcv0 zb^ccZ?q5Uy&f+N;nw$QUkTbP2u{W}U`mbU0|Cq~@F|{*yu^?b&{VP2EUn@S%+LB3! zBPl(vwR@t7Wf4TtTXh^pI5&8$e@kKsRdnI-)qMF9D9~0&`2cm&*ZO}Kg?NDRid5qh zh2bd>C;!P;G?L0~Dox)QSRIU{+ZBtD|^K#v7TlL$* zpK7@4x#sEj4=0b$!|4q(F0T(~qXk)AR`!mFu&icBws-B#tCgQ^Hg>YeunTr8*LnR5 zzp~d4r?P#>?p3}!dN_65z5bFd=Sg|h^;X&KtG4&KW^m7Ev+CcAer2n>t?bt6PTKdu z?xn;iiDegj%e*sp`!}_Ve8%CVe}1MbTit8@e2eorHOo~ijFYU^OtCI$^ljm4cl;&P zv^`$bJ-R6tmXW(XHP&G|xUX9EyS<-Z-{v(tsSb#iDRZ-%`=*O6N8$L)9#JDo@V%Oj(gumVRhJY}vhyo_oId`s_sxBp_6Gw>-N$BGY0Z)35_ z0hjahkQ*Ip2;;u1D!_`X(=lEsy9wY3TMAg---i*0%J0FRjJ>rc&h?TV=i6P~@s6xk zN!ttG1l-($9pdEhur*t41R4@=39CKhag&FfHWf~1V!@;4QDGpUju5(dTRtd+XFFMN zRY?eT5{T>Xo77pheQU2WG%Al0ha0QA?bd|EA$nY5EP=$~8Y*7tNh$B)V*82+x7bIO zYoXm%NpfMJW1#61NS~?C={CkScW$^1zxM4(FK@-ckBS>V>SafXZL#L`w!btUDGVwx zauZ&`%}v`=-b8UglCo$e51d;&l5zXGTyVN^r=iptIZ=~HW_FQkO#s5@d>#mY2(RDC za>OF93n_DQ<+HHusL|@%tH#xS+QUo-n_9(;dP7z@|Jn4s}R#;p@7uJvM~;G$i$_|t6nMxH}xcDdBc<7uf5fzI;8 z=-6c0yTfMBD5;|iI)p;y>xhH6rATC$lfMYE%(2Un0v!GvuKwZByo_ZPEh$rj+=(nW ze%()(y{)QXV_(kEMgeF4B0tKy{o{q)1oyM^SS?YFjJb_UYM#Mdir3AAb@^zRT$X2(R*Otl?u%&Mm?09v%4|E~#?&Ht} zPsM-rhJT+S*Wp?xF>op-u#8Od1#7nD>ptfS_3T;{6rjqeDHFc<-WeEt7sv)K+D6UJGOQ61?1GNz4YwWs z@L5VV834Kq76W0#+Lo_7^#GbDROp3-!P-QU&?RUL4~rj8`fS)nJQiC42UTo`Kxhl= zfDW$t9GZZLh3OJ(AYYj|2q~zT!17tCAuvo%vH~a@0^0sphsSWt>B5~Uz+q19eTWOk zKH+ebD4N*PdWuMa?JOvIO8BgJ5mH^m@ecA36jXi%m0?KmLE$MQz%|JZcc_wAHi*Ej z_Zx6Dk%-jVOAWe`i(g8F_wWK{5};^yYzh+EcJ1cvA6*a4o?Ll%pvTxmgD_1Ldl#Co z%FeFZeXf>dP-edjhRA5<_m5~C#tSx9N`qC-tg59X=#e^xrSd)N9z8g^pNMp09hN3% zjq1SwdNUG>cOHimq8K-*v5k2&&jq}NsH3dZ@>M`%&>yvH@xj0v!B*V!k7JoW2N(lH z((+lbjoo zLI#wIzzH8Y35mo7U2?5cT!9-otoWN5=#^wS`lV!O9NX~$P{baVpR(G7X#&@L@t=aN}tkgqKq4atOJTN^RWR;trJ~*M(xE<jo~!nIYdTdo?jWkj&(LDeFQmxyeF5QFCiOmK&>mr5+5(xZ1q}^kXq+ORT;IeJoUAAr8wr$(4F56XI zwr$%syKJ6*zy9W&IWZISXMSa5Jh^x5ClV2P#l6;w#UQkKP57N*b_xR$8bIFA*X=*; zOt76O@~n9CXx^*{3u*=*ipL+Lv`fWOU#7%OHzx(_cz-=jRVF-X-Z%NIpP$Iy{30_NJaQX6p1cmlM zKi1c8Io)Oh8$DM5U4Z(+geX|^Kw|9GpEhO|c#xI9i;Fw!;gU}3o^->zfsEqNS%a?0-x~@T_cawJZuz8`kyO+*UaT+NW@yvzm+q%e3^>R|a?x~VlrYjk zUiX!>g~R&xzac<1rsU`mRrc zr;z4hk>){>5M&6tGKSpf^IKZi*vC4&{OIrI?njb*>gpX2IQmu6SxP)t9S>jW@`Hb5_MGiN;vflcSTM08 z*|N(@IJ91Ie}PfKhumjuYYb~c|vut?$(0=6`N zd*!>pWwqcO@`&sLsBiP5)46tC0egI_t$$ z$gzN=ISg!dt68(f9oy@hBp9sr%Y%AhQU5-m1E1u}jYnLh)VMT9uo%EqHFO-U}w@3GS|Y#`3>D1H}^O6M??%Chm>%H zu3T@OeV9Q(KTe$SSdKKjhe-l>{TGIu>u+|Z7IZg|P?0@VtEx5hV8rO$CP_4%YRWI| zAq~|^=DRMl7Qce8pe5A9e9{m)vs)Y>z=0iOSKj$+0fJL{@p}(HcF$tvNckfBSLrbt zTduI*Cu@>HtM(D{n}iUpl|or*ad?*1wYvR@Jqq*KCrTt>XyNsJURxMer2w};gV8G2 zYlj)&lj={t7APcIw(aCUUnlq8w(^B~myQzN#pHWk8$OP%Ic4_zWG034asrOg!A!4GT4-qvH%GJ?Ci5NpJ7JUWyH# z6@6Dkdwnst4O3jQ(>&*Zw;I9(F{4s{&wa`N9E)Mu!qgIsI6{zyq)lo><13p5#5a^4 zfRBnk{Yie;I6DND#z>3AF~Su}=S{w?dp%pz@@NwdYq^U5BhxVW5{1MAX8_l^^vKC@ zHCI1z;LfiFK@~1~e{-rG*>QrKp%rEJ;0$l~haGSwAc)6mbe67**;8|6nK+=U`&qojJKCb;vsV#GnPGK!+8a1|2$%eTXoa1y2y_+2}MkGTk;DgWL8u{sos zkiukVIA{ju06p)0q6wNy4*~?(E}&C~2Z8%f;a+Hh2-~*c@Snb7;nHe5(4L=9?=9;* zPCICB8mq6;7&osCo?pLk6)Ey2G)7K<;}5F)UmkXHe+>ha_%`_3Y;*G!17gvqXFQ{C ziOs2+7(Sw+-BUoENK&MPU~;#Iz3;5Zhydt`?c^Z$w$9S0#mF+LQtK|4PLBZAM=7G_ z#?9$hDo0g*w@ zZQ!WAy-pL)1d~q0u-AzSMAZN34Q}aRjp>BR^Q{*-D4&iA+bU#sFVE;NeLX(X6JdCLuF#W0CaEp-c71=#%m`p~$0YR#+xz>qi2=z;q_j zz(o~BUvDREnV3VSyt7hk^=sj*oEmvdx7LK@oF!45r>>7!}b|>O}W2v@~s#W}}$fH!6fJl(RfBdO!^=#b+d`f_l(JzPRYPl7(Gs zJ-|selglpHG!&U}B2ITM{$jqKIp?}_-4O7(1{3=bQk9#gHyk~Wa?2jg3>m+>9hKXZ^R_406 zBZc9bj(E{be+39Q%k@xWH7k>#-KH*xggs^L$iqkGmyRR~UJh?VFlI{9ypx4LC`PUj z%^h@jh|D?&>q@)MJGhB?=QQ4CAUil|vJc>eB+dFmL$EWsc5pzj)H!SNd^iz8tJ_6oQ=CBp0$1=Zz;hKwj&%a!yWPDwGIrm^4_4F;iCe zSWJ26A^?4Ox6dgqf~d`^Ln)6BHYI7u!g7_5l7!NHUg&g@5KRBX;J=~AP8Yn-_CXctso21S#*0S1>=V+r6fMw$q0Hw zocLCje1Fi%H)7@Q*|FGX22b2Gke>-k6AEM2LtX4|%3A2rBNOtYeu5@}Jo74omA0Lo z8zCs)tLB8mNF}}|POqsXt(e#InRpuQb3bUv_-ecU5^4goGF(NsL~OQ3)Zb6VaDeAkTyY>Mv%xZ zx-%Ct!5kCT(nxG$E=Klamt&T%4P5RVhZj)%&EY3&Tg0}?0Q4nIJ5*uA`n$o^D4SKJ zgrh-7p>r1aS9B{Sqv8IVf6u^u3_dR<%xdW8SRoM8f*adb#Loqj#3A;YbS_r}8ThO? zi7#nOelI*%GD;Z}v3PFS2&OvmmB;FGkOcU7mHEJ#`K?bV{v?cC8v4+ZXom)RiWcX` zSVr^@a`a900r>f5o^b;ieMPx**n8K3NpSC% zfm^VL7=AIT{hqr`p6H746NUK-cH~f30poQNz4%-^Mrm!W$s1|k?lBhKiOWZLVYd)- zrD1q_wmG2uVBslsf2_kiBvkJ>dpHbJK+R!KG&m4%1Gt85n#t(J z#k6`dN%ObxDMv|0bdI>mSk2U87t%3xULIp*2V(6WM_>a84UAqS4<-4;gG0e9W%d+a zw6Gi#`F6MP7cF0Pmd|o*F#=-DAEx!2Gw2ARKhK1KqwCGGZlHU9ss)8mk^)m5J;_Tk zNKX$9vf1L|iaxNK*eB(+clq0>3kEA3-JSxnsPzyoCq{Z>Qy5&Aip7en3oihex6f!> zTw|i`cjO83ffg>!qrw&aK3t&Dp6buZ1`S6abJ*4s)DweEv;o<#6Ue(w|8ZZ#ZrjBa zja;k}tk$4w?1Jtjt1@ILA~2~|$;fjK-U^5Yiz+tpf_D>gi`<=7NJQ=F!udW5x?_Z; zdyG4NXyELgK9}PupFYEHLX`_Om#TcC*I6_nyvNiJb1+bV9Yrl(m}lunjU8|+|Kqvi zyt}YdGlZa?M_K5E#?ZRJIT1O6N&@Wq*!LGY@0_esyu$Yw`1*=3o%en0RWS1p)#*8M z%II@i&jtOEZ!iS-&BXso)c@G>Kf=q+O+femi26^^;a?=p@z;r2{^a=odCo%rm!?_h z|AI3M{a>19q5n(MEcAbAnuY%FdKP-Nf3IWz*W-UC>%S}hH)PGi@b^@7|1DYnv;TkG z*#AV<%&dO`AOD%GtF)w&HbfJUo-1;HNw{yY^P&UHrwFv56CR z0rE6?dVG5-EBm90Dq&}eHRw?485t#yx;hVc zb(VB7wW`S8Jkd%;*za?jyO#6W2OB5t9?JL9?^dk~L9(5%oN{j^wGpnf#4p>bXjxy| z*p5fGsashS2|b^?Lb6w@t;a`Sy(bpakM`pPKZ}lY{|K43lLWnrdaKw+hmoz3W{%)u zH>-rKXP++Lh0UtlahoCJ=M!Ws4aFw>i`ea(tG$uitnz%cFW>L;jWD%uZ&LcKszIvM z@^Mnx^wRp;imAP&4YxJRs5bn%%{K4HOEo;z_TGcgoQ;|*z0c3Ukb@k-F=ehMD(Rd* z{lPNPsxeqqsHBRqpB@ZnJ&zPqC$ft2YQqM#psSQFsH663O!W3EB}tzjdrn^@$$S1x z|GgG=JiL-Sg`Npqb4aG1rw&U?mTH<#Es{6^n+Fm9@$(hNMsQ78NT^6A1WkHmft$O6a+pj*aUmo z96OEH#B%daAFJ}+Fel9EXu4SOw$9h%EVpw$@H7nHw|J#S*tE7r7czm<$bHLisiU;R z#a1QZlzSvUIX(hG<0^Ml_2O)wZK;_T-yD%<21FBWBAc~giKFJLjS!7{rwTe@Oc>6xNZ|N8_qiJ?!Fh(-WMOgjNIF^ce;f- zeGFPgN{&SHgyNp(8n{(j)tQ>%xn}Gv6S(`3Vp5l`T`aK;L&V`WuYd7&u4GpCLfN$_ z>fR!(F=T+w1pPYk;3iI-`oe+BL$>nWC;g}vdiLxC;9N!h{TiLE(Vd_h)-udCu1%|( zKK&{Cq7%WK%^dnpPmSc*&0X>AwBHVbS5?;Q?ah2{`EusW%(Yn z%y`&yGg{Ko)NWN8eOY023Z$c9JpkD1*o@xqs48QG?U8+xrQ&Q z13pWC#3%vN-$kNo$(y=nSW%*Eu7of_f>gYc99|Dg(GWA|-XKf_NE*c}`V$ zp`^W!Qqa*fc>p27A0IqNs-n*s{2XePZ)9-{^xigGS~FAg=vd6lN2r5&Gd4+}L!3)R zcXw0_qx(KI1``aZY{CYq8f*CFQsge6V=gOR-2L!3%_D}a1r$FAQ>lQdQ-qv&3deGV zVfPnqPqZyYW_Nk;C!~|u$I+7Tm!BVrf(p%SZ9roIpJMu#Q6eb9Pt<#!AT@l7oOph? zw%+!&H#bTTbb<9{m5*)YJXUMYOab?uV4-eS7ss6>Xt1B^z4)06l=wKeO0@M}Sg({3q$*-D5Ye6%5X2t9K9-pr@B6x8^dvL&WU0(Zew zAPF>E!USuXuIL0Gd zikdaP2b*@P@_O&rNoj^1koZzM-_~B^^YL}@L7jj5*Bx%{CrM?vponpW2*b|f^~@5Y z!=~m~%?ck#%dN-E4?uGiGP4aB2)CJ^{A+gMP5xHOdz;}^EU!Os{`~m*CZs6SJWOIG zeb^taAroq^HcVlXJ+f#t+UFxahd|3raH|KO+@lq60}NFWtEJM4%-JDQ;0FP#QeM<> z#w`MRCsLM!S0GDs9#{Ct(4B+7$kF*h=7T|!wNhc|_};V~hrdPUDuK-vp!A(*?OYSu z$S08sq{mTz2(9@Nuo2_JY}Ii!oaMV5J|f=e??J=Aq~?>R=`cGFoI}pDK=IyZ?R?kS zPCL6MA(HUro}MQ`dtE|4%+q&5vze03&bUAO507vOl|WfnvB!2PMXwF~Lj&5ioTWoz{N_?IO~{M59lnnuR0O-%bu1SPL{OlJM5w_|!PR zz$PFtSHbMxM48nAw`s-V(z6Z2sqF)uA)uey-!YzkVyLc?fDJO(ez*<6wW|li(N1MW z=MNT~pBK;)Z|IQJ5;-6wncQo)ULbIR+#`wB6H~Nb?}S+Obvs{?Sy3Rn`aFRbylJBM{CWw%CCt=Ss?FJD zBV%qAdxa@eI>Zz0R7fZ|HShKk+0D3XO1QB-_ak}eBYlT4kek`^U~DOm6T=1@j{y|?OLs>D*=;&Lb=i2I zthTM=1SgcREGyAMWTLk*y7vc#g!Z|u>{l1^ETQ+H-UHN9Wb6cw74(DG@6H%9-%w-%j238j?d~@D+ z&tM{ER>tcvgNqi*OQED-)`%m&s+~At)lrCS8+*PVXbD6QKwyl5VCoKN4%@2WeYC-( z;7@g8(mV=~g^{VIb_vUjiIP8Le1)2|8eg+AVI@RCk)UC^vQ&(#9ISvs@f0XWVYYx^b>|8vmLw-E@Z5YVVz#i@Hin2Pe(t4amR>Q#kv7y~gW9)Q+*WygeM_P{5CET3 zY#;BB-__SW`^We4$bxCgNRpe&#+(uol99#kWN2hJ!di0`Yywf$M^ko(q>$CaFt9i( zQ0BZ+rD9v!*P_&5R>;}BSSk>}(@7HeK*28C9O91AS^kynkI`JWRSnEgTpc^>&kg^os34%mk`+{tW-1@mINVs#mHZdquV^$ zFh5+BR!D^{HewP5wr(+vb7p`kJS*YBw?!fbhgRUZ5E$p=F;VxR1|w9-S;RzKwrbY+bhdu;Zs)Eu{WY_`8bR;+&66kFL# zLcz|}J|aqty9m?oMMoMlX%y1rKkTI2I|v{g3dG4QUC#KyzY+Uqd`%^zg^$dPqZ2j! z27pU)%;^SM$^kMXo{{Mw^(ciiR*h>J=fzlaX!GL^SMa#mD9H{#38xcfa(GF{mq0C3 zu>?Aii@n0ah>UJt;|nN-sn);ltiJt zVkF&Zn@In5OH+MWJW3D3d3wj4i5-(XXx%2tMhVC+8Z=kwa2k$Np#cn>bPW7<>y z1A6CFWhz6;1R+`))Y!W}6Xn!DM}c|i#9}B37jx5K&w-+Ua)EsDQq&vClPu}`)i?&|gc5RN9>0npxTY>N7OpP}x(&Gz!0+RoW>t z=2gJAo6EcS)I{^=Kg*f_= z1Ai1Xm3{#9c4(kw_h(*Mw;$E}Di(ZC?MmeS5D|_q6|jXE=B{cZo=sbLv-nJzwf(QP zgjbICN|~(?U1X%F$Ap4%pC!FeT?Lk8(L4f`N=4msIx5JUEyM& zn{g6OSD9xU!L<1g3@X)Xz_JqS-%M|7Kwu+ulM4K1l!UJ@{yh~I4nDBNY5#wBq?Tx#6gjp0s z`PkB;7}h88hB!HPAz&-G0$ALP!CIzq4JT@{y_hho*}61u z?)ETO@!=h^**62011J0Iu2uKTx8NOj^8U<++9>)*VN3$+6m$zC%0fu+6L%P!MEN(q zef#f@|7vUhVAy{U)c>_L7W#ji8pFTMjN#vA_Ls<5{$Bl8a{)yH*1z7y`j^{S|7siS z--WDy*Zt9}e`t>Nue7oKedT|Kw!hc^H_*oV_h|pUv;DLEf86B%jxw#l=UXxgR$<%Zu+`L|B8S434|c z@#;~nZ5;`KA52J*m1jQ6Qo0R;eqzID~$)OM_j|?}Twt{rtuDA2= zoz!=kqhcNR1Ei|3bX=LR&qIZ3kA3d>F{O7kiD=j_%!oAe(_nanEfR*PM)`QD{O>0! zHU*ECDs@igv$7`L`}Magii|Ki-TUa6Jh#pGp(|KEn_oWvz}N9g_1c>QZqK&P8MWu` ziEB}j`|k}$(gvG~&rtVcF`V>b=GybND(xm2a;BWvSO}^=?_t_%t03G6j~}}<<(VM* zLXOVEK&>?|^~OAe#?$j4sY+)=jmK6J+HTN3?SDbj1^QVKD-!*}h`mby}R442t$N?HA@5XFRxD`9X z94!9V9(jj?(q=jqR#nUgb8sqGV`Zkx&5b>jjpOh5z!~4~k9I3f56(u|K^rk`w{>B= zoBaIP+)sy=!R*EGw0tk2DQ=~X2WtQU`D02EtZuPlX@xdY#nIWQIjMr`FSj9^M_&=^ zTj}14R!B$T=Xs+GwU2%}qkv*r*co)$Ll`@81Ery#n=9CfjUUYL5dbaj-A9Mf%E3Gv zgZaqZg%zqiHK`|pD4u@QYCL?8&nEb$f6iF@42fAiYLo;aM3yN|tU1I?GZh$9J+gWiYPZ z)di|eXe^nG!#s)G$0ks9cqgpQZX8A}bM`80=eWVHg-l|CDdIyecC$)bitH5W_HIv8 z0NYm?bolE^&q2mriGL3ugdY1c4w#Xp#{{Eohfz}EzzKaw?tca~y1=q{m z=GggTWV10c<4VX54G%n8oqkk!bD#V{+XJT!)G4!Z-v)6B=8m}k(+33p!RknD03p$O zjQl+s*>Tr(3HcMau+F_bFP-|e!23zR?cFNows@D>#2To`BRKOZTSmAE?uMH@<>yt! z{6?PVI{{iy%rpA88_l1u3%}A@H5ZwgXr!!BDx^|}m$?T0xX$B9-MW{Z)AZ!_$02=8 zsEb;PH;Z+bL)`>)^CMXfgV~uriJlK=66By|xMH4ioM+u@L`S~eAzh& zY;9I^ozz`zmS?B_v3(-w&!WnqGyu-d%D4J$SF?EB2yOk?NGCJ&Xbali*z-YDIqBb!oG=kyJ~&lvjAUg*<-Y5>sIa*VaZX5ylXnmJA7B zbzi$24Mm}WpJOElJ9>&hEYb50wozvk$#QW9vw|5DfDqZkuZ6vQicvMz-Z-yGtOREKpnzer zy*W|NO^tXO{Sqo0yWBT>)tCLmMjjb_>htMhfg9mW#KK{ysRj%f49#In0ed6@qu22J z)%xx|yau~-pCfGnS7lX9PkXAxi6h2dFCfA(nX zV~*>7wlsI7c#D;*UQ@%{cOMCuo=gVdo$HOvt}~2M0*l?oTWRbC<8R?ysLo2vkzS}DG^jZdtEnC!pGx}3f#AoN}h`M3j+?IZXNtdYfG%DTU z7fyHHK0Ql8{&2S%Tj*%;hFm?@yM{94oM^Vj5T6HyF~$MAA=%%VD0oF4+mGS^Rl}n7 zF4N%s{+8~-!tH5kYV0?_0Z>|j+4MHY5e)}aub5`>4e#_r^l66zoqSp5w;zPB0mF`R zX1$#dMQCV;A5+FUj^2D3V9IDewCEM@+cFg;~ruq>@h;QeuIg;G9U;|i?Pvix3Gozug6H@5A2t$3)+{Gf8gC zi%C~86H)tYrzqv=d04aE(XQJN_%7j|NjE#v7_AQte&Sg*qa;-iHXPJdsb;-A`+y3U znTtg~g9i1jWMV=>?j83A^Y-SuH1T?lkntxMNroNn#&fuAr_yEegk@+??2_N;5Gk>Z zwdC+YsQ?&c5RNn?213bH_*A=SF+3EuC8IlNyWl^*2k`p~#Z zH$AU%ZIeaZ} zlFq@bI|ycFblb1T0@Q?OjYdr0Bh<`M9*3l2S<=-1#9nUVmkH-0-7NcE#Jq@#R=9oH zm35f4SF~K;VGM|}l&mc7pq#ymJ?=u=1s$fc?YN=bS8gIM^0rkvN7tpECDfK0G5))x zy|tOp`G?qS$_mba%`a26M|+qX?iHN-z6S(Cx5w_w%w}De`RS3u-e$)RPTM&Bi9)%l zk?wUpBK|^}gsGakmd}h)*+q`pSXIUV8yCIbL(?RE(MU@S>KR-|S(`wt5>{0|O+o|EyGG6}s zsm{X9kfRNdP%UCqf)gA8&QlhvRN_f}us7p(d{+*r8;3lie8!=O5SSecEvWeRRV*<^ z2_<1hwfJT82WK$55%|wK;0)8}+zRis!J#}y#n&6}I>?{2!o5LV`OA*5aPsM9B4&a* zD*ae{5-7!aIfUF;z=O5Bt+fKAs5P8S47U_W%Ek6lSf&$hUlEFru%!6V?8A-_{!zxt z^=-#lZ}C7QBa!83lcF?<>8hpN+)Z!jva38$)dh>k)}}wGk$`auUI^UsdA|aRflQLnWoBQ`=z=r|bHY zn@ITdm*B3LcCzQPo0sL05%Hn%@esJ3au*F3~_#@S_iwFLwlf-a(pU}z>$x9QZ z<~$TEQ!%}pnO8x)DIiLCCL}l^ulJUj$o_CpDD{u$IZbE3_8ul|ca6_Tx<)(V1QbEO z$+tI(q_j8@;w8RS;BLc4%432N4{*s&Hg0fx-c~U=ScmGuOY_c%&_Swzq=fHr8ohdd zQsK{_@wkoDB7|9ChP*U(Qpcl30xsPb{Nx`#++&~5n#BIxNY=Z^{;53|A5)LINvJG( zy7q4Hp{Bg`EZ2=AJ<2p#?jjbbEckEZt@Ar4w<56Kb@-Zbh-+vj2Spwy(zKK zt{FiAkGeZ8=8}aG96kkTn@2JxH#Q-^ePSGc-- z>S}Uvr5_cRa^7am)*LZv)P%y%tRVEyu=;Hv@ehYePQru?$QMs_;hoL|QKOxQBUPXt z1lYM}OE%%3{+vpIj$Y#lc;_jXtzmZrW{O--ZPzAZNE7Aek#VEs%`Rh@qKybD-$Ox5 ze)gz%R66n)RfT;r5(U+`aMZc4keL>F1J-b|bmR)P1lCRrxb74kS9IrslYG(Au$>K= zvEJ?RhK{%u#;nF_Ul(+_IT{m!a4ZEIW4-y*>P>P>k@h(bl1&ACT|${AaUKmpasDO= zRA=eX0DB~gAQYVZ!3qrN)ceCH1Px`{Rl76Z-77s3$s~o0v??cc4SjT`$}75RLMdHX z=6(qg-6dlo#wE@Z-_ZMoXY z2@jsX_lD&1q_nLGF)0!gj4<-kAL>gWi;J&z4>#hUtpK=gQr(u~K-8cpON z4-#_Oi23}QaJHHfV`!QO+B2SH1qMPl-K)aU{?}K{AKNF34Sb2!}!jJ z=Pnuz7L;;vfHIinK?mt^Co#cROgE09sB=#RSrYgoB~HBt^(bMhmy{G8HG9Op2%%U&8OgXyK8);&5l!eXLJr?4D zUy|U;VH6TUGin~IRJH@>1&;Go|Ej-!dulz;rEP8n*#<+iWQ-}=+bo-RvgNA|cOl|_ zYO}=WkvW&x44wK7=Mx^)-31eQMK_ZyV3cEt)RaI-vk}!9hB>F&tmf7|d-On=*1%!X z(H@#puc0os!>wB+r%dof`cCd`q4`48Z#+eQr4ubTP-^w)e9E*f&I++n5oOou${>r9 z33D4wRe~Ix7dyME3WO6ua44#`b$8JIrjFO*=aeUiPH;y+>DA5y{nkGRM+^% zzILnAoYn2imEkTf*6y&?-s@(0#6+!|pHGrfE{LA?96QouRqB#Mj`Q%b80k2ualwcA zq@m4tpO%s**=v4pJWsiWg9wG(bkLf+cp8ilhX2aBc}$KaXW8Pyp1L|M)=#;cIbjo1 zGnJ>b7w`9&A=Z(Vq(y54#fm?NUgV!YW>15%{1W>Q&XIcXOF68!u36h&hHTCLS2CsU zss=Q{>};l-=Mjdv#TRDr`4ZpowQnI!+Q2tnHWxB7l+8?%;x0Au!*42$UaKW^i14T% z3F3buoQ`K5+bcdax4+72iY!AH6o#yI<^%1EZ`-VphV?Vaa^XBRRKna;9xYy zE`xI>S=a9w9nyXW^3TPUZ2}EvwT5!sXjgk z4y#N@+$*J;VLq(A!SQm<&s`>nJ#f~&XTe+I4>G`F#YPPlcJ~upZ$a~Jqle&=#BU#J z9$D!?0ZB$QW&dBzYqk~nF(@+nZ3?d8UBzpIx;i8s4{JL z%UJOzZj=N=&g6`C#C9a^jU|!tu^nW$U{U=}-6t{yv_1i89I)0ihU&v@s6=mCN-k1s zaXH(QH!MD$63GuzXU>nEbbQL!u>)?{v@JULqF8$#Q*$WQ%f9$}?mt3vr#S^dHR=E| z2uqkYd8AkDJGim;-<^IE~|G}L9*K*nZf-Ku#kY)P|vVRKh{{h+mOtAm<)BiVs^eV((UEr2iZ>w zel4uiqJCC?l8(6<2iXz@-5=70v~NyJk19IoKA>{ux_` z2g{u*CX4G_JYVM~%?89`z(PxHUnsJJnts3+at;~E508{HR7|e3KXc>)*P^xBx}B(| zrF-TI*Z6tAi)!fyI&!$-*o&L+zD;NuZVYH{u)q+RnQCKDgbqbdrJ!i3KWjwlc*YoJ zSv;bS2>_vOc=xzcJu6i2gShESKUnP}!!GbPckVMo|%Q|{n4;*lz!ZJlODOegN z$1pJ>YYD|!jA3SvSym}u8bnikjFfW|ITRl16e3Z2(G`SbS-JK1>-oBa;w#vaxzgWh z5%63_Lr%C{mT)=s{t1GxPs%ooHB)oT4v*AIRq3v#9N2VK@uqH6Xx)Ngz&YA2+zxo) zVKWi_0By2I{-tF663uwBFCD|Q8$u2FQwP0%@qp-DYthiO#B>ATQwFI)vQM(uyv2S3 z-mNVn65eI(ozc9kGDdgjjC$Q#A2^RSlTMmRJx>a)TTiRF(9f4D4vc!-?PBG)1H0cp zlNF8EgL>uj6(XyoPYxA=gQmMmKaf(3IgTS;42Sxy;KzHcmodyv?ZKEF#t z8Mkfx`DecQ-rR_-F*EI4mRAvUnpsvZ{`-vH?rj3P`^#;rm#+gnez~jHRU^?Uee*#A zu;wBTH5H!zq_&sVlTFj>fD3Kkz0#bEpYlxY2r6BFJ8 zl(Z9X%WipH-mAV1+)YTe-ApDP$Di?=%@2^-N&~(h5&p z4jQMDH={plxb8F_CpRv`d0@Z2f1bhZa3Lrp!kwjE4^plU;=j5Arh6QfTyzXeAx2aM z169x5jNON`L2$EmK7sGW`AL>5rZW)q+5HFFqf>A6F} z7sWD`|@JMrhdDkwErZDLvYTG}B{ZQhk<>99FI+uoq6> zx$+tdkiaB#@!;Rk)u=ar_|ub5e`@2%x~zfz(jQ0n1>$!OdNQQzFt8$E zem&Ai=XMpHoCZuAyC~|NzBj+mR)2MNHMdw)*+`6Kjl+9V841$+RlhCCrB27l{gMqY zqT|16Z5L7gkJ&~6*-3!jJg6ww@(Cn}aj+Z0zp}RiVl0<`MrZ|y@f&7fG!k6qI>H(P zV>LB$3I^3XP^Dm`Kn$-^uv2_-AW-C&%#LR7q2z4^{$dW(^u$H#KLPCWQQ+IwP8MV;yE zOW#*$4-Ty`8p}r3Dy*)+XzfM7LX+3$X5}bFvxZ~tzHthGxWLY;)xa~`1qlzHgH9op zEKr$5F-mF+8zgsv4kgx!i|eVSP!B&6*0kvt1TowWR1kQnPi7m(1FGjw*NP9NpYtNC zI>!o!)IdBc6X%0uuS>oQT8`Zh%&rGko3>%B6o=2*TtxHOPlET>EUR%rP}MM)@9xd! zRw>54#p%Cv65<>-77Q!D0 zOAsI$4!O#|7zGr#jP?%fSlRI!42X6>aJFI7h^Srw9D($@*GayMHjw}nS6^f6*g)p9zH=$W7q3q1f>EOy7fdJV;K8oyl!lj%}<2JgQ zXPB61p;$ja!xBb8E8A&}?k^tl3tBkb;~B9-0>?boT()jVQN$?jSA@#c?f6v zlP?nX+d2z)?p34SP5txAgV8LofDfa#t|9T?MunU1%IgtZi*VgbsvS@wZ|P}LTnH1+ zU->l>vNGm#P=mv7uWaH7;o*=fGi73yglSFpcAMj5I{Wy*Bx%l}LIN-i66~2rr@REB zdru<$LBZ2U`rbLE$cT3PrqI5i=~zh(UsTyG{c=gIt7L(wz2)E(HeEu%GWlTQq9epe zQTj@?I#HogIl5C^FBk~+(5r!L2SOPCY5Xnb{p^6xWA_fFtJMf{S|e?nx`DI{$?q zXotL+9Pp=|lA1k-`}SJyqt(9H2Z| z`A==@&QTHB-!@lc~V|B!xx-j?AVdFFO(LTB_2~|muqQHPA$M*vp#M~gdSImX=bSGL+U_Z$ z3D`s_FwFkqiw7}XYj}MRx9|N`N4NVBkf8H_czeen$@X^Jx6ozVwr$(CZQHhO+qUg4 zv&*)+%&uGOUF$u2?>PJHxDhArjmR%EWBxHSB4>{L&1XCVLOnnE4-q7vKv<|B)UO5U z-R;Gxy5ApJc{?=_)p}A@RuQgree@_WC|&^Eklos(?O?stw^LtCX#y-M16F75w}Sva zVa_Z_t*(%Th{9rHS^*J*ktP7J=)w3O_qj;1Ywj{-aS-;GF%b5d!ulggI5nEKwwX_a zkKqUSM<)T`ed8E}KtGS#1$Xs_e<~hN!#L9$<;z45+-H-M^{RwK!j!7MBY^03>Xrl~ z8~ZP2H#Ou#8>voNM(fGCH0*IRkYY2`h}4MsYFIobtF2)zs23d5bGU+F%S*r#@YKbk zpzmGPMYxVvGYHrd5^uX|&kZHWcPcjhsEBN}L4fCT6(aXL+Vp`e!;hNfI48pvfl4q= z#0oGC`~@fX3$AfhV(?T8S8+Qg&TISqE@w&FN3dSz~in&dMx+qJL>BMqkA?%Wm`aL!vVt95m(0_ ze2*47KBS)m3Lhrjv%Hgmd)+#8it<1PVNhpQq8r3{VLl`wDhqHRa2gVCM@be^-B?gE zg-b-s5qSSz7SYt(Y0!R#u8Dv}I`NORE4u0W(H6O5S8}5v6q9~AiWm9ej)5p*U>MfL=rT;E+@Ld*W_}kD$xcogjb(Ynf#E|Z35F_y z;@_sdn`}SqQ45W4o9>^(vdQVosqE5;4scEY$1xG?ZqL_=Un17; z7pBv=t`oDPPf&rYU$9rg04=%J3otY!p5NLJm^QQt0V`5iNv|qM#%m=v_upwWEQ_{b zSl;u6@0q)`BU0+bgo<}JZ}CQ)PR?(jS}@nH?tv{~97N<@D(lf5J$td{kAN2Th`+lf zXIv<;;@_@^7#|M&HLR5B1YMUFJx|I?s;mlhPCvv$We(F$oVSaJ?#H`+45iWLp08|u zH=xtxR2Bm;Z0%`#N!41thI(D@;P&V@^${O%XobSwwx{bFXiR(Xl8)rl`hrv7w25aUh3(hGQjx>L6@3lT=o4170gEmYibtOGTR z$*_ONU#eZqcI((v6OyPSKWsmuB1Vt^$u58;r(>;xniz}J7tIc)gw#`QkA)4MgolJE zLx=DD<(2z2T>f#TumXdrhA3MkVjASSKm$_NOiEKcAJ%@BPe$S{Avx~%Mt#O2D)(On1cI>W*>v{YbqRuU@un5g88ShR^-C?a*!2kbsDy2b^C11Sn`)?w40HWZZq+B=h#7zZo@rONi)WcB1+QfJ~PHjachVp z@~UV}!X8Pght$9xq*S62$pRN>Vh0jp`w(Zz#*Llg3()^bxW!Lbv)|sZA2LP&C$HPz zls3{u8m#6v6i8Yhr>-6B=QqXY$#0@tX4zFo!kfhOR9JEEbIpj_o_|!X6j62t2Cg1( zGn2(S$V;@#96TM$aXWb3ALpXi^6BVG?a`&@W3tR8^_6L8+@!nBwpsG->oWa7`%`K!P)N1bgkyG7%^zcT`g~hWa+rzWSpi`n_b4MpQ}#E>A8hr(MdXi zC&~PE5qH^~gK1aU@xx>FiCfwn&X-~h5Q5j<&2l;|BTeI2^N-sUc}K|R)@;)7IOT^Q z?F;i183C*`Tv}c&Ny5}@RwMQZpX`$0S6PHrZ%8C;;p_NuPIs{c8YxLJCx?b7X)3=J zv`Kqk&AQ{FQz0`HSO3K$Fc~Vicjj0sH^Ys_*#}v`K7r1QM*S;UKeiD#1|_JcY4qVF zMiZn7)&Q00o<1f8JpEhDh4KLuM04gye+~QdwA%;@sb*tMDi_>{R>%!h&nQ9 zPM0ERc`9j|4z=#~*_#LNd&pPu&=-2=%TBJ{D_>{b%c^kMJ$p#}DqvXl$KvbH4g_)k;P{~kF0YyN-l=YJxO z%%UF)G~3~Y3uyIp`e#(TR9fGTMOt4z->yg0?1Yb1S!t|ERo(DZL!?WZ zsjQ-wFk3n|YPZS!>YlzEf`&bAIJ&>YWwkUidVD&iX36N_*cvB5vt4Lc9lk%+Ykj$S zwNEq|m*akx%l=V(7TRJyk{^optNrBfd+D@oo<}0J7mHt`Hy_@{!d6J)FpCW3imG)}&cF)W2=#ae_)e^zV6u{n&&A#8O zXcbKh>|gv#4@w3axyOpt*)lo0iUv?ziUmwLnbL8}!ADlxt~gg0A(cy&;!s2|lS@Yx ziT0W{iU}vgi+#yLztr#vZE_^|&GF@0xLl}74;a(B->$E$D*Ssy(R!VT5Gy*qt~T1- ztSVO6@r_eSc6`6qslwCxqc>JcB`wzQ)Xxs@vMcS?W+luT&}I@*Dx5)bj+SL;lb*ud zE}rJN_nxBe(SX{k)ezMWv1wjgw|O=PCGNOiA5@}XQa)~(x!qAh2D)y1fmoXCsm=6J zVE$yERnJ+Z3JCRAd^*E!2rvHf#%v~IdhpB!l|=@T8`3}uSYefUugwb;0zg_3EDl*Q zBhFK(07Rj55o!MAxE-N9dB}?WE$V1t(M*<}p1*kf|GwEf{yfy87ba9~Q&m}-u9pX`dRl)?Cbm*lq zMs-=Mw8(KnnDp$(@(|l@bkBhGsC*M^*fr`u_~sMM_O2$#0jto>{-|kWes`BZJZarL z2iXGKSndI{P{>(!XC-rner-%Dc|FLL&UZv-yQFOA{8iTnvSsZ_yu+!Eg~z`q!kf1sgSROq~*l$g;MM}bfVYKoSFx9SXEBhFy=V0rb$9iPb0 z3u`+^i37{ei^AJ6SZ5R!Mx8Ri=9|VwjzFVCnJFj)iJ+Cf^aWR+1l+}$4LsaHSrLTK zHhPx`Y-J-3Pjz7dEZ+52oYF%!p_Bkvm&mYU?NSN1XOZpimu}%r)m{e~@@oo`V+cLJ zw;V> zl`@CD(8#9^OqtiQT-wNn*(}fJ`h1F0U$FB5i(UgEnPmv^QOpbRo2FbEfeX;S&WAqa ziguCT2VLVfaf4~-&eoOtThZV0FD{Q^jZ;q@E5oA9HRinb=cB=cqjJ3W%RTl>-F!xE z%zYX_^dLlZ=&j_2n&E;BK=ZR`N=bG;DJ=^XPDYb=QD)zWW4Q^9(81jlGDmeBaJ{A4 z?`u~ot+X7Piua{QvYCV1p!F#Gpa}L`OyAkOTb`azC%Q5xFN!}^m{C*^4{P{z@UbpG zDUPqUO;adB8w$=<8Lb9ZT$AN=Xg@L0cj5-b^EJ6?bMT z1qlV!3oO^RgBQmI(t<5d6w(M=EGU2_iCHmOq@TOo=%3J&Q<2-`SFHgNt~909$D!uQ5B<;Ne|*cA0BAcQ~!|5$QVK?zX0{lUaTG?5qofAMk|+Z>NRJvb>`J zR{Y~U>?!TfVTB4koAgzP!34%(p+l&vd&sKji}iCi(IHvUQrZdAXK{P7A9u{xQBW84S3xjp0MfACAVTAp(wrMIs6+rkpM+{-M@}trlxp5w zEbtvLv^+{sMJ2@O%x_=2DBx1=D#I#68m0n=zQ(1@PMSci^5ADgeW;HsKEQ-Ok}rw| zW)E7_V^rgRIwGDQc-k*AZOgAL8*ps0=%UG5e5l^RiIZ{P=jVXs0+mC9pPEmoJx2|W z4rvZZ#;H^ckOfMbF&1L2Wjt{}DC@U(Q<1Z&L^-hj0+3AwP2l_>VHsEnHMFuL&ZjU- z259G3Pey~E)-sKh<{KZApm@xZvJ z4~4OIQ>J_G>h|UfXDFM)jtfwi#VQDsE<0xlzrU&@^I%TrdCDVLHg*#elXMBiDDKz~ zqX6aAc7oFHl68A{LAI8~@bxjB&?bax!Aiw{+RNG}X~~-sIp4+R!!p3(wd6w+XU_)( zl+uC#K2f3I?YyHx5Ul-PTOtMpzPwpqj75UIY-^dO=P_v2_qMz^Va~svpc#I9H49?c0$m5cu68nZ za24tppc=p(s^P~woP~Cv<{{SAydJ&ksf|T&qL{S^0jn#+$cQR}u-h=?5Wmo~Ih9-y zD>4*kwIm9W#CUmetUw4TEk0Iw???=29db3-FoQb(_6_9wo?3>zyX&g%j}AFs;)2+T z-F#l@duR|)Ze=h`v9B0XxgSIPSft|eB7?bi3)k*nWQG1?gz11vJxhSolfa*X+#zB@ z7wZe)PJmOe2Ph=5Ze-LV58a!N5ZWspm}7e87C%27MYWnby^aq0MfMJp<5Qf(}WS9p2dt@p|VuN zt!MqX0`#%SFRS18_ERvfm|kYAlt2IZWcwKdgqa>D04a}h2gD*rF{BZT2UXy88xmEn zrJklaRfenC2pp+JEP%jTt~wGMx33RC$$u5;v#VZ|lx78YOAB|b!a;e9LW>T(R}QI| zUmn<>t;y^sX) zYMRF{SO zkepukZNsw4OAJ(p(!Ou1H$T#wu#ISozi#7g>!My%sxWMQNd|`X9I+xNv8YTO!O!Te z_J?0Hg969AHDdUY`=%1<3A%t11dwv6WxHGtv8tePak&DX*tZp=(%j1`O-HT~e1r*n zW{M}XN0_jE;1tJPpV=mNjy;@nVWe98n0#8G_HOI&b_v06{LK8|y}0WxE^E46CZH2p zhP-kZZn;IDUO7Rk%3!07_wp_WLewxk=d2=RfUqN{tKACX(xtPNVDm z>VHfo1cHHqsyE|FiWyw!4I)+3p^e9UtGAjERt_{Y)8CkD5ee2!FXQRV9tXpB@p!po zLd(n3DIjeQrB&N-PA=oXv?r@BEU19*^bHVf`n2w}aCeuhHw1rnSmjn*XI;*JQ$VndeOb5rtj%>mbbKjs$h#*!uofM`1zQ>zG>Q!$w4IJkEz&@chDOY)8_X5pG{`2uY*H3ItahLeM#ipuv|^qAW!P5_tEw5sUVa+LJpSD5>JLB_TwWI5(aWY zrX8I7git0pO+O+e;$@(K<#p|j=VtmE@B==NB@jeAOOb?4b{c8eOK%#GS8C8Ju7#VT z=O?y-9tg&bdM=+Mqn_7O3f;_Po{C~nPOq#Xgjl6T2N2?H#h^nufC35i_Xd6K2=j)+ zB=lg>t+Kyt2F%^kEduM~bo!ZXqcZZW?dscMQ8XoU9<=bx@&ovgG~8Q0XAz!Jp%U!M zqT8=sJ_pX&JOgsV8Ul@9I(+o!`7WJu3sG=TgLU`HFsqM^KZ7iO! z0r@3Wb2&8%>nW73h){O;sOi2He$no7^iO&L!+-&(@n{XeAHTkK)6sHcSk$bMIF#2zXK2i{)ql3jg++}9X+mJgJ(Xi&{S z!Y7$bpVc}0<#5-67JFD*8fdXZlo>lU(#iu5dS!H3`i&FH z_DMo^+Zp_nV3GcK;pG-6Y6av`{@Onh=RZ~{6L=Kx?l5&-@EboXjEd?8CqXc`$8;-i z4-F`z{ZW7~{ahA5?_EvN6f~f&0HxO-a{S&5MI9D6gO0N%b%sp!8Iyk+^+H!y-g^HeoNj|qGoq>KvM%cKk+(9-W>5~ zjNkZ|6z=bqpjR+_PH!xR!El{y$qMBY?Xsc;CTuG2;yo@c=*B=WQ{#rDE5Y>OPA5I4 z#I<%|K%TeUlSG*b?NUx*FQ~jg0$x+6B(3?|@q}K>`2um-MN@RYzX&p7z%__-AG0?EQUCom@t`SiXwmL|j_Cc^{1kcysR_ zy=BjXQ``cF&6C=1s}&@QuHAX)@~n1VEf1*@FHS0d}7^>HVsu@RefATFv>GjoJ%hO1a-g_7%;6{mrT^8 za?S*SIQOb3s$<<$B)?wow*p8Te#ER!Whp7Sq6wN764IX0;{zcPte>5~Bc%z7QX@O9 zAtm{VDxjRg%6lTyATws@MLeo~+*T(KnfUI>>FCF}>CO9#dFrjDLFhPdrNNx{r;bhr zkywDw%U&Wwd9e_Em8O-RPhi??2ZyX=i z0eZt8<^fW?66qHPvCP}2HML_3+Sv>j4i1GdhSQIE5@TM{#NbS7SWMB{0^P-*%U_J5 z3B(s&NEE6u6W8`hKhvr-se(hdwMDZlQM;EPQeG<>cJGWwYg0 zw^QEp=Z!5nlW_Um*IQ(jaz8DzLr(6}v|heAeg6j%B_2|j@#cuM;i!}J~X zd71^+yZf^2JJ17Q37!G9m$vGx9f7Ox%8{gV#F@plp8KMfrJ3l#j@8vg&@2+scZ zRR1X?=sTqQuWkGvpx|Hg|NC(vYJ{t_H<~|?UwtaakEii1vj-}x%TJCw4z2@_v^dk;_^}$ zmImq2v?4}Mw+&p@(eu@wsL6!9pGynrYr$EV>GuyiC{3;2`*+91PS3bdm6n4{H<#tM zu;co@{ju7u_gXfo1Z+2?d-o_GJKDM5kdqZ10a_`bGK6^Ji|3ZRe zPyy%G1gIU-Dzpt#+$pW<)lc6x@NrbD=1-ihDMmeS2cBmr8-~rf_O1O@w7%>HdtM_` zB?`yP-*IRaErMFB1aHnV2I{TjDw>s!vVk;L;)1zH7PUXc1<|`6tZ0iwwjwNh$QK#Y zIB?8$izVr*6)b2`^2lPr%gWfsV&OojwZaz+^PqqQ$lF(|*xKq!KzBqcP|~0!GbTjt z7O$?h6wgU4&2JH#s0z?dvr^MXV!p21I}=3iBK|B9LrIdkv4zNXR_+m{GEODp*+GUR zkpRk5z7}9oTF(9Y8gf_2j5?4Arj3kBwNTA`)i`Tx_f%Dz6*95VWi0ZwWZg*X{zx-j zYaI)K`tlUs+lBDEkL8VIq_}$!CGI$S+hIyd(eE7$>mLmfj3Y!zEVG|xU!!K$V!lvb zE64b;;ocm2D>6+wy+4n?gIZ=Pv| zoetl6EpNVTLO3;sI073vjdQvPMrK2Ru~oj@>WZ0H>{ZeWLdhsfSyYQwI&;j6rm;Dm z>~1bQqp8O;r@dx%+Tp8uXmnG=&_r|8$!y-h-!YvoM=Tj70e?)dv zMRgv**qQ~7k7YbS);pU-{P6vfH(q|Pig?KS({590=UkdT$HLoj%-d={Jyx(RH-qQe zNsRGjxG8(XyN%S)zmC;02>+y^dILro1^@_2y)(lxUluj^fevIJ%QHtXrw)DE@j&ERh{{YFjZP|6npq;l_f5koKP+*$tVDuk|iakw1Q zJYfkU0*I)^!?>t+^{3!GlQ%uWE1VVp`WM3apOJx#bvEB0_8QmcL#AtvBYhjk{5;(~ z7}zTV>S5&y3OFm94Rv4aE1NA9+%Fw?2uUD@l|Hiu!&1tS#xNVTwZ9;XA-VI~o#pNa zWZJgWw#xnx+ecINFzKtmI*M*W=r-yrZZ(i$W>ZQxgH`6=+fhj;-Z9^t;~EIxpk01M zI6npvbzFtdb~^>*khku}u$d$Nm^*!jS#G|#1w&}o&YtSNXgMyL&p4B$RfN(V5WHl`h00OAfzp{K|mn1s@Y6Q3hD>RdPj{Vp(N&a{78d+EmZu z%5C(S0#baP$#b?J(9ZDihuTXuo95Z(Nx8kwb=d%AOqI;p{@1_0Fr1_)-jm{h4h z&CsNb-eGR|prN*0||ODQ^rzWjtqykmfnCpS}rj0 zbXHA}Xhf1A?uH7jsE`Q@Lq_?U+R_k;^O}21(s1keLuBz$sSgt~*1kM@I0gzLyxIi6 z$f8o=0I_|Ic_$$^;%8V z-egjU!&3|%pzwar+6SQ)he2h*r=H@2bP1L!o^pHAaO2ST2QO1JOWg6&7C2g~1y^3E zAlmBcql{r416j~MQXVJQ0nRyBLM(AD9iqUe>PooJ1|t?|>I%?5j!W}n#?1zMv^n;A zr9w#A6HcW?RnNtzQY6NLXLwKb*5a zk;#_+G6ceNiLpnCq>ZlFXtk6vVEH+mTS6>NTRPP+OGn%B?%oyNWm8cm z)DzO#sJVt^n2l(l(Y)s3k5EI7j0`nu!pIAdUeb`rkE!t(D>dC|`Zb1OU8&jiqqt!o zLaWj-vLn*mv(=9SU9#0PF~=ny=H%9>>G#50AlU`0?~0pz+A}&062o%Av+X2OE6LUg zW?X;;^w11dFk$nQ^zP8gJDy_MUgA(eS1iDd4m8#8ysZzFQY0|o4vcx~+%(+Hz$Y0tI zymcCL8~fjKk7)PIx_MYj!6_JNgKtATvpW$-%T~g4xS)kY@Hm^ZFr_aRsP?I743(~6 zj_R&z2iPG33cMjC`WSVED-sKpQR#P@U1ghaavN=dutnneG)}diNDGnP(o*9mCtlKW z8)+C!)%3Wel!_~FMICe=wHoDvZ9b04Blo5ij(Ra))N^VbWyTzQ$BulpJS^^l4AyhE zz3_!kDpcQQ(OnY|jOiB({6;cN0ouBVa$3g`f23Cb?xAbLflga>A>%of9e`uu+$xB~ zPf=?xWe((>ZDCNQ9r}a>D33~k!&;Cf;I_2KPaBEt&7g<-l=P8BR|%}AA{J+Ns4&XP zjDrW&BU14|P#k8JPiYkVI0XSf}PTgml^=_JOOk%vC=JoYBhQRLNzxsNq0Hp$pv%k!ngBS0HP&mHOeOsET8{=q?3CW27Vl}jbc zD<1%xIFsA^Zn|?xE>6sYMuLG%NiSA0^!8i+D(yhykcm+`Hx#~E!{M!*cZjmQ3h{>=uP+OGl^P4Jse?a`uf_<7}=BnNOF zp$0;JhDt{PwyTx2F>RFQF^`^2B7jNw`o_6pnvo!~(4U5M1;(1eb5@%fz#(~Ax1)e8 zxH0-7eFt;!AGE8Y6J6Y{g-R+UWR%}G2BuDhGUs5tbF$xT^*Nh{^6?P*!C!>u&bJ*MG1`y z8$LHLjQZgLCqK}sYS&D$EROMzSz=QQFi3jqn&NG-Rl$#T=*N4f(Q41klkAJnm)whA zqk%dc>|;QFJJ)~38*VJY3M&S+S!<I8DIK3*Il%>mN{O<3ykRQtUJ=)GyjMp6i@Up=)rn?#^kMoKk! zx$o^O{n~mI3|1IJ^HOikM|;DrCdnJ4SqfJkx#*O7(OBa@au=PbWrC?F*~dU0%E=Wc ziKipd!2p&&u~>&Nt6-Tvxa_GHMnz^>G@SU36IWvww?+B$U^9-)!SwXw*V$$uVcqww z+O5OYKsykL-ENT~e76SC-WcNGf#BC&v_%1{n}ulDo+Y1S*>XZ+*$hEJ6w~ZW4V>2Y z3epeEq)_-o0x?pu#*qC=&+=<5{6C9S?)cKZ^8N;_~u! z`p&X+FU%w6I#6}(CxScIyiC?K^~|pO@)hSpm4ef$C=apYHy>tl z9|4!oS-{kn4lL9lMBUG;cfci z)Mm}$7tQDp{?JZbWk-_+CiU8khcbLTjIo!z9AnQ@R5K&qN#ZJlA?MzlSxaWZli>qE zy9Nt8zon+^8&XKVxUaK>PfI(RvM*leG1D!Rs1wB*@-6qkwh9wWd(b%u{ZLg&I$?>%HTyP&=1gf-GJHQr(W zk`Cc{bcTzh+Y6J}Z^9!V68G$_T^+NpnTR8I#Y?08h_f210)(r?C^X8t25#yRu#Z!{|84`*jjjcCHV^LeF<9v( zuYGCdHg&3gkG=0)A)w8mBVBp1gmGG&em{=E z-zgUA0-Mzv=0;cjrL#`R=X3(I!wpdR=yI_9^e8%!-DzXKq}t$q;QaUichb#Jb?(sm ztf>!j=myh)@&{cGLy$t(iTWbno(B@gDA1h z1^gEq!5T3-Mo;*_+_KAjI8r0oxHN%vZCN`dq)l_GYG7pT9p)p*(ZdBk!XmhnHdia!U^k|Z+f(&-OGLM~ z7;F;t_%!6I+O7TNbbbWa))Mw*%mH#jZ#jvCmN5pJFJI?4P0+MM#~fH4Cf&@5+hkxo zE_)>RxqJT$cu-y5G4@4IPA9i})*;;^iX;I@F;%8mf$fM>LPyjyw2t&r#gbrpuB=C! zwA$T;l$=q&37WcK_QrU@gG8`XIc07 zAYSqJ*sie2ax<}lX=&hY)DZG1fJGlbM)2^T}u<)|^5H}XjMx@ViMq6~V z136>_vIVhiZ~Tp4JvEx5wjXqo$u+Wh7;Ke;gu{u86DGc<`?ABytkVE`q`O=PFW+6^b{Fj*8@F^2)1 zK4Shf^6M>^r7Oq+H!~C+0OHdX9TrMbiVZD6DrQB{ z-_d%&elvNmh?}hxDHR1C)P9VtV0@0Z`C*e+a*B6bDzSCfIlI6vCQRXQ1J|7x=d+1A zaFqaLCYicee8~1kg3)d5@#~f@arUK7DQ(_=wZxBdP?{FTlCD%%VDAe6CD0%FzhS9= zIkEo@AO9CiW%wtzhT)&w8is#zYZ(5?tzq~lx8}cK(!crZ{|zQ(_$LPE|G-KAa*O|a ztAhWL*3QI0Ps_l}fzQaoOv}c^{Fj0Jzon?6c8)gxO1xqC&U055QK$Vr%S6_|#sr_7 zmd@0|%*D~fiO$H*#$M0Hz}cME-q@7lpA#i)P3?p&jGXZq{>kO}d;DKn@3Jm7h9-{q z-&6iaB>Z2IKtgubc8*H+21X|Me;wMJnc6#=8Gh%>e?M|@b~bUe#itWCakVfqQ4|yW z-Z3vPKAniWvzU^zf%Csc{Pl+DcS`-gMvF2td@ubs%BB-#X2fUs7XtqV%Azbx-v@uY z>Hiw?@5#T{{MVtQosp7>^S_oA7X9YWChpGvtfu5*=X>~b6OOx;G zr28k9o{^E6<-e4nUTMnM5w{`qT&O*e_&b|t?%=hlhuRSH158o%gN=oSU`s)XYuFP@Nq$^8$Z%%5OM2{v-+$q0NJ+F1vlhFMLC0wZZ(cWl~36iu=t z2o*jzaunun%#Canp|C|+2pqxk>&$|o9*lFFTDLvzUT3Jg^6e<(N9>VOMz%V zy%-SWJMryuSmhXy;$oypO)O63Cd8k|_$_Tb)tV5f*>kF+xFVxKmJIfMG;IgPu$D168WEZ)#W|s)%iX9#PW1d-n_CX(Mwns%=3D3%^?urr=LdFty1cw1D(#=AZS?7R zdu^mn!Gg%KS!Yu$ppxg=21}?z$&;APD~y689U$-q?*}g>&Hy#x}VI9$IMmeOqdt%H- z@<%2IPN&SzWz-7PD_n}0r->1z!&+AH26MHVuOaP z^tXz<(5MqZ2Dqo+z0q$*wogKkibp*B8#)$0U(jvZG$2kr4HqhRs)T>~VL11ZC@c(C zNmy)A<3{bB2m~obm-hZ)oc~M-I}ug{?P9>y{h^y>{C1m&uq}#*_Ip5ua!tkk9!J3a zJ`0#a5wkKqFh&P{5@b^g4Hb*?$bfa?mk2tqK?6N!H*f5BP*eeUx1t3GVfAQPJ2Zk! zva$t+0le;HjBwAc>h&rnHj$4`g-X^yjkHG)P)h1d`helNxk;ER(1?9l)f}-n6izw* zgR3OgaVRviYHy^qIj=LbgSl;S_~^wjD3MfeZK$#Vvdd$0u`Ap3!1OPd7_I{cv;1wF zKpqC8;L(xq=u0G-lf{y90Hsp~&9fvB;DX(h8IT=)O>Y9*fcsKt)E(e)zQmx z;OZRQy_SOfsoppu%lav@E-i%YWX3hrPJ+t*p6YrfM-drwOH;e|$J^njcgV2rt1i#=*A3T&+wGD=ftKGb%8D-K|b}{SW&1XCl zOGH&WIm?8(m>&f&M~WXS*to_Oc!}m@p^yq^PC!pZio(wAO-MI%dbJ^y!(*`{r)y$m zo#Lj2jZUJnP%yKigOC~g${i}K`Ivu|7HB2ItgkVrS?xPjwyI`vXxOoqoJ3IzkkRQ7 zLs%bGwssq%pQNvYBoyCkf18=kbV3qRRpgBG_u6*_NS2*_g>4(94YgS@H8^{!=p(bR zX*y_*VjYeh)zB_wudCzNHAfC-_8skzlDHb39n}no-IkFs%(+S380>7IExTV6PN9*N zyJ5soY29E{iRx-qf1o*zMKE<#)j%$GCF^~+WxRn8Y?IU~t_7^Y+O9Y{6qL3Tkt!UxMq6MxX+h8dz6&KQf51{68l555@&Zc%EVfA zZ!nuD)$iA&O-^O6=^g4PgOJ?9CJcN|#Yna2+@ROTnbWA&salL)-80s%*qXcaP^bBk zASxQxbO~Qhk$b#9R=G>2tQBVrW-@@wHT5P>obEOQZ1*)w=eD>C@)2jCi;+>7_j|@<+-@qiyo$JH=5>n5bO#6Z{vz$&Z*E zO4e62T*#J~j0L6Nk2k8y>G*nmoxYJ|`(l zaMtkU%)M)>Kh^o98Tf>1jk?Mof&$B^lEazTCy$Xa|C4^_aPYwo+kKa66e=?nAD@U5 z_Y*FGa|!9aIyjM$PND+TWC2ScDwQOTm$oikK0EI5?cX`%zx;Tv7EaCMaNtm~=) z$gEu;h{b*~0UvP7SF?Ith*2@D?=4F7J=NW<*6aLrR9W$}*eBff@0;MUHAvNIyw9*J zeEcRmwZ6uli(iQ?om@{*8(#o>f=0yujRyYfdHw$+EScG885tP=?}VkfwaNdWE5G{- z{i{IWU%K@FUtP)YpLHeU-+lA{)|HI^-|0&F?^2BaY;XG3m2ucC@I5zmr%(&m1u;`Q zy#mGX{P+#VB(MZw=D-rgm!aO`2w%A;wtmUqs?ycX5deZBBcs-XcC~40)?{{aZy7&~ z%5opM8scsIcM8!e8;Em< zDHA!N|YRB0!7~E82=8Ex5c8iGGHi~q@eqGS%b-bZLvVsAtykoqZBp;vA8Bi zX?7+8J=tosjcMG=pxXhbbd52%88N^)u3{7jQXpOhN}l1m!MI_T8fhZPDdenva(%19 zFg+*3uskPXzgP8jfh56M(DK3=4Ks|DD9pB4HcuYAJAvOv)8Y10)H{O#$>Hc3v6SES zu7v+-p~O(Vwff#=k_tL9RntYFW}#K6ac!kzgTN#F0{XhZ%R?vmhO|zQ7CdD+m7!p# z_37os)A6fUNj?lcG@)7k@CLtIMmx)0%z$`Uiv*AI)!o_U>Cp|eJ9-Xu!SV4VUI4@C z(b3&Cmb=5<_wnG@K^5r<4rDH^2A@(fjeKqp2vUKXIy~7tK}`tk9(Z2hez3WO8KCCv z#K1KZVVQ_#1et>5)LObxGGt{@bM9a2uyVL-RuY4b+mj~-1$55bDOyF;LDh1k0*r<| z#r&Bg=1|05A2y`F(hr@4C~(+V3r6h;z6c|k;huQ{r3iKjX)+*|W`T#-;^g+2nU}ao zUhxKrGCyOU^TuaK2?^e zgd7rlg|;j@g90bB^!Q);uVsv;t!|pfwgbV?=-r0BYx+`A4Nv!o*H5M-JWf}!Qr^-7 z#6%P|qc+wNy~;fKY>K=nqm-z7F>$;oVT02 zc+pWI40K6P6Ld99j4@x#j8!r+lQZ1zkG+04xQc?a=3|iXC}h~1B?72HW^;rigY=>C zooVZs1zT)av{L<`CXo9ACEo!YN_&@5N1ml_J2;JDhN_IZ zl495r4{Fr1=Nzg^a}0P>DSv}@Q4WGECJqL47d6E+H;T|TO$#L{a+~6a1%(+~ejM~( z4~mqfT2N&@qT6Hht}74CK+VYGx5TGsIIO6t!?F#Y4gVddYMwGvK#m_j!wXCEI|^h= zm?l4rmOdR(Qb?t;=uX>Hs-|&T6rpIeTaB$gp4hv*N9B9YK+EqromvXn;zl)9kN5EH zggFJ2bvg`Kjg8j9$BlMcF@gM-No6`)^#hfqEo!+-meGO12qAK!e`+u!<0J#QBhCOOXD=lJb10TO+HGxcI6IiL_fQb~yE9hEU4tg$XB%Fvp0FrSP_{dYz z_>|C>OD8rRak2zcddAI{(__;p*H?N}hAtRbs>O(tq7)GPM0%Uo71=gokkMHg$E*+C z*{pG&A!=%9`bl)}tYq3d?s~s#Q(5iXDy5=@3|lBEvkpDu!MNK$cdZ>6BBd3C@~iS& z!8(t%z>rXryxOiG2dii{U00SYwn9<`sL4sWN;w!+bGcA$FsZRsGiHZZ+unt$kk+`! z*hL0JX@Qt*S=vUnr_7|cbHrU~{#qM`K|KSsR7{4^UZ*TatngJ`N}U`fXELcGflz9- zP>4TXV5Nb~A#F!t8d;LS0ydpD+>R`Hm}r?M>TTv>w7D-YVM*+G)S%93WLd-Jy>H%a zM`o4nMQT+|+G)jRkO$eYh#@`#qzsk$wkPKt!9Hx$F1*YqJSji8g?M;t#P~nW2Vsoh{meu z!Hd0dw<3pGJ|i41ovSb?h=c)p3v%Z5-TlH!Oa5R7S*ezIp(}6cwXA)pboiOGw+q73 zYZ#iH?apRzCwj}>N!(W_`tK)O&z|1?sf)NOEumvm2c(q$hrM?SvNg=Mw9~e^)3%MB zwzbo?ZQHhOJ9DRP+qTWl>Zp!BC%U5ly7=$9F4o0b?~3p4i8aVEUOV5QH^0FKbnAmCKPv^T8IPT9g-;H8sC86^y=8Z0GNmQ`&(?;`tRNFa!r->XN zq*kgK5!#sx$E3lapea@-hdfc?KUrrvbbRC}rvs6}Dc=a9!h73Li=3poaZqaxG4sz# z?yJj47g&Xw+sLPpCwD8kqYvNbyfvQ~>SZKw7FCF zeq9bmF>seGh)A2_HT%}JctFgAmdoZQmvcmwQ|9gfdKEf?p-v<&sPkfVFUG|7i{$up z<=)q9Pv-c%zR#b#wA>$8;CX8qKlUx(DYy|Ty=uZm%wpWz^1IrmvObIV{*C|szW7Kn zDIwHjzBDlzl3K--(4YEKc9ot~l@nrQ#Cb{=WY@CLYI9b{ooeFPfWU6orEqG$=9#zh zbQYR!IMJNmnbthsF)n^?Yq7_ew;3jQ>yZ=o?r(p$ovIiPXr!`{HSd4AteZXFK`XE{ z`*}16P4meOm*qw4Eb`+7Fx2ul;!#Jl&PLTo*=hCrRK>~VLU!T{@Yv5Z=wGO}fAk&y zmwNk`jlX|3vzR!35`+G$=qOSbPsrp%>UvjQI24vcgcSiINS$}}sNmT(aT(xR;hjlr zPiK%!8@$Zc;Y%As0d>@yj%Y(!OOiOeGb-=fv_rr0c0;P}ny$&={`?boDg&MdH+c2t zoHEKU@pqmR+7aox8omR5t6ORlPHX+e=5F^2&x3Di&f}3GGCrSqQrmc|_L&IPqh|f+ zq}w#Z3&lO))mS6xx&9)gzWgpP@E2G0-y%9%6Y~@DyKwcbv`(9hdJcuhW($j7?ax7M zDuSooX3xBqZuASc-rTm+|*W(^(m*};2JI$ZJMss4+*sD@O2qT>(BJjges$UMp zbB&-n2LK!LFYV`BcTa7aHvU@bee#viVxIB`uT8#aLscOH2lzXO5#UD^3;Yj>#a`qY)b(m@x(g(bgF^J8v~y7+#_ z$IWH~r+E_ocVSD514Atz!_^4^-Xl!bM`bj!5Z)UlL{hv^9=eau6;S?(GGtIOri|k- zil~r@O!ChIh5n*d0f!a&J-9fjJ0UMZ^63!IjAy=MOa zGES>SOxCGeQA{4xNRzOry(#nH+tM#@{zfsGm11&J+OLZ;OQQi z8_4m3zl@;Ij?Ex545P?fHSt?b8S-iCe#Sc_{mH0{Zyji>_R}z6DglotWPl9@+5kC~ z#IKA`MSGGYuS)zVmBy)9g%Pm0Hp;U%QC>GQR0aisa6*Z*VOgFZZ*NIOzJpJ6a=Gyx z69$M~`FF5dUGppp?anx%zAf-_yQtMZ9xZbmCArx9?o2Q}cUu(X_b*CK0m^fNwahgi zO{7ff*{m`J_~I|#t3#^r1H8#+NSb1q+e#Dripv6RS2d-ae7yFZ7ce@JTp-%=CKDvu z!@8S}Bb~gtz(7TXihr4p9=1}azqod(`=sV#XZy9fxyeRavPap-Jpqq_72@nlYQ_9`5UFjE~8IP_XlN&}_$Tw=AmeVUP>Q z<=kb|Q-U)e!!Hgy*5R1I%HIlz;F zwipnH83ur?FC2jEz>TslDY?okf?I)F>)w-n$Mk`10+wLN>eeeDg#F1On8l8e_q8+C z8t(1GTxM>~IQb*^eoNiy{~45eic|ZYLBU@ja!d*0T1dl)1Zp)A^_4f2vES)SH3`ViOCZ$NPH9LDXtEvI=}<$X98UVZi} z>yR0dKOvsi`CXqPWHxIq2A zeTA@V&_UJ3RH({3pcc#1WR!MX4x+#&rxK`(qCY*~MK3dm#pFzFol=L`2!A2Px7&O`34uO+?8++gyGCz2--E69{oa3Ni-^I^a%FwRby!j)3zbL%Ui;0F~FIOUxzt z_DU#KrRj{XEoY4m1f#KWOs9;RZ34ouT`pX_%<%24H(M}1(X{Y*j$9zBbaX9?JZi^> z;FD8o$Qilmg3hSVL>T=NSqGm=&tU>$KN zk3QV`!b%uomBva_Lm*06i6tu(q~x_L+bbC`(wMS?4Dp#2V&HXph!FWyx~J8z=-1V4 zDe*v3mMH$k`3id?l#Ngua7XGNV09?5-yI-?RLZQ(l1Mxa;x=5%>!xn%!iGg_(=%1` zFvieI%HRsN%VjM7EDmrTjW6c_3Uv~i`mF)EaDh{s<5@LNLDH8bGn;$25?dw8iXe+a zvO87~CIu!w5JG1$@j}yAwx^ylLurRRS z5e&gZ>bTQC0Jsb~pN;`rD=V9zy(m$b(ttuT>k!OT#9G)WvwtSBzm|r>3tvEvPY>~l z7HXzt&yRD={eEqqT`Va519rV7kwkVq7X6`x2fHg;-P_IMvfrEcgk?VZvX6p15@323 z=kAKX3H|ahxdSy02B(L_IC#pycq_xibiCT9=tvN5X9h;4SdeO-4(I5a81^-9B7arF zuyaLaASTc`UqM9osApqx*eCz5#N1T$JTj4a$0?a`jOuYBehdK<7g;5FJwNX=c%+2y zDJ+YO{waFpH;lgORbrn5v_%O_!v3xLML+Y1>v{gAOo1>d1k}Bsgelb!b|d>ipk)Ph|`sbrLjB z2=)|4OyPn4yM$UW-$ELI_p(pSG}(hVIAeudE;U}6pB76o2KY?K@@#BN|6;>-~Tt7qi(1(O@hX8QVg6oYe{N&hL0ug1h%98T$XN383 zpaRs$tWSh~{%f>PtM#-jIzu^f>YFf!UFK z_Y(tp(OPU53H|xrUJmR!h6ninitUf4xsj)&&!*l`fD1whEn5P;uK30gl1`t8Xa-dT&jN}3`wb+R>d~rp(d>qCgyV+= z`zanN{TN(dG0_rwopDjtHN@00{K%}hY&rCB&f=q-r6alMN5!c_Im;M1OG~&w(FP-X zP6N+_ZTiOcR${N~ zOsvk`rUcE(`(FX+9O%?z{vZ$fB~GM==M+)ezb2lM0A$1o7^)E1KP6_u&0-rz6v9jj zoGPRq?o`$`j*jn|N6=FI=BUR^2DjXbcD1MUv4$vAI> zf^@|aOcJ%(TEB)kxlgc7vU-K?Wg*1~R1f$Q7!dTDtt^S4U_1`$W4LAvGKk18VA~n- znz|2O6OSyu^!&zSyD>L%^IX}xdIyC@k%=j$ZH$;Z_WYX`A#w2+4&Xx}Oj!x=W3Y(o z%P?3VCbqV59QY|&3R*u6pe_wLI1dWa3XPgFJ1HCyD6MOrG&H`v0SojRg3pI|JGufQ>XjKONlaxM;wJ{^gZ|D>9*dVD_dbg`1n zvnmw;pi3fu@M}hH$gg?uA*g+QoM9UU-GLFWQ1pWq)5R0$h|D(xeMN6k>`YKb+_@~s zn5UMc+-sf4@&hTJu(?v6^VB(;o`2kWWO4Oz7b#5GLu3U>{_sl0tk%Py9G}-53^-Ys zwgOa&(dnYvUW@PvaL59>wy(O)zP*^mu!;W)oLN01ohd`>3 zAq>gb1R7X&2@U1cl&WimCzUR?@gJcs({&!7>2dz%np;x!vj+H@VB-Rl!S4X8wL{^( zfG?Khxg3fI!VL=Q9zx5TsCO;>0Ay=cF%}eeTV0rCky|>)OauWbUf5i%ECoHEr*Fb1K^nLf}JLwZQ_jAlOILDLNeiXYZRcbZNAdz z0By*=a=Pz6;lHDmQaKh@WdONu_z**<<|1O@$Uo?oMWY`7#ijfo?=$|LOF0|!fA=}F zQtq|hqetmHrs@olxb)NKBKCtKScQyZzi6ghG_Q$GwI&wJC&qbuKEEYuwe$dp${C(! zdeXh}?zAK;w|p{2;>VhHIDzd5D}FcF1fCjUA|h^u2zaZtVtjI^)rmm{%#%1wa*QKU`n7TfM*m$4GH}%@V@5}X2mVGY@Z#TF+k<_a<1(N zVO~2>_PqMA-InVSyCb-TE8}T>04EVF&nyWLdqOBXBKvImhk#&W0^LI-$88?LXepexzz2*M(%k8lg56x$ zy;9$?hf1(X+H_R7r?t|BLjh6_c9EwwijnZ|n08l}kBMV(uPN}MoO(c?^z9?@ zh#I%KJBw_`llwvwURlmNx4F9M1R2llhmB)#I?wFJpC8ta#Y=vU?%n1ra>%mYxr;WA zDNuQ3%O2b$OqRJEx;>vIht~sL9^TGSkE+bY zb2+L+7ti_I_)BWcU>llaHS4@zjtv>^PJ;)B`gGmRIO3vtWFvpO`24)tO@A-W!5X@v zcx;Wx@}c*EFi2X|?!CRNJ>pk$Ks&#f%Hp9dX500h=`?Tz8{VgG^ttginAuoyjdML- zJABD4K`O6H(&^bz*5JKQ zt*2)v1R4>*jD|)w_Cw7aNP>8TdjM2#=enji3RpjOWKlr@c`1c8<;BjWIjZszMP(9iHcQ>1Fn3sFgHl^0o-8}O=Cp4Xcp69aJ-&k}sv|5^2sKhU)ec9M6V7qeakQXItv~v1 zXBfm6=qn{o4!2Qs17S1>=}%-g#YuCXjzs$q4m$zNHo>FPPT&F)2w5T1+zhf7p)XTd zD6$`N)d(H45a_GGbSg3)R($5!4%xu;EEz%86)h@R7P~Bor*+~(n|$7MRQl^i>BzCj zATJx`ZJYeqv5341%0p>{%kZ>U_QJ8J?gs_-P06cOcH>Jr8BO<*cnGho`_X+NQ4SgI zJNN0pvG~vb9{P^OsdC7wys{mCjvR8x{{Em7G_5$~mCd+&pD3PFV(I*s>3Q6x+p$yH zRlH_e+wF?;ov{6QiO11(?`%j@=Rnb;kMh}0_Vs1kZ`J|v19%awJo{hJ@?SkY|8M=? zf1|YgpR^#reZD`wkfd7wx6<;zX_>k#qM9UrZ{q=VadFeojxIj+!Bk8$&*b5&OR1(| zD>ivv<9QupS!ikJQyXJz zm`Gl+3Tp%HmUgLLKMD&Ioob`0OsBMnp|^!A4uD|AHM%@6J_94M1lRg0E#>3h82^Ap z7U&0-<1L)!$u8KrL}{JNgSKnl%&rjKw+-duiVPQLAF2r%OWKn8UVf1MUi(qXhL+FD zYn}B1y?L7rEle9cvi?GB=sBh4B#xZsqSNNd4oHQm6l$7qBi;0~?zTXcizgzMIwjs= z)zW;U*s&RX%z_@8W>Bf%bSMW62pmBCA$x#DcaVPQ}0zFu6qE4V_Q*byOWM{A7tWEWkM1l5fL&E6WUiSNa z(4jUw6O0<`A6Q;Afq&>A9QXs_T1%D&Ah{=yWBaQ$8;}d0(E;c9_t6aH2^I?6A7Uld z9tZVv_9^UX+>k3>P`W-hWW*2XJ7soWw=oJTk2RpJ3_j*gxi#J!8eg*fs?IV*bt~qv<*lAOEQ?dD98muiU32$Ku03 zRpi!f?jLO2xXq0ni~pmZ{2cw1ni|h+#~(`Ua!RCLxQ*tHDKfOKCqC~s>2Ph3wP()@ zRJGfuxmW2MFurU6^_~Ji0>_0l} zKRWFHTZjG6;irEOmjCFm|LCy)$8{LvziV6cA074|9rpjc4*RcbcK?@4%g>${+mC12 zKliVG7QN^>enL|J`Sl;q_+Oh@42%p6?EkG95Y-4OhqHo46OO#GiPf?rV;iE|8vtsN z8&_-xU){U8DR8rMg$8j2;_~xxH8j4O*m(C@^jK8h{QIQ6KK18)MxngCAv9w(4Z!H4 zM}``w^1>fLoP-5V27rqTor;T#jPb|vZ)6ngB?U801^Q$Mq8=FfD+2!%fZ5r-P8^f# z?@i)G4*;kMHUK!fZ-8oSfNX3QZqMix?G0XjRscT#*b1m2n4CT+`HG)k^hjg8GlC0v zY*5j>WZL%!D19m;5IYFSi}p7n96}2yr>|60F93-FnPl#zq_Cm^wr_hD1;ok0mktF+ zAazzIzQ3hqb#-;+P;+#pGaMRWWDQ_7KAEI1)(qmw4iGc&$1jXrOH1ev2Syb`b>Co; zkz3i0_N=yqiaKC~8~s|qG@#ySI>aemrd;5iB6fZ)ZNyw4K>u%IsxNU@*q3ul0Gvu3 zpOSZ~Hy1*Lml$0GMMW-+P=*y;kY<3?083v0R7_zNGPaUs0E609oBX<{f=$WJx(J4m zp_PN68`0w-0L)t#fc+G}Z>;Ir1warxdqXFX)sJ6smoDikipgLZ;%lpGK)o{2Bi-)_ zKpX(lI?_cP-2j`19vH+zcZJD7U*^ z_nS=Vm!&m0!b`(yisGAWKdsLuU%yUAFT~5W=9n{GJ}3~z*K8{<7ktO*IQsB+!`wH- z>o@YtHT5@J)VGZw;}k2aPb=ov&9~3SP%}REb2kKCx|6f}Y(=*+b$Xis17 z)E8e@$|Pfxw;KnJ4$5l?^h-Vax0V8yrMbzIM?brpE8q|u+|bp?1)ad&!7iYyeP?Ap z=FBBLEbxk2ud@stfp5xfC*YAY+(@@gVhVf@>FfFf84^IP_;=I`JD{cgH@-GN#vb1Z ziOi|Zz%-n$;wQKr@Cx&n2oRmrz7J*ROJvrUT^9=TC-$u^;L5~rI_!s*-6CKIE|ZXM zSey$wln9^R6vt#ALF;Vv1GqX*x7shg&K3S|e>zaBZ$BKmsPBl6`czBj??R8=uFmyu z`|d{I*WphjXg4>mURM5lAJV;}m+`lfn{R_7cMed9ncc%E$2Zlkl&fzR>|x1@=$|Kn zf5E>B16ayb`>;OiK+!unyOte23`cj3?DM_EfTKe=0;dzwO{}GLK{l2{TK=sg=*w@n))~?cVlm<{#$RY z^DdDh;&0NjeW`TUCH$()If_(agioK zkc_$dP99$|qv#UFbY?xYFPa8cVnBun8%Ilkd4c*g0+LlSpdAm?n`t7xTi!2r2jjhd z(ryJ5GWjcDxVe6a@tLUCU>9 zs-uLt4|RoE#Vr#=IF` z9%G*BS^Z5j&PMPZj@3?*MKWiQYTV(4sTL~s0HL7Z3QJdDNi^%$Xm}a4NTpVA300sV z8^iyewsE|JPGp@<2IlV1N|2f)%>oI%T#^-%tz(0dK8kFyeSxbO45+z>Z4OApHY-!F z2ktojJ2$*%_}j8}l{(nd+%r6O>s8`~J?X&> zKc)cuh5$YbE>G$ZzNk@?nsyEu%zY}kTKxsb@Pej9UF-BF8pE?QJ$TdQnPd@{og^Jp z8CtWq8u@2iu`BT8k&vJF}Z?S zo~8dhkf@PO7%H1sK?+^=P1lwa^;U>{BV@DO-J-cnd@>|$&Tg=i`_eI05Svl4>Bv|X zq!hX-bXpbuIZD&HVS?7(rSSECZ)BY;M z0ZO|CZF-6sHfvS6c4MSEaMVe2$)tl0<7jPN(5ZS|xBpKv7?Z z=qojXhg_^iXectT{UC$`F%>hY|x zXuW)kJ>Gx`F3VQoFZE-Z2Ev5imQt9>#lOHf*VotO*$yCv%n;Bd{HoSo^CeWiX1}& z=^?DZT|h+2jI8#6*=i6CLy4G2)(%A6W1eX!_iqj>RN%I$BFViQxEwaoDuBjG^xLtP z2-L0_l+GZvn)rB_y$<@c`K#q}D}6e?n?0==GDO^H2O8`@OlG-DOiG@tBWJxkhw>RF zv*-G;w0uaaxd-+rf#8f`$GFtTF}Z!`yh(5pTuYR)LMuz@(J9$=^|YH|xxtPhGFc{} zo#zmF7KGwhwiPS9IA(VMCAltvU>cIiN@?pB4b;TEU^iT1KRUlir899oZIDz6&gLTS zwO_d!1nj?!No4TowBRP?-vlEO!nvLRSwQxUE1}S&aBR(#2g~b{LV=u61zJU)L8$_L zJUG`2+UXRRXq9pu4AaMAb3dk3VH*I3X$B-y=4#;!HkH?0r#jv5TuT8B-!lBZ5;`|fV|CM^@Sl$Z z&Iw3ThJ!#a+lK8nG z#ROydao(pZQX9YT)%$Jj(Y>+N*WDT)wV~Itg6A-;->8tp`jvp4J8_XqR5njzqW&@a zP)ldj>PF3Wz3kC|2jeH@|EI+{&R8VR&23NOxt!Dwv6p(segB79yiv|h=>_s^r7)fkWuOg zF<7LCyfAG^coDox){ZiEV;7K2QM}+NEfezvt8_GMo{&^3^>*3ul4+qMOJhF5k zi5*h0uU}W*OiXE^y^cpkyVd(}6Od=W1%3nC@N&y$`&fl>Ic+i=BfZ+xYwvQ{IA!V5 z#hvFRG(vDd4XiyF^bJ6XyenDO=-#(3T#Ww%iCb2R>(g)4+G2;5A)NLU9D`!fB<=_Y zAGZ2qaV}1e$Te>se`c(dvgx?qk5+%4)A9pj*>R`f?gPtEmW!f$LZm|XTGx*$!cPbOB-=T`|(MCtZuiZiXDziFa z(d516A1wuLYT-AWnj z!Tw3UY$>Q=a{jFj9O5ve0!tjIv~byD6gkanyCN!X-G%sLa=49Do&CEc>bOjp5n9== z_F)&Z3dO6cr8Sy{LwOx#72Ya?$UB>CohSL53KHA=p4wmK-q`5&P{2-cx-G|tlr$kP zl8phhiM*V0R7Rv63YEmv^D^juJf!DXjkt>)Ql`3-TS2T1M!(j(;WXM!{xPH|MC9NpHo zs_-PHn$(S?p+>U{OwkPRS=XGzqsE~H)umQ9h-cFvlu4JmM`J;<29*Z(!OjhHwdL5M zA|YT%YmeZb7{VNHahwAkpAjrP?5(O|Ydn|qMNL90m{DJO{FYyLGM_Ciyp#r?_U_&c?o)s6MK_?z_}%1SFZvkSrbuI5 zH`BGgMf6`0D4$$>bsq6)Acb-pS38lx@yPD5%qYP9S9?h9QT4&V4SMfJ87b^Pa{1`r zN}YwIgg>Jq<%TC-HVBr*;uwm3^f;1-;e$_I9@#^buM%PLNnyF&zPjI$h-wTWC4E2EpXoQGfU><0ft zyXW}fc}b;RQ}uuvBuQuc;tcLhYIAqQbJQJsZUn%fV=8zW`nFf#VC>N-Pktfae%;J{ znvbt^;`g)=3ua0xy>W&s!!l_A;ku)GuhU^}IZDk-9(D${i6=xBFX;)m5)e}t`-9-z zt96VUFvq8UyIW;hy)De_!p4x-_1t_=1dnvU9CG~8LeS@5J(Cux6t<@qT48cM10K?) zd~e(+h3IclH@R#s6M(OAjEY;6rg1BplyE3s{NUfR<{m(t-wYQP%sYT#$(Bb)TZSsO zB8hxGndyiihV!ZFBvg_MJyFT-_78NEgOfiZNK;3IAslg#jBZa}M+zc<;0zhazc!#^ z^x+z}T-f+6w@zNYLKE;T&$(Mui|ZK$0|JXubP&nEcxUcYfIQ-8S{)}Ec#Q1jE>enZ zz5@BnK^^Cg4Jglmprnl*@UStcI!+nejiWvjFFK8{UYbjVP6!FyEi>J8nQ_pJn3nyjcT?jv*tBWj+ zXlRPOzYuIwH+oztARWrlz_0bDS9x!sAo0ggavYJ~julzW5_*lqwiz5mxS?imwtKi* z`$Wf{2&46xo`d=q0-#}}PZbuG6^>0YF0akgoi;%EV{w{FY#4TiR6bVL6-aaX3#r&M zA{FefuR?5AHb`r!E#VqO)(83#Fq6jMRm<|O%Mn4|7#5|gp26Rqc~O!r8hsG5Ue8`r zEtE9+PktOJzQ*Li4&V<2j}1b_A3!;n0jl|{_Kk)SoCkl6w?ZYyp!bQtn2J|4d$r*qEzaO;X6aU1c)#PoPVq`goL{x|XVyoKcJMt}B z@#=C;+xuv;YI0B3Mh5c&%;enW!yc)RtCeGp*g9VspJ@K>S@pCvc4G! zvB2L5p0z|z)*hkAqOTSc!O~a0)`=D#F%Vcrqc-OZxLW9ud=X`D(F_O)EJ~HyH_X^F zo;{+Fh{#rtPc&b0hXWZ|vLwg}vt+o#PUx~l*3SK`BEqF@#bYhM{beU>u=iahSPX_v zgrSRnP|nHiiISYaIsaQ?YAM|+KuKi4@D_v`hmnln!HOmMM{Q#7@l`@va?}rf zG!kr_^6n_Y0yLJrqmBEH449?@@%)h^UD1TQ9&vXySJrfC%gKs4)nFaT7w{=n4XE6x z`c)y|+Oow~RjFJz8_ge5t3yjUGc6I^dbTQWL#m%iF{kOSqyuE^p#Ac-f;|LksJ3J`gNjE=Lod+t*nHA9v&%lRFH$Q*wSQt|Vm-J4syNRVRr%(w8&5nmf)0H%Y_{)X0hsL0j#np_w zt?@J2v-pX7gh~VAJWWrQ^d)H=><77E#bSqR1=K6>#;gUW{Z~Dwa4NJVC6t*hi#aZr z!6n80Vre^k z9C%;paCD1g7NSv{#frrNa-5Or2Yi?dtu}NN-3!CF*@f(9_Dmmk-gAAI;p*Eibr-0#oVo} zU{(o%Fp1G0MI_x|)cB>@N7NYVZw8{{uTw4(db2>{KoFg#rkkiqJm>IhnDE)nuUi`9bqOB4R zYU)zB7ii|SH9HS&Ie9JI#mCjh^$m?b%HlyJ0kTA3)JaPW;LQCENpnB}GOb`O z!iKv+QVAXX`I3+WlUq!(hbRFw1JF(I6+AWXuQ*BJG!)@TyH%%fAg@e_A)GII5NO7> zS=poiLjD28L{N7d0T{U{Ys;?}DqfmKh7MwX-584ozH<5&IFlwQsCy2DPk#@>LlAg| z0b!<J)RjiLtq)ces7#SAZNdG#Jd)%ZwpP^CBjg}C`EMkqHkKB6J%?%$xm z$}3!UUh#ioq3}1hL4;?=%-$y#o`af4L(czHdzZ&nSkg2-YfIL_5uPDkWGM(z|ja_pEJ`7e0d)fLc zCBdO#hw*1{uFI?5z2iZqcOduowg82}{sbl*8WHdUup=SMp|Qu zc3_U$Y!3c|NVIg$>G-0L$}V|1<_+P}q<^MVG(prR2Jv0d0q%z02w~75TNqpKYR~%r|*U z#}--TX+(JLUvq;+u!^f-K6*90-gK#fwDICxFH|l@@&LbI-YCg3KaF^rt6N^QbUJU_ zpccX{D;+P5kx08WX_~vSW^a)w2WOS=OzP28 zeHGOePkOqzHd4Q;^W}9)2}GiOJB0MCK!eQj*m4Ve|`E^dYUB;zA+qokJfYg)R zBfN&ZVfBJhN40OpLa_CBYlvmL_^cvMR1LdySOpzjMu(DkTvqxJNf5lMT{K~-I2}|I zYXz9K3p8CYb^2lh+REh^b|%`g4*7V?jsXWC9Wnm23-gMjN;DXyJhC@e4HI_ui}vj_ zJ9EaX$V#ge+YI6&nx&k?41jD8C-W~Q*5mLETem(JWft~gWieKjR5=a~X@;&^A(D*6 z%}2>og%jv+Kl#s=f|C^r)fdnn#;|V`*@C#ms_b;3bK#Q=ROdxkQ!R^@(xcz+9~F&} zF&VpmK8P~s{JOJM4_T}ynHpXUy5p>C7rjaThL}oT#3l+~L2hM?9uW7(Ye{BwoBwSt zGw6}dPJ4{TGTSVTj96!O)CmWJWSaI4GGvcadFlqk#uCxhpc3aRJ=cANU*XYQ%by4> zotuDBB$g>`I?oKDqhd^5(`Qz%mh(O00%3Gi&;W@ZUbJ%kc#>gXN8)pMS#F`Wd{oCK z$vgJ?6#Sr~2IbuMkgic|q~phW3@ix{V>+q=dUf!&c-+gOq@!Q)u>y{52z)VP7M-=l zutZUU#~kI6{(70gCWPcZt4izTC|%?lmGyr4+!}(3QPjB-u#a1Waf}pwOc>@uLVRV) z2(hi5v8bQ!^@NdJ25Oq2k5alke2d)dzId2i5hGfod+wQfu-3~}n1#>7rS;_JjZkB! z{kNKMs0%MzU-XVYdn3bl8Z?Q3MDSG4hul_X~XY`A_o5HmiOGg~ za=LkA0ldj?3Fzr%Jiquh(~A6|*~Wb|fy9!-tkPrSC8M5KucM5Q5a|P!J^7Gnd#~rz zT!{(i1P(?ij+y8srkiHZQ*qJq>+*%*+D+E&G)8fH(*4Lv+bkMSfr$&CQTkKF2Z9+w zJAb|`nMAJUiS1BU?AN*TBc3Kwtr-+`2g>Bi6_m!+lU+UEK56u5A5lF3gcFuc^Oahb z8m-ZndNGf;334);Yk!HSAk;fzi$nxN@>#Q`{wpFGpdf+m9Ip^zuf>%n=yu`Rr0W*& zw0>uF#1zO>ixe8|)17F@usAosFeN6lF}-gZTIa@4N{mtA-Nm3f8&OP*WC{GYn6XrJ zcj78SAv$)RH96A)pDRjGy=V2T3+G(;+$e`d?a| z^QZNCFPG$VGB#`05xs9>vUTeqF1}K~w;cTkHb_ zf=UA7a5^?IHPECU+(dLuK8+hJflScJNqb?x+@_Z4L0g+G;^Yi~khs>RBCf-2QN!P5 zP&-NJdFZe8#0`ljm0#(BBjBSlkv~5cjehColbDXE#kos@y3#uBa1&|PG#D|1_6K0m zp=8r?bIKTUXU`~wnaOGCm<6X;G(;B;)DGdb>54Ut`NoJ&Dg?$GL6T>EKAWZZB64q_ zCym;Z`x(!Og~*9x*Q=YhBP#}Sm%;;Kk*{oM><<+S5;V8J&B5?iET`C$<*HXdyW2Wf zHgdj4C#DtBi@dsay6>sMDN{e_-L+&|GML5=L@N9 zR^pd+$?dGZ931Rv5`YkV;--Q8;Dt>%@kRE%yGIBffU3v_HdIUjDl_M5;b#q>QFo>^ z9x5)GE~p4pgYBg^>SD`()_fOA4}e^B?~o3>FMhL_qz2-{4ntd1BX@`THD3{5W@agV z;x8bm2N?YJ?v^g#5^UDbm6Ui5cS&G$uX=}q_2y4pS0ha290ci{@lfgn30{0l)F79h zpjSL=#ew%5NJU1BA(U)6o~!03j^>=e>8}Ctv0j4!yOg?1* zrrZl;sw_8gI@=cOnLhrGsAnT#58*MLYJtABPwuPIwY1{HQo?jAEh=eBdxL8S_}>;U zbH~%H+RD@#mwO`=Wa@b3L0Z+gOr^RJ^d#lD+#WjA%>p5l!I>N0*Z_fv$!l~47@Ilz z(nwDF4N}Fy;!n$N>Bk}lpORoYrwW$e0n;56D@sk(Q7!h&D>G8wiTOBxdEf6wS?nEn z^cFF1gLNfjDD;jU)#RY&N;cz@mNw>k!k^heV>=sIEhl%)(2L{?$Vqk}JcK!yb- zQT-3b-YHhNF50%-wr$(CZQI`a+qP}nwr$(CZQHE>K2&l}a#N}KvNE4mvYy5o>Akhf zTjdd^f584%`*LvB>dAcM?^v?%qt7tOI1z}KerLrydKEKy>)c6LASzz7=+~l>4FnS{9_2y^L zr=9EXEZT&UL()e|YH>1|_IAqD>g*9#={-KS*Yh1E--2y2-&$cie zcx0hhcN)2R-!RqiMBYV-NDD+zTniD%et7A3<42?=GuJh5kHa?V^bg#-4pOVeA65;xpm^cqN;+4u?mIxUcnGWLdyY5l2jb&KwE z;@M6O2rl3Mv2oh6kDHPoi!N^@A3S`IqFjV)Uv6f?yZgC@Z7!onM&4bJ-nrl~G2*%( zkGcN$gKH5_&=6^#&{o$t3jj|H>Sv>SH`T8GWvj$?1v=T&HvhzMS2mwOH zhG=nN7m#(gr4mUJi}`Sg1dSE^@UW5>cs9__yWoCE=2vnbE;OcYZ?{)>7nW@bcv9Ifk6$H|<319b>I7L5!#skcOxvtiV=?(e z_P!ON5phQ4yj0f+mc@7e9PaEWn%Y5({2D*Lp`^%SHnKMQ)ifvC!S6;2bcNF=a~%v& z4H9`aO65a{_e3PUa%EOnp6_i0gG_zIYkst4SRAqg%*UgP=HoP6X%yr{*zIIOR z>Z~C*GHgOV9MykVzeKJu58TtkVVKam#gbB!4%EvMTcWe0F8@#j(X@hhGHXz+{sX%7 zvt+IFV+97p9!8B(I&S?%e;6xml5V%#!sT?-Bk=^K3GX5UaSh)ol(p}6WGjl48rB!) zSxqoAVsGhDx$Ws-=Nw0>wOsHycRsho6;egJOxYhe7+#R zn_#vJuuv>+^G6lQ%czf-GH+j5P)3#>AMd>nQ!x4xTw@TbCgQV)j~g0 ziP8;m^12rEiU5WbBJm}BMp{_`EA>RfrINZ5sC>DK+~AiHb#eB?*MH>?uDx+nHq&P` zXth1VcGGiO}u2 z4zE_!$m)#D3a=Y%eN%K_>}bVf%W!{RY;ijPG@#F4$|jG)n#nL5EHs%4{CG2 zFoa76P`=B(nmwStCeY?5&x|`S?#n? z(AsEAakj~~B)d+RyUf&w+bN={z+Gl*nvZRNCJ6tTXLb&m12#lpd9oSBmXpZsn^ z^C#<-#Du=c`C{0(!JfgG5{+yQiNpn{{7h%ddhfdoe>RHyav_MNv@`at@6r1*KWygP zK1SQ$S)b7TMOm@^lw$sBruc}$RYZFi{5SrAY79Q!UQ=o^{kvY;EKc0gp2;^hM*gBn zH;bc(?p{7Kg60&D(OtL^{-DIYWv7lSic~Uv!Lf^KTqh_sU zRl?*?nYQ9emQ7+Stje9*K$J~!y>4$J(I>e!pHrh6jYva?xRmk`s7s%&F_a0=IANGR zuuSYTxtgpJyQ!#%BMbDVR8&UqV$a+^1uF(@ayleNk$Z3MPC1tEwXHP0MP!LSPucOr zXfj;^`pSiQo8)u?Dvzlt^LIYtNdQQM9%Y0t)L5-2nH$oXwb{44zn~rSZbB1L`X~4L zb16)4#Tr%ARRoab2w_Syouv$osQ!gLrv%0etz5h``v<%jWDN0tDOCPX3C;g^ke}_p z^5y>s@-wn>{J+=#7361RV*1}$%KsDO&(=kvld*pTfe-5XKL>`PpCG6U6!!Mkb|7aT zjX=Y`Z9mr_z#vWDwQuL=uhPq^a+h=Gr#ZI_9yOy9EGa!C2vcGc@NjZ?bb1m#UP0*$ zxW3Vmp{cQvSXuF3pze);-@k;4X2DrH18|5xzje}VL4h^(9FqAL)Y4-B!11mPz;zA) z>+NrAp6}{v09Q4&UOzzG><_?10k?Wkfe1{3I{?5A0%b`K4o}XYnrdDBrtd3=eAY7H z`iF-HhTjV~geGv#pqT&!f28>pER(;?1sn_Td3y%%Af29HYLJ@zmZ~aZe(0#MFmUGi z-H?re4XN~4#C<^57O?Ul9f8?f0_cAH8=&NX*L;3#W3f^o3#}mCUnMIJjZRO196^8e z5ZrYf5C;zz`ydVfY{0q>KrF{d090{=WBU3-eE_lky&}MR#-<FVJsI4g><)QZRtt z*Ecsey3<($admVv^!goq+{SOjQb7_X*!N)i>nr$4-Wp_W11My>NBZM}98qcxwNsl$4aPI9vcnAb*~j8qHsb z+KUs2&!zF#u+J7>etdCoasLx5XafFv((@PijWN)3IG}c}u7Gdv-=+IKBqnU^ejw{x zCZ=jM}R z5{zH#_dPNP2XJ|Rcw}&V|A?F@)c(=Q0oZ*H-uE9j6jg%vJ1UL8hm;@Wq z*FQX>by2Kq$VLUwu1xJe(UiVtEkBxcTnr#37kj|JhkStabq&qGvDZ%3)7HAU&?##C zKb8Q!w9`IiDZy!6n%{Lg*Z+D6^Bb7wA>W=Obx7#_VK*+Yv;w()Oc?!xFb?)UE`Z&w zXW;eD_F}(Bd>tJCG)8{qzlrUDG>3nw?i>I#j{iM00ck4#B(4F_Ui?Vp0HRL&O4ful zezBJNSgfn-0IvL^vg;&kdT?s=@l*T|>bUOtiMFg-|LS{hp`(WQe|Z4V(7E%YUi<*` zoPLw;QcP)TS7!<4{Q%wpXz>2Ruj?S{{D2;TG`a<6{Y>BMJ~F#Fx%+zQ{2u>Y-v1K+ zye9<+JMX@Z zkE?JkpqGA2#dE3lCSuTaTM*7>jTJ_l9LgPCc&rWOZTeER0Z$qnLn+0mHyP@!V=Mb< z%<=MYc2goi67asl$VazlZfIP0m!T`Xfq)zL{=tYx#QmaqUH^2yd8UwcNz((82JrDg zPQ$*wv$Vv$8&>k{7Rq}}Y^IUUxeg#L^016Hvy#c23cprs)qA28yX5cscz0yku=9f= z2nj!fnzL8*7NV{VXXu>R*n~-<#9Hx&(S{$_Zz*~AJgIffEt18!huD=qfHKSpr>exI zVjnOz)K2Y1O)AE#)7&{QDtf2J)0N%Nntcr<3Y;O{NygqMZ>;JXS?i3MV0jrn+v;_!z9cE&Qi)a?Hy635c+IuhsSiTc<{`TY|xPLw=QE zkWoOG^+h`j$GafRrajtCjA`X>_a4e5ae30$4mO26Qw2(ps%L&Dp>aPnR#;@u`Ap@h z4%t{dXTk7pl_@1!Uf92x#laZPt5Ygu< zFBnw0*^HWXHpa5MnVkd@A)rlDJFFgg7Tx^|p_qIcy^Lm-5q#^$qMa?X+MRPoRV)dcBtjgjW^~sg65WPaze8=v2;WulsDBEEntY6tEDg$wH2+3( zZr_3ee|i`}Fjk3R=tzkxY#Jx_km2T9OWZz9Nr+;ZbrGe%7-$NLp38Nt+IXg-Ij-oT zs2Q)d21<4nBtD{7u@LrO4-5epj|6fE73v#Ffyae+v-{V(%${6ng>jbFou!IvH4EUI z+{;-hoPYB=_pjfsQ?yLL?yd#`Qgf_E>jtty2dOM4mAF(4;bv@dYcFG+^d-t*Q||0- z;|VFK(7NfYF90Q%(?BoIew0*UEHu3tkC?)*@ zRzE6B-w2YTKywz_cG97BBqWh!_0ZqML{T&5WQ#^Wg}?#tGx|alKQrQ=L}*VxRGBu< zJP!WqQb%UBIj*82j%r~W^KJ~z4>4MA;ITtBRjKf$bJ<5vTYgQcu6#6v;A<=5#*pXe zVWqv**XZoi{a1QPDneEKap8snHld+W>!-g7r&sTuNh$p2GN=})h8^p3e{pvjy6+BqQp-0vrnG`>_V{Emd4kW)DX)N)&Ha%%I%U-)=UU z-afA#s+{FsNoqYuPX1Jb2zba(zxE%^sRcr{QGxVdmB2& z@061vGcikZ-@Ko5wA{{uux{?fl+=$$im&b&qx2O3$FT2!ss*|c1vlZ?Ifk>U58H?M@c^|j~3 z429}skIb2G$73Zn_fqvYM@i{Yo&$DcPAe49bJK7i)pyvaBV*0p?u)!xWoGG5=nFgt zZH}zkx984fDG+D!x{kvc|p$^S%=D$qF~y{50|UEz9FYz20r;tC7$F+qT;qna@gIccT%q8(OeNo#*-FN@( z{Y1wgqHM>)!`SRWt@JuD8b?P3J2W4N@W+kaM=rq!cZbE?>?nAQwX;}OE~aZZZ{(lr z7A{qAf|&Q35gktlI)*c>ozJ|9>Wnz?SdF(c^t{v|`<%{mGQ1kw_G)mK?Nm+Hyv6G$ ztrp?Pm2HigPbdj%G+8x@g?A@T<+6oyQj25AFUe=Ks+HB$-aVFC@nh$Z7A7wdf7eFg z0;h-;JAop2C2=D2$_8c_Fq?XVS2{Ize=Jci-eIR(2=cy`y(BmPL0k1wj=m|vK`mwP z@r6I12Sz)b0Uhr4;3s#UqsB_u2DE`!%^b!-o>BECo16Fe@d(Fa^iYMpl<)>h8l&$L z5f#Nj!jKdaMi+BM(&&FcemusxdsgpOT;?(p?f6U9mT28VoD> z3$VEyTE`Kn+Ig%wBFJ-YN_H6OsW2x9PL4LT72GR-&U}s5L4QQtmRH5j)+uH8E}Hn+|j<+AF%jMjh!V zX2U%m(~OvZ1pF|3D#vxHnyqTWHFpU;%FVR>7iJZqBRa*n0PJd!ORU;ed(#8!tovpi0cmDrRiBh)(N}xNaPrIQ#)x%f!)3U>2aoS zKkJ!vw%0G&#NV__y|ii-0izl?_W^SHrBY&bb6-K7WFM*Q(mLQJmv^((c>6JX3@<=& z6&9)(*LeVVf{`GPGd*wYWaiVjA`CMhJ^4j;PF`T$yZ)0V3(u$4PmAqm17B#>XdgqNC)n@9VXc_#4mO`%FCK! zSJWfRmEOMJIGU3aGE=rZq{GR%v<{W584L&3Zo%`8EOnT?To0l=-c~D04eMdX45Mg= ztgdC@Yd|MaKl_1XrF|_M-g(QoJ{up$lMxs_8@4Yvo^r7Mpu5g4p;?rp)pd9xS^v{B zoTYra_&E5MF{YZ65dc_5yXgRbRl@p;$85?pP;K|szE^Q&T7);D^rR^oOn?HlN8c0^ zv=?`|TnalL;VlV`X^BH=N)B};|4~4`&x&nwa)0aFaqaXY4BjjYLie6OQ(mlfCRkIt zH&q;#B|w;W6BTi;t%MjC7Lq^VN7(GkV6}G3;nWkfCoO#M01-G;hCjp45G^ zKn}McM)VX1n6i-18`lh@<>hG@G{BA7J5<{F#OGwjj7QmiqF~W?d26W?Bdt|&_yOth zXO{?gerZ;aD^Y3|?{D^rf8atHBT>VoK~C}y?$8<5qy6BV$8G|NH8Ac>iad{97FR(O z;r-T%XtLS|uGrZ6W4Iw6b;Hs(N5zYCzVcv-M4{1)W);&xMHxq!xpF-1@$=IzFG*Zq zCZ}x<9=@D~Lt_EX)g8|_ZM~UG`quyKS=k3(XdvIU{%Y7<_WLuc6ul;eNRvy^_j!%TAh$vq+W>A>drp`PsnbRdwAD4u+TW{gerldMzxjbn2rRNI`QryZ(mH z>UsaHHDfnT1=@zEFtMk#ky@bEThzqSs<_si?;7{44K2f{j8rR3LeiCj8Fx(VuWJ<~ zp3d|+?~Av9%naQy#{y{Qp*-+i!_!13qb!ZZd;BT3`+a~#=zs8D6rpb=MD%X_iOMVe z;IN!c)1Vz4OElpBAmX24bBD-CV~#6tDZ2j%9|p%Uj#JLd^%Edf8T9zFjG{_JISo8A zWAr2nbmvUWD&{B0YTI<#+hd8wi-TdbfoEpY`_;+qcrsbuOSPhz{)tj(b?wVhkX8C_ z3qHtHhaO1GYBMt_nIQK{+HBQg>GG_gYV{1G$FRvqlGQ-wh)5oyI3VrD5OMDuO%x_5 zdCysozl3WAM`d_{H&QFU zKMR}@l+>?v%tJ1@?FEffP2erEYCg!%<&EvN#WvJG-__oM4rt15(Qgk$TEHAi?o}Ze zO1%iWn}xB-wXhizw&8iv!3jWoDTe1{b@n>gvIR5%)J`~ms^t|o z=hf!c5M!9xp?{iP=O>l7pi2Rm2wRqjFYxzZ*X}|lEttAvnV{QLJ1O2h^Ce*@g%<)( zRUaDSQ8sjtQPv;^mVuy2-0)j?sLg+v!Q51&acL1PTuH{2j34elcjhBAff6Szg5x^F zLo$*nI}Uo}T^DSHrf>WbxS~U*ttCI2?JTo4EE4R^>)a(==1OPQ>-hbxC)cKh3ROSc zZYR@YGuxCjtb<|^Ld`s!%<9;mTYV&kA3N`Fwdkdcqdyp8Cb+?NAIlyC zp42jXE4g94I}@)n3mCLqyfw$;)B-iOyfTC}(ST z#zwbtxMwe<4RPr_5SWD%b1>MBC%7sHf1c|ubAwl|demaoe}%s`ct1;RNY9Z%eNxDH0R`O?$+zrrME!}Sd$@A=sq6uKyReN zm{E;@jc_=IHlP6~Nkkkii- zHKpt%Hl2JU>-(Hq{4)Z**b%`mm0+n1SUi~5vXnVN^^}8)ZagI@t2b899YZy8k{CAC z=kzVwyb-O!m`~1^v?BXsx(Z(yTiqW> z+tIknu1L8@y{c9;&YOykQB72i<1-X+G2By}k6mu2`QUM(e$=Ckd<5eG;)yFqPhjV}6S@AZx7-szL+Q*DlB zSx4cG+h#~oPFR)&Be17%mt-|1^cUq-lju{BN;9ykACP%i$Wbg%_q%%X^vxC!GFrYS!fZ=j;&n(3f0^?a6)4uE=g)ow8>-n2 zN2ZyAEO@8oQAV#JZx(ntmC)3r) zbtf)*yvO1vRNVSW;x*;Nop!qZ&?7)nXE^tT4{IZ##J^VWRb+h!zK9qe=#CB$x1%^+2UZSC zks-g!Bn#_tw?lEbz=NYFkc=Vhin&jB>E$8sZ<@oBM@{1Zk32)^q`_y!++ND?-fpoG z52FP9keG@BWw3Bbx*Dt1i2hu@Vxk!Wh;>567^h*})#9a~LQt@OeG~+JVN?}&^YYnr z7;zp5u<-hKY6o-w)?I4ZV|UKfsZN|KM10_8vRsj13({o1Ytn23F&U^KNX^M;9*|A! z@wVtbYo^1g!i$}cfe}N1@&A-ZG*TWh;E&>Zku9vU2lz6b-k-2Eezt`09! z>Ex!yZg?TVnu{|#tV9cJ;U*nrZisF&!2kxmee!yhrg$`ge0dIJeWAgEohSa87m_;i z_U$AAtGdM5>Kj?dOb()z8}I89mUJSP+#$izVK7FKnZ2Z@2A4!ixPHcu*E;Hpp?f49 z`7N*~m}W&DnHH_8|M|M9W&jq_U;s7`TkE#?*n(}(Ubj1ZIs@}8w#k`bmrx~gZ5mffWGt7Bg$P5M{pP=_m8gF`m0Mvj42g6t zK!S7Vi#t1KK_2Kndhy6Jpf6}yrkbxj{gi1=GFCvY=+lgdF?}L!nox%qi%D`e(HFX} z;j^&3z?n02yK6Ja4?iG?MJ6f}E1Tq!0elqVw&)MNf>N7^`T>Hy*8MEMN3%6W*jXv{WqbIFiAO9$6=jvl~s8RZ7@-y{jqA^y9f z_c@I7C9}(j@>6^&i?$rTQV?uy^@IeD`SpS*O~C2LtH;g6E+MmWS4t1-at>^=Y+tiAY5E|f5jHng;R zl22yKVWJ2J9VPAC4>`n32_h%Hf-Zz|ahulC!wtENWt*F|u2g!`GFxPd1qJfsnS=5c zv2?<^rL0=TecDQ;6OU$nWK6slE~L7NyEPTsjLEZ;g13B^(v?$?M$c3ny#n`MnXB62 z&gs_mi4H^5H3u@DuWtSc#f6pLQq3y`sjG6l-MUHaqfaQ9Lv?<$RobR`I~pLB z;Y2f_?az5&Q9lsb0Y_8UJB1*RMGcJ~u)y>NOb=j!Q+*o3zfA|0Vaw$hUO1rsh!-CC`Tx(1x; zcch1;?m(E;sg;^lgtkUK<)^LrINKV?7sw9YtBZCD;x zSrTCjvpR2F5j~*}6-%>GveH#9zLH8sZIE9oNykia8Hr-7 zE|0jX`H=$;nHQQH6y5m~< zcA?R2U?=j0-Fa;@>j!VO5S3vcq=NgNmJ=3T-!@a-?LCU&K|A%rMd<61i3V5J|EC1c zIy|FFqt{J~(0@8>5_pt^bAo@S^3W{kBWNN%8evPF)KiZM_|q+@IHvOo^x$k*bdVGl z68pcF;f=4!%CuQAbma#LQa8gC>UtNTzC?WO$%ZejC?SIZ++&#s8*ruEwj;yloZ5k$jW7qI& z=CH6POpwIqCz~J8zY^tx*tf~Je64{5X<91(1808CO@x>r_bk0H$Uk(n)-Ryiq6KN8 zlK#1^)X*@xoH;RxaYRCKe2qMt2{f=a=UjE$cK^L&Vd6r?R*dDuBr*67IW70V8GV46 zkK5-1#>EM{#g5NR#}1UdggE4%4)cawDq5S-F4nz9YH<Eqab?F3JEI`?ElaoqJhQFqW^Zg7$JwT(BcQ;F zySTCmz{uwbXTh)2#;Y0|CcMbjCUE zH264eeV&)REcjB~`5&m>ODnTvxoh{o(bt8-wgUNMP~7z~=RW@QN+t%Td1Rh{JWsKO zZ-HTWFlGQ;k_Kuf{axxO0jI)1)5c)}O%uTx>FmqCy`}5Nf31ZgktU%gfTh-|(Lz?u-Ae`F3zE~jIz*@=m5)Z)Xl4n5cGUC`VpVI)GTa9fGr7S__ZuGA2 zgv_xh;21&3AGg>r3)2m3Gq))KheEcctvD^Tr}b60;vKjW^_VfUI!<)Rt1?!?V>86y!={`8lOlg?Jt`rc*Ii6>+dC7@a;ixsDj6;{UD z77BJ+Lx?DsB_!M${?7VY8HjR=#chd9a7SNPXAnxW_i3-peE?g@GfK6oav4MutWN?| zrQKwKXD^K|jy~I|8vBOH^B|exDupGHQvOEb{v)&UX-}J%Ix1w=f`3i+RC-KZdJ)4} zwikRD7m6gIgaF@gWcC_L9v|y=E4)YBa5LPykfc-v0JzPX7Ckk6< z%(Ef}C3X7MTTq<#nHoq#V(vVW4ZD;n&_-5V6ZYK*6nps#_}m}F0dq9HH=yi37z)Tj zXXOJCN8(2#U859XI+)u3(kd8pLxzVYQ+EUZM3!Q!RL=_K4Hz}AS?`$s+ME;l(n_G0 zg_tXW+}p>iyWVu3ItQDH%EIEGJWk$Zh96~Vu|!}c5ba5MYE_bCzpY{fe{`704bsuQ z3L6d_+RaS@hfKt*7HIgk@~O;B7wOvipgdRceh<$#*NRWV9z3+)I<_tq8Aj?SdDW+d z6|p07v7}sCXyO#pZid+Antk6RuogZ<*Ut8DI^YJm-?iQyJXfh!UOp9iKvJg?G8c{QcocxcBaF6aLJ zS6V_wZyN@hhTx(Kq9;KdkTy3+@cIK?I-Y!lDHG{RFA2jHrZOJqMq#9{VS-B7G2oII zw?Ys!w2!;Gzk>L=&MT4KkYmsciD-=T^vl;muQE1_hPKAgqfWE`1_ zeKVPKORbz&E{2PH*52`j74d4VRP(0$WrfTuj?mni18G}0i3t`Xr8&$^|I)@n!t2z3 zUuuLNVlnx`B3SfGHd@->*Fl9?4KOxq|3 zG50Q9sJepE9<)Y3+t62Ec>go*9NV^IHUlLs{qgQ|m zFQ4y|m6A+|(fg;a)(DzgOoJ36@Mbz^NjdZ z)&Nra+cQ_b<$@7k(6|qJT;pueAcSAN1fG%LkMH|6Zlvxri0-GsH)Zou{ z1cZcF7TW@j?!Cv4V_{eujc%LP)ZL}PpEO0w25Kq6aSf{pDOTBhyZV}z3 zOC?O%whfo}4McY!XsEoxJsizf>g5BlNO@F3N*N`jqUO-~J zImNfa+Hv2Z0yJjd<(=yfMHjz}qi09hY*dxE%0G0uf#iH5nK%n6IlW+=$A=m%Dgz4>maE-=}osrUbE zviqJB$cJEGYTbs_&eDuoPSHwR0Xb^pPwM@qMuI5nzu% z)MX|9g|+tC6<0k+7z$6!J})pj^;r5Lu{`a*4q>r;WsqcqsI^0sNv1K1;srD>F$O`Tv zvso8Um}oZctkkZ@)*peOn{OLY(50)uRV84vYD#%Kv0$t$yr7u~QYH2F{1dpinuU4u zGFBvQi@t_?;O%EI-ik39VWr3ScO&sdC`{2bfEdbCcE_k$+frdaRCiH$3NbU7DJ7PB zSM%UxBbk`t`pNkM5Uz^g@e9ALTa?|^r(vx?V+bNTOp#(@+3RRJ>bKxsLS>6D*f5BLq3w3TWw@NH#$!&*~iLI3b1x#wZt|r z8Kvu5qDESK?~-WNhYhcIMb?w-_{P8xh#Hqo!q zAV9`$*P|3}=AFc<7>kBno@>o#A+n`EoaS^jD>+wKZ3$Wc8BtT(>8fgTt_Gon@!)iw z-j_zZe&piA`@?60>tgu7WQke-|H)bYgMza%{*Np%JHvmbivJ4}oP&Xx{eL6DT|iZm zwYS(LBqXJYh1p>Ua2Iz7COI ze*d)g+HzY?J#MdJee}Kb-|$?`e29HPz1nFtP~n6hpdkTD0HnCY#DoR`2oyvBAW+B* z%-KT=b^84dK(5IQK#-uqB!9$%k$_-?3K&fhpm589!vI-$bpV8j004amPUoK-z)=75Vg*qf5p*%f2ApLa`fb4^VaMhg=PKyaSg5^M1Nf|1_&wW5rAL>faw&t8DJvh9{e|g5CNVq z0f=F3{ByA}Ub!{C+yL~jq5>C^Z0(LkV@!oG_Q?+(b^@xff) z0IVqfA_Rp&{tVy@Py|pQK%@ZffckF%>f3UP_UAyOeYX4ivW55!S-}1H+1s%J0!sm3 z!-V?(+a=;79YccUBj6eM_5C@2ZxJCN0j&$)C;({J1EYxD%e%0^gnqX!oW=Qj0@DOo zd_V&F^XK*TRnPzT{vjRjU&ud~;Xtm*%hS@*Za<2@8x@scFW~R@kzl~@!9pPbNe4*( z_HvN^zdh4u!FhfgfIrC94tzsje-TZxsSk(0E%E4Ti?Bk&F(UGHC!{E({vA@~3|9A2pU zKR8y3ItCK6azRqKRXhX3aA&btt(+g*Vw+7N9?{or$wE7hI&36 z$RCe3fcFsp{=dh9v8h>1A3uVH!tWEHqO!9+>B|1={a5}m84VQ}03iYaJ`jQC)L>Cj zKwqJAk7jVMzeGj>1pqMd0%ZUT&tCut9Kri1nFYzfKFWfFy>JKt1RzoB4?6)I0tjLD zCw6;vBm#&(d2HPwSf6zJFF!{Sz(9$5^3TtA?2ah0`NH2VFdz^`y?eh%2>=kEf$+Wl zK8j5b0)%tPKT=_qzrJhzH7Ja0*jD27T>mY}FfP?Fwj6b;d9Q~$_lp~z2alu)U$%0oau*SQgg zEa3N8-t1XJiEoc%s*PV|U@;_YX=~|OlBI|$@FDxJSOl-xEwtrj=pKTV zEd{FQ5-f^H;zAdoJnbeCiZaQEl}Inp9^h0`4JF;Ogn65J??M&Glx%I)rjc`s6&INI=cCT^)-e~)5S`y ze`@#QPngQw-^@dEuR@$vu!i^CjLUXyd+v_aw(mFZ-tYT&KLTYG!&|DVkG)ts<#o@> zYdX0eKrx>*bBU%u@T&9*yL4d1D(1o_aE!{|MaZ6q@Fhpai8^RKwk+Ik={zV0c3boh z!amc0uaFWopRNx}k&}q`y$He3)QE8c!Hjp22RBV)l?!^3>}RF|g&cMqYSbrdFdX>4 zNjp9YgGhK^1@qIVFpLp->ayUWvQ}rUS%@G$*qH?`;y&`R8w@x196+7AVRZ3JyLs7Q zpSr;=))K$%&J|83ALaI(KBNSPx|t;3spqHf+{vhxDZvZ8O@-dCSEgUII4eueNO_L`ED~78=4pi013%Zhk5~PQ1dpvP47^FS0#~LHvr8| zM5x9NpbYs<>%reC?gV@N_{+rvhUp^5M+Up4Oq1BxlOGrmR^=Xjm*m*hm%?%bcb?&H z94h)R+{UbF15=xBN_MtQ0?4mUD0`Dd(6;#9hfkq~a2P(f1I49Qw+EY3rqbm3L1U8u zTZR;d0!feS>`L28Y2GGjdY>7L%yyoetFX+E>et)0eEVT)UO<-;b`9Nmq7bRIFSJ}l z(VtUuvlc{`t`}KeR#bO)GUnOkT`+*Fyq%tCx3TuRET*-;=8S#nh8N=6@$+)h=L9cM zGc37G@U+SMCgNTgoWnn16?`nx`s}`hcY=0r=v#0{U=;4A*y}IWL0Yg0*}9}aKtL~c z&>jc};GtPkt8S`nEPm{rSTM^9g1n}T($;Hl8{7LWSIk26=&rKZkj&4dno(jd9BqYI zCXMmkA63-7GJN#)yiiI@JAgU$?9=|O$_ls3@OkMzrpfQZB;g=kep|mmwDenv&Z95D zaN#94Fk#|)H3_egKbF{C6ezZizrGazLD=DF~YadM?9$9KQhI zInlGC$m{u?_?kO~b}JwEP*D}iE|5#)m;b$FVciy5$yrx1mI!(JT*j_j=G zcKp7GRbi1*MzOtuI^N=sNGl7`w|6KT>dQnHH_@B(E8`zwp3aW6KjjscGwBT2$Ds;i z>GMd>La>GT=1gk5@Rlvhea42>OKU~g)%^qV;2pebIil=o#X6HqbI0fMPRq>oB$;N| z5n&-v+uJp98uQq`pXxfn`w*l|J%ay)VUonOgU7e=5P)k`v5P*w--m(AJ~ma63BS+k zpv7N8)eZ{*Z4U}J6JCvfAr97TBN`&3JPLPf7_3&(7;J{2)=#8&(V5wcKf$9nr1(kI z@DlYv?KGvmgT*@AK^~}%_h#*g>S+gfm{6U7+-)Dk_VTXdJ}F@lyD-`5Fpu0EUWcs6 z`R04n3~!%>#hk^yc_?{)T{N2$Co}(wzt%g}T=Uwh(3~b_0fvt0s63!%y3|QRuXTRf zbHzVX|AK&S-|bUz%jBZlF28;5uqkYVBbR!fbUd|%FURhcRAc-*jm*{3Qs*pcqC(9o zKu@!Y)YJX#D$#{LY_&TBTm4K;0oa?wIxxt0JWywr$(CZQHhOTf1!AcGWK1+*Mt_KIe?SxBK>e`9EbwM#Q(`;ma}Bnrr5q zvJGBjNTfbg%6Vl{BAA9@I5kUm!l@eAf+~v|=z2%b8}QECV$1dX%#j;L@5#ao$aRR~ zvpG?37N@_LeUPHE<6PlvVlQrJa^+te(Nf)#bAtSvfmGC;Bw;@q6*fIPhi1%vb4vj+ z4y#WQ(RyeLr?Cz;%vwtp-WT)>ES71+hhaMmjP5pd>4(&B|#(-$t z>1)HWi_>}+cl~=>Z#u<(QQZjiksN$6jY_MGhO3 z+eojc?=AeGk7t!(+%}yFbtMN(KD^krPhWe>Q9~+2(qcu!+QI+Y9)6MOd-*T$1`XaP1diYZ2&o<(|7ekrL42NfRfaA z^!0x35m2wvEK5y;ud@d+uX|0B;;Gdm-0Cr(gRs{7nqpoE?%jGK#70r$apijIlv-4^ z=9m)bGfid-?{vShf|iEq1WgBPBn_rnGv>)Y<|Rim$LQKulnJs=T$K zx{IS2+SL09($6fuASA#YWY7_bI&Sj{A{d;sGEx-QHa*khILs_(2IQd^5#6A>y{*jg zP;xlcE(~l8ueeWVaW{%DDB77P=FL_|@?XI12Xmk_Z zc*Kq+4NKDsQ4wQWqh;{P@KxF8pOQAi=I|u^xNj#o?opfc-7D+%5S_kjXvkQ+dv9PZ zmO@z{LOBmgR|D;jMnttD;QC5aFQlYHKJENj`8T+$lRB*GcI+ZQ7s@i3S^L zBuV8VAdXtV~q%g2R`R-i(yC>MNDpfoUG&veh0+hjhHb zNRo67;RRYp!he8kU#aP!y;pBp4AK70p0%xwUsYIcQ2Id7-3l+g+AER;TEs){lB-WabLGReI6dj<5lT z3>}%+3t?Vv_DOQKzYesyq#Mu#um&eYTkx9oUDO&^MO0XN7s~2!(Y&Zl6i2KJPxKg= zxHKG5_B#e0s_B`!-h&Jo|D^G{2y$W?^dOTn;ioqaxW@9eE)Jtzz0Jew=8a^Im&a~Z z>HnY@3wA3sGeEaPtX?)Ii9SNdNp>jQuOY_YN^g|C(kZge|$ zSi)-)T0N>LOLWm;jYH2gj!O+lWL+B3Qnb_8IkR2f{#$+=wUL989Hk3+=6zX2&Z)6^ zPV_Pt8YU05Wq_lUb!{;2QbE)fV2btpJAv)3o5aKS&Y=-aLyKDx%j-sODx>=)i_qK2 z=5)GO0IJ=2uC7+Q++3nWUk0uvirJ-NiJQK((*ZVvR2mAdL-3BvHSvxu2$7 zSFkSCowKcqw4}{h5%n~xJR0;cvpaew(dH7+Nh`$_pQh&UBa#A_UE-mV=q{C0y8;`Y zujvh@I+=IuRS)s&OvO?KyYQ%;GZ0gpd~8M~f$1j92*D&2DDVmItj4!TyB~RpGQOB#OQvP*!Va%gef1TymaQz+wn1~_3*_+WNpK> zLbbtY>!f<--4dM!{|aUEjaEMRtY)^Kn8K)yzv|ouG%|gTB|oiQR|zJxkYH1lP7KW) z{ml|maA$3~L7C(z5d8aVzuBtlhmEj<)-s=#e3s+8;C79-JpgB3#UIm8L#)AN(%bsX zn8e-$rjCb~vV)^+?iGJ*QdUJeNI3Bq;N+(R4c~hau|gokHqmW&77@23GI|SSl=mLP zKk*rOxcK&TsZa9AhqenZwvL$)VU-kZgHv=a=+2s#^tWh1*x)b)X8K`o{Vvy}i;%Lq zI2O*@GI>ZqOc{rq$oREIa$APXtvqK#X}VD?oA)`% zhd5^uwyXO^cYfu5w&nJs-ZO-UlI8veeaRni+3P{D`-9{=l8p68trfL67;gH_fP!5& zYicI~`Gm;h&H?_TBxkmmkK0nqxh&J6DZ`aeV#Wri_u^BXuJIjUpXPO;pL@jkdC=cl znlVvBc06k@WrvED>6Nx6S|TrzA`ZMYb@?`u{MP(6c*H*&i*-_rl|mM4OtE6HNsYy~ zI;7heb-*2UmE5Nutt?)tFzCd#9OUCrfrpTZH~tE;J4eX~9)R|+4MDefF-;>br+d(5 zVbUni%?UYoK!?el18Lm+ydSDsH@Yg`GZ@K};HE<`Q0Tt>2r-k9E`MyU`?lVUS3Z}5 zu0*TJrwQ+zJW1!*qa`rXZZHMGdgdXHYn=~LrPJx=czXq=wtBUE=2OP|M9zV}IlPd(tXH1l#Y7$#o50Skuo zzzO8ieOIH9vY$i1s%m{sA8%EPFgcuRYq3kUsHt0xJ=d3y(kQHMVMS)HI*&dTk1s!h z>1$c}p0OYQFcz}@#@lAPz?F4y>uK*WJ$sQDqF&bc#_1sJD#AW#pA{WR)CPmJTLtx= zrCBtT&#b7dw)&w;plW7(6gs?7{a}=1GoZ4>c*@I=bB!jj1_;_$+mLw(f?7&)Vv6rR z*!$^@pg<^i|NbMEIGZda6NOvE`x#*YrF1$+_!&0DNGiisneV3+7E_O>85KG$?spJt z({!I-($*EU1~&w2kIGQy$J?Rq>Xs!k>brY zyb&@yVM!L~%Ia77L(*7nr@*FaC5SW=G%?yq0q3+uQ1`9}Jb@!6b%pf$0rK$f3|t0Z z67t}%-mOS}Xl(JJ8>@ZHY92Rpb3EE8xg!S`SrK0)1ca7ELLQ=VcAz13jXuuJj@r;o z*3Y+rn@pAxvZLJcm`p4Rc&@&q7llP0`OQF;p1|Sj-aq=jtE+~nBvJ`Dj2X40bz_kz z^vQ1QvysZWaooTxJ?7nNqWjDN_piskK7R~F0^V`r_H)?^uD zJ!!vrR)5Q`yY!BtNM-H|<@6Ngm(Lu4z>hS>?vl(x z)T5J=tqH_SHhbaog&>lv&JH;G*(6l$O-gm*UTMLbNQ*=2R%-vWq|xKdv7*&8Bm| zh0h=#LzOls3QNB-_>ONAVlGmQyR+3H>~rgrhR}G>b_GHm2BAuIkHwZ?=Pf1BMiasf zB~uTPBGE{RwC`r7e>v5nuT8`k%Eg%!l1F80J zZUgCDxL|5w4?a;+ghp0U_?o;9y%UKZ7arqHo0MfeKm|x9(BqFG!(OYXExf+JD`fFK z4L03!^jmml@X1S1J^PxVj^CH+>z3^hWwu-1cBZw82D%bj^n>HAq3k-6eut_r^zf~~ zfGu?~3e8n%Z4C2N%R@@GKIbdssI1+2nzfE9o(NH_VRM+ibxo!fZ_xxc5+;0;;tv?+ zkS~l2TLqYEaA#gNm%r9tS+?X!6UU^cDhUL58-qNLDpZiM%XLgAY_JQrfDk-URLAw` zVviU_g0sP4j;m?wDdBCb@?1m>bc(pQ2T7ng8ygnX93i8G25<+q#@gqiC4e z$c=6;iBDY}Bh_ z6uQpmsCoGP8uljOIA!EheM%G<9*@I~`Fn@ED&6b`E5npbnvtHv1V;7oScs-}rS4YA z6w0_}^7F?kFE8pjxthjabjgX7ZhC5&Yhm9nmbQ$L_4-L0Lt&qL#g8w@u57vti@9^u zO-*2_G_&KI_T@0=4?KaePd)wjDZ8${0e2P~KfZf4ZL`GnMMhB}gg3#=lKxZWk<4O6 zWf3CBGk%4on13e5klEkI+5n3$hsdE&kOz^xtC)MhV|SHxFS70l-lbws#cFg2lAs9( zgIDkY(~_E=NHOBoU~2AB_9%lwVSL<@GF)_DSBh(80U)txpesr^p)_vXDCz9p{$xIq zOs*wxI`h*I;3x1d*F~wV^_rQd>V%j-rY!-=*33LF3Na`vo;u{% z+b6y<`(v-lnhSfv#%uKrx`dFUnp*RRo46`|sYOPGU3Icrk|4Q%a?FZq!V+AlaE4Mq zfd-z6uM_^KfSB_?1Xcu?@zt8sG}shRx!AgwbcK{hWkHS z$x;mf)(%LW4}WNGNZbk5(H;=i$Ou;-^Dcoou0I8}K64()atGZ!MU&ODa{qqkZT=(S zRgm+3IWMW#l|DwQ*6o{P@TZ5hQ`LgdT70t7?B(zH+QwQ`nFyEfVe}K4k)qsNi4W5@ z?0Z9sSFkn|grO3-BOjO*DG3v1%Ob0YN*YaX1g^B{36h%E%R#_w6w%XfG6pI9fGa4? z-=}u9jv6@@i)q?9%VZVYo!jLj6y1|f*ca;ab!v*U*RWuc>NK^*LyObq_Ai#t0PMcb zKWxaXm{4VFsxz0;o$RmJCu939K~KY&)utQS%Wr_?lB${ip{vjOU%L7NwzhW8PWW2* z42*0)j}`G5nEsjmxiYf<`^xd}=l_|f|M&Rc^Gpo?9{*<@8|fQd4U(GFL;h2B#Baj+1~?eFz4T zbH|3m&-XJ?pe4ic$tj-({!8=6dl$AT(M1HTZl8WuU%1|`weO%N>ts=niPzJ~WpcCx z_%m{u2XdLE(@En;qSyQrm(?W|cU9*mHLn6mk=tWB(xis@<=n9#Te{>1NZAWZOx)$= zTw{SUzx54vcbkowWRrOe^M=3YRPV!QkJy+aA`&bo9p)Dk%1ZBDmgnsk9ccIYbX-We z%U%T39P>-|^K|)rzqYs1QPo~Qe!yvtB6O+Sm#Zi?J>Aa-kT6TugCb#JMg;o_Vnhp8 z%vrt9*Q^-Qxo6mp6A#9Y7M>HLl!06H(UhVnw7gXbG_oS@38V!4qt9&0h(_v7L5{7B zFFeVb=ztc+RuWp1V&fml^{_jU^#KQxxFA&XV6kI~{D4QrIIgDApe&vQkdCfFEd}m2 z;0jYA!CJcKH&r#pDcIJ_Br$30c-czwv;~`{OzwJhP@)5OAK)62 zbpv;NJdOBK#5O0R{<~c9pp1EEU*izwz39K2LT1PIs^hCYzF$(c^+nj8LJ0>z=3RGW;N2bl!G>bM zcHu0lG=)#Vp>o>luph0W5t4QW{v=4;o9(6+b&Oz#vK1h2NFGQgkv^}4(&M7&I3Ph* zLW%y%e2sOp>3W?xpzJ9KbS%rbue={SP2z#uFq9yYMgt3=V*qxae}x8RxK~<~DoTgh z>dMEOR?3W{`-;=Dz?yHr%*NjnYFc3ds}o|g&1PiN+DPy!kt&+aMJR)fmhxm6`wkt8 zQd5->qo+@w4n@m;JRx~U4!n<&fnUJ4s@5^Ttlp|+wyw{-R}md2_y_I!@e1ziWn)am z6xd(_Ia}zLgNGm(JHMkt%8_c*#;^FR#wDkh<&`5C`)EFtnMLXC;1fGGly!W1h&bci zx;mDN4v~+E1?E5c#OSuT>a<`Gty#OfbeXk1j#&0oN&Y2ORiEIxHV7+teO3>DTNe57 z)44$n%M}Kg%#WJ|^c7~>p@ESY?7QQsp3>CSdnn@s^#GcJD`twum-65h$6JIX)7M0S`)-YX%m=);kW*Y-7RSY~Zuc&5bbHg)KRQgS15o#v4jYB0s}%rjoh|W&CJRk% zIr}2{(~6igf`4|`!N18>FwIMzuKHM88HOP?=(?uJv2R@R-32~V z5*IspyQ^HcBhJ$CL^4XGf-VX--O?5dkV(!~=14PZvL~c8t+QCg#2y7VBL|Hs`5jlG zQ9AJD(P?f;<}Pt!S3c2;5)XA`8Fz*Kaz(Rsf=zdGcEp8)c4{jqf`$yV-w^j68}&&l z7=dvv-w!#7e<^&7O~=__`!@mFC1RHH?ak648}t(@F+khg5vxr#zqH1=EDvp_gs}G{ zU_hOHCdziMR)DK&@Y~ zuV)`XrPyF6mF7fZIsC)2w}Jg;F(}sDBzmyv`A#!Mc0K6eDYKoxN?gP{Ln^63S#H`6 zDsAH-MC3I(W?Y1>iSA8`1d2i75Un~2yKUX*6)4lq2uU7XNH_LmK;?Ch5nT`W`R>vH zM8*NZ%{t#E&xIV~dX<5%hd}*Eus&OSC&eK$0+YzMXnI%DU_5)lJ@wFNYDd^t5+QS)%=2`Q#SKfgkVl5TLAxaaD2Ug-5xy5-z_6$ifCI9FjkQ6U0kI8rKXx(at2)i&5B z-gMK@s9z zc=#9Z!a)MTzIVPf83x&xai8BBDb0sB2`Z=Yqx5~m~xd2 z@1iRQuE!&vq*bue?Y0gLrrOkzDg4u}D1A)&GFsl>)@0 zpS;1Djl3%6`}AL}(}4X)wC6|j9jMEqz^1t57Q?+-==;Q&Z`wVlx^~0jHmAJ4%7Jg$ zBfRI}k#G7&%xU809mxQ8deo0cE0qr0;4F7DerZgtAZ-d9nFU>s-ej!zdumsZA6{?B z!N5lK8v=pm5QbGTKZK)H^%9_7v8(~Y+U zA=jRLM5o)>>(qzw(Yo5Ny{p}X(GBZ^jrObj(Vc3Bp@Y2F<45~7skMujLd%crVe^(- z&@*a_Y8rf6YXHCVi@dL?tp&r;Ph((w8t4RsY-n;tIL-?3$0`5={R(skl_&{igkdnB zcZ(~OXi86-5)#y4FjL|7)#NbW!!Soevj*U^ifds>&7BG+C2D+H7n;8s6YC~`JJw!R zyY^GT;nz!w2OwUtgh+OkyH?v@K#Pq}5D}UhrAz?iX@IA;bc#+oU0=EIJb>c5GA`Jh zG7}AOn{x*_UnU^~N0(iC>HWWSsNSn?JSnlNT;GgY-E8azYkiiG{p;*ZO@T?bz#?-a zzyrjSK}u9__*y{;vnv;>9!T#xL>Yx$33R$ekgs2%mevG>H_G6HeE zF=4zSQIE=gCHvp^VVrxRdc}IB6D$Fkqd zU5-PeaMgm4KTi)WRGE`5*_Y#3KsOk}V*3Yove3A^8q(ERa?MHF#tP}!oDUg10bjyM zb${t;V+GCvnD+5B=VLZ|=$zMf_b}B$AbcRtgR#;YxZ`%w^+9!Q3*w7Zo890bH>-BaRt3kP(#ZgD+N=Qk z3w;)V!P0q$W(DL72V=W7ocEIGqESMjk{73yU?@H08{Z8o3pEpKXz0q1kSbk&f{{&h z3?oB#v1~_6Y7Ep;kT#M_A7MNZ98~?PS8HF=PfV{4J0f3 z|7t_vkzMBP+ZHt-m)Q6m1 zk0q?Kr00bzH5kCmL#1YTv;N+yw899P1w8qUd%iL7&j)a-w)z&>e+Y);c%Z zhYR+S2?#&Jv;v5G)+xh^{{97`3silER+4+!0WYog>RlDoLc0QPp|7>Pfa=)bfq@^s2ehbb@9F7H7*vEwPpw1p z$_3EMyu8?if|0#JYF)oGnru0{y}g;Is-_b#zSgG>y=5!%LtB=DUVxxq&>5LO-}^hn%LvO5Kh0%O{N$zl zs@`qx9A=AlykG$@!3V1wn(RhLjVND#os45X`J31%Fr)s-A9n@XY}>#D08P`#AkQO2 z629C74g+%-6ZCN?lhMl#7zIkL-*0xL^m<< zC!vrDfXl9*bWlinhrNoi8pbeW6N0vHuAq^9Vz*LSGX}LanDTL7*S$T8ZfrrE2iT%0 zqWo+T)Oy0qqqAIc;V`is+;6&bxr%_ky!&j)?im;P8ibm2`kGrwv z1~RT-J2GAeFO>{?QUQ;~KtZFf2YdNN}BFIH)Scoc}ZXUj)|6_l087^WBPFjN&M zaV=mQxQ}TncM-{UyDFh{3`(~y2JWC zhZz+EqC*MOpGdJ!;Ti=@&iKpwu$J9iuwbR^c~;r?B)s$r?l?q!w}jG!R^543Gbv(*a&A0+z0mDG zRix|@^Q^m@j%6e-kx8m*KhZzJuTKWTO8ale0k6@>PK@;jOaWDW)eLwG$ zu2Q6qG zP)X!|7SRQ<{9lLppci>gGjwCuH)ivPKiqQrs}>Opa~CRT>o|=%&k7Me52ZbcA8rJQYV=a^<7`ae_hfP;g^=t`ZjH(x|$KMV|cTAxT z+6y0fWnPI^ZuIVnaDTwUoN9{`X*u8N!mZ*dQ-JM!IZ)L1iO{P#Do>myBtDF(@#Ua! z8@DzF<7nrB@mWM=jGqpej^=NyS+4%j0o>{M6lGO4`qnmm86*6X1A=}$KUcjf%dJ@$ z;`_!m`V+Y0W(VX_1wr-kn8gIZhgDH7_LWC2F~|Ff_myS7dXwrm7byY?&ww}V5InYe zKhrUC*IPgM4f0ikF!4V?iT_}L|H0S(A1IOSU(nxwgZlo34*s7(eg8rh|9_yqe**mH zDE}MO$H>6U@ZX@m5_Jiu%{IiI54BTql@NN@z#b2Y%h&Y`>FP>}gYkd_i--o{cZa+C zn=&kz2}GI{CKj{^YwGU3i#n`7MeE^lvM;ZD-{y_2-vah*lIofE44GT8isQx7q7sRX zL+A52)J(iHS{{NNs*y0dzq6>gXzm@U7t3)*Cr(cbDb4O2RAN!$Ha^Ft zWSkY5Z5q^#Z2*aUbzP*s+jM$@11x^ykEd~4RW_2FJ$h=XP13tyN0nu_uSy+n2`5rqKBdOckZyP$dyqrA4|Fr2fEK7#LpS`Rmvz?Ipt;YSV)k8J4=|?1v zo82|)gF)16SaGD#whNlLJ-`3#9&2z+Di9VPTDA|(xtYu#%rOPg_qBHoH5WuQD?xGv z_}*;E*DX-1L_XG18TK1yYt0T6(v|JiHCTN;GoVP)CzVhCJ+|@M#lE}=5^>tH{T6EM zdFJHAuUJoJ)qm9aVkQam^V@FLytW(iuxGuvmO4>CSdj7BeIt)Z1J6K?Rg@fy3pN#} zg&EScLcxhOL6jLK%siG!+SNBTKO`Fp935z!CW<35Q;abLd=_5>5`y+8Ku-$-I*4>2 z0-`@IZ~owz#b;_hoHxESR%UrB{m=tsFmI%QPRkam4Td_nJ2nkR!LsJOE@OB@;lA z79ct(JwLaQwTjeN$}e{b_gEeQUmjRHE_e1B} zcM{RS$wA9N!-A9I4Csm%R zOl6=Um#w>B%9~UweX)1N+4D(j6INNCY}gGf&EnAsr3$%-EPHrNKviZ0^wiuxM`U~U zt+eA&El}c{!wo6TL~F1Ca>JJz&pIi8Z;~mT*N?6a2i})Q@daS!d0}i>RCQ81oi&0YLawURZ$K(r>^6YIC0QoSeA2I#_~qtFK~VVwnrkC3q0T6*%{ z_9YIBXxI@=Aqvl+aW{FFwy>B6NX@jhj|IoY=)Ga2e$L4etT9$iI?Q|TtOdy)4QJp( z|M{+gE4p}bQ|)OOWukXp5)Yl>+#e_U7b|23)XXn@JV6~Mus%)ftok;rnnO-1$B!;1BQP3y6 ztX0q4IKuow7;5{P{L|pr`+Mib?^Q^z=&mcm+u(uKBU(U_w0&yTP#7-DgS5g@qis-+ zXLOpdv_A}zXMtlFfXuMJ_4KVOf2HlpnhD_^zRC3x57@GxA4rj`uG8uN(m>GTtW7D@ zQ6bM0{TM-=nUK9Xq5su<#h;GXQZzP+OX-x_EtZ{u^~A14)0%=Jzm&d*$2>Jbb|Kew zQr%@n4k)s>x)}cR{a&N`ghL`)r^#JioMsi`yVvBHDS_ktrLD$?rcPy)!r8V>@U*`u zXIZ7D=q%4wKC}seQ0DVYt-cSp{JA^}lfJCqSKV7}6%eD`GZDk`5$sGfMohXmS41`6 zyd>^Jz1c+P}I@;C7+CbTh|BIU6f+yFfC zmT0hy-wpn!;&&)wKy}$BDq4^Cy(78V;maP6HEnS9`*Z3Zk=J3hka-FFkxd|y@%tOZ zRiXn`OzEh}60^C>Idlj&18As(D#kH|B;EEWkSu&tcfngJy^-0ed1zIVbfKty-D|gd z94>2n*J{+t8oSlH`Dg_`sM1Ndh?|D-FEH1Yb+L7~xhRpa(Amh-wNQZvnHV@*qjicL zr)(09xrMKug4>MU%_x6+!B8gW%!9YWa4Sm%2-mMP3E1*EU}8GT^#ADX;fWAu;g2tT z086RlpqNj@s*0jpeCUpFzL?Sx;h3qe3(ys%i`OWA*QR+qeUH4bKxI>WO|G`=zUo`# z|2DVD7coPp&=UvVwPa0yg|kgSd;b4sOicfzM*m4u{!7O6ujVKN+rJu||21>^cYpQ& zEOYvI_VqtDN&oKv|2fkCmN_voF#NwVC(UiAO;$v|7rp+pnK>p@N$(^!C#D)F*}Cqw zL(Bz1(FK(p58Cl1^^Mo#H6k6bbN5?`rTd~4k=ZM)KBLWncUWgsvErr zYB_2u>yf0l(PDUM@GNyLdMjl{JyO5o-VU2>mM2?RJ=cp~o!MlRv+WP*Opznk)k5*k zi;N{oAEg`BCbg$)O;d{#-VcqOuOiD%%O;9W$evf1CoxY#j@A&kC`TbCJDv5FPRrK9 z)i{}M7^yllZXW9nOIJO;*}bunBDfn^FLdJyD_#|v+}o4SG@?A|L-4OJXBW2K_lw9u z=hWLFaMBn6S{Ni&In>Q6zOSi767LU?RJ0I@+|LP{LTc;PHoK&*&i5|6v5`ijeB1Q0aO2n3rM`T?d? z6A*^`&7@6BFZfbgg1Y<36SgNO#kAaSra3e6w7q zSJ=lr-acG-hGw0}GglV_FW!w>i%x~btG8C`6T;EIpH?qz6#7F@fC~!jjPRfW?d-|fWpxV5VOF`eti<8N$+kbV%iXKLcQZ5uzB%MP zj&yxi_ernc}G)!?k}`&<1!Y;xLO5|_9rk275G>^Mh4Zvb=CFfz*8)B z{$RY8PLzlf{Ky2-AXWQsdIuTyn$L2oNt=7D$E$v83W0opaa+VJG-t~o06%{FXzbli6O=lIE<6s2*|WU=xqi}cU?-} zL6ib8f=O7_%FAOUb%^8p1zpmNpN%q*%yqFM4k?!hq*sp^GdXyrclp4|!NZV$-f21_ z!c2jXVIt^wtCuO(%n7Tf=SItSf&z z&`81ojBiwBlzkwIoU^KQmPD+$=(Gemse&^O+zVZ#KE4@k&-L{uItHBUEe?$l%FiGO=IQ1r>G0lnP{q-2i0;bGdg|$v=*G{LQZ(Lhm%cwSu@lFmbs+CD}r^ z);@LC2q-PK!p6XqS!j={53XHXnDkW|dGk5kc%BPpQ-1Kr*KUPSgR7b>g<^(Z*k61& z1LG0;dW;2raSyUpFP0-_{I1bX{kU%1G?TBN^`5GofGSm{kBNYMaFU>qn~cq^C3aZI zdXWV^u?Qe7Rcf`87<{AHe#w|QhPzD*oBohlZefO{oL-b!uO5K!vsUtfHR47lpt{s} zwF-$@uB}O>txS=_Xi2T$K3&GSlGzh@y}vYBol|Z%KSw=7BtAYXv)tdn%5>LMJT-{XCm;t)t+v>>5Wwz zi{=l9wMX_;P!RADeNwQau&0}d4+^8wX9b<)qAnU4*ze;b7If8Md28Vt4Ksie9H>tC z_zJ5x01kv|44U$bZsf!6^w<2dgEnm%3g<8LQ;yq~ZM*YyV=rDOJlQN-dK4bzm}yr^ zFVsxr8?EL5k^&o;bw~LOmF^Z2&B-&G_e4Iah&tnNjRL8?c2qJhH zTFG3Rj{(Fe>Y^{M0+D%w@rOI~^3_G;2h(6I&8`7U1tpgdNr!WxksyqR!1K@ZZq)J` z$Wt5H?6oicF#VjV$cx}$uVdzFGKY&v` z$fHPNBr@TMO1A(&!Je#1a!2=?vjtiiLySFQEbqpgyaitAngF*A6R3 z;7hRC3$JA8Nvtlg2MEZWk-W4T%X8pIn)u0$hN<*U?JB2q=X> ziHNaRUQdynfr`9pxSLEH03!Ay%cD=5Hb8pM&f=Tm@lw{d9z@aTfD%O)H&VRuRtRPq zbVQUHN%CM%1FH6@-SOim6Z0X3&@Ye)ETtFnQ_YV;PAXvF#A`?t4}VCTO8~B*niV!t z6EkVYVR*q6F$s8(!&z&l3&y^*Z#%9mGodl;XqYW6K2$r)|!-Cwmoy$SbUoq-2Pl^NDf446lV{YM~|h+?X6`; zB3#T&g&uiifW(mp3x~hgbX{=4bqx(>kQP#m$th^_LygOYnc-c8Qx1f%Kq3I3n3((S z8g2x&G1X#?(}gM3Z*PIv8VQB{yivJ);^ua{So_HS{#=_EVe4~D!wLz6YJ=OMGHQptD)3qV&8J@m z7mZoD>Gyq$K5`et?<|HV>$SSzkBTVIs(oMV!eebD%i^hFQNI4mukwDnD(~ioyuu;t zSCFW8SmmC-E92vj&Qv3F+SK_?{Tu8pG`RTNcDrmhSs5Y6u(5A|HZO6S9^uRE$49$7 zy2Wj5u52u)0ON#d5SmgktC2RmOv(m!;zhbC{i0t_dzOIYXcrlaei*bi=>k^|dKW`D z?E(~C8gek-f_TL@S{m2k<*4ix+xro^i9TOE6o~1-hj0EhR|ogBj~JUAUbz%xg{jv2 z7wUk@HFVoXOTHH3bx>OH$##Ya&~0w$INnckJc2f4wfXO~ajL{vg5n)|!xGdi8nz z2$llf#-8y3=yzyhd1V^~!}RMfMV@KVtc}4U+@cV(e?Kq)M`N^K=5NQS#^3>E#W%7j zFJ%d>i-5=S?}0B?*j86`XHN-gd7XO6Ef>?&nL}D-_6-p8H_>aV6%ZGA`sHIwO*Az8 zj)jT~p+ZHn$xE?H-aGbC)Tpjn_FNg-Z8sZ?v>~>x?j>w{;mgR|xWniX<6GA2qx}7% zL#&SkZuWJ-%CD7E@=|M6DT1YNfUwgmKPyPt7(WxbB$(*_h6%r%kyzwY-h?K*03Vg_ zh>n@0VLs~- z$piW4k_JV5P8$7Vfd*xOS~t71js_Eg_Gy@D`Y{t?kgccgJ$YWi{K z1+-#12}myVVl%*D)%Ei=#};ZelAynLUbMy3?U%(HG5x<(3dIw`xLa?q6&(|(lL+6Q zE7>XY^z?XqC?M0;wl)QcSJ@)c+LKuuR!)8|jXx23q57d`T>GCCtLd7g zik*^m3CLsR-6?R5?3|^~4^k8}vX*bW>@SbI^H~Q@q_Z=Rqr(%B)sG;iZ!Q$6q+D0C zTauIM20DwInfC<^KLyy4hEDHQYBf0k<7CNAnpDm!gkb9qk~K%(ke!uJlKNw^g+7B` zR1ptFWpOF~zjOe0jpT|zRWx*kMgFl=xFTZTfSi;s*MdF705kkzClh*OXu!m1XXaq@ zXNEo1`33o((*8>_Z~VP4GQ9^}3-v@WQj>@-W-Ol}83928g(c%M4h_QnSyF$=74@_i z!eu2cI_)Kf16br^+tkOe>v6Nto()N$xKtd&J@{6lOnWoh?v%$t+8U^x9NMuq1AsuB zX`Qf=gK6QmDOusH&-Ukx8Rk!kS?_LE|#xZV3+CQo?PJI5xa z2x2_kfph3IKf+}p04?6Ih z(omThLt->^X;+5%6Rm&JJz|r5bK|g-0A+k>T>K@}3-hOzp{0BHRd`5*1Gv7RG4)-T zs&WTTv&)um`g|ObTSBK&8Kte8c{-NRB4|iJD+01N`02rEDF}v0=V8%70DcL^`P>-n zn3;Zluec=K1AOsjyr8R@f+ncb2)$?`uqkv7=0K0&&O$M{ECSs-oGmx0WN; zo%?l;(}y)^$d4B+EBNZ+6DjM+yR*Hf!&Hn6dyXUZFJ<%>mUI4LlJvPWChk)%D}oD` zn3JAO>?lwixYZQJH)+dOUCwr$(CZQHhO+wRlWJ9lR8iAktL^D4c@@c27 z8gNP_ZG@qM9?P?)FrgSm?q@>?X&sQib~Ri3q`!od1js(nj+i-$xPJA!_*a1_qGq6F z^!0VwMZmtiIWgbo4=QBMKoDlO zuqO{x7HPdqazIYB?jD(O%-;hK6?gLT9LbLMDw9ubuO>Zt1 zpPb1auGp?X#6UWbvG?{wyj;YCQNpFcjjve9jCf1jvsUhlQSFGWbxFYO0+&Rv3#zsD zq`s>xJtSgg`wD3=+F^c4Zdg_+kO`>uD*`!}Xn~oAJ7YCd{&g<#E`NSH)j6o<5|vD| z6QIp&thsD;t8qOT@uLon+X4JD0w$9^z^#>71^?8@B1$3kUp{f_zXe#4P55o#2;gFp@kSJoeb zuQ*BAoSU^pT+esCUep&e3po1DHBR5SxZ$hl7g#l$7wxDL;y`X&8Vx;>v(m~Rg&XD* z9j-MLO&-!a63zuAty*{vY*?;^&SyVeK;w7iW!VOaYy+1+mW0#sscU9ip=-qNtQUkwh7`YD{`B;VrCfUerKTau+5zN%!@O~vF=!kl&W5oy zoWtCGi=E3bDz1^CiXI*o0KBnsO;LcprGm4*cl@45C+gN^arAO*A0S8CUM}oB@Lqn= ze)QR0Gxgvc0fO$~k)^S=~<^^&mY9-L(4=KEt=F+`I z2Ptk#abBjmv|_7W2_%j%%p8D4@7W{}y;{%Y?Rpijk^$es$ zV*v0s>JQ_z*;L|LBq}VpVc}cH3sB3XX~7P&`y|+t_pi5ZA4MPO>&NQwJ~$C}6!qK4 zzlt}~eEOhO=;la{y+Cpcor+tFYH{OyK>B-S20xc9E7mBM?x@14yO2mUa`F7<=5P8$#*iCo%f3wh#Y79@%#7{S{pnuewT)Z zgCPh*3^2I76-5JOOD~C4>ni`WoQNT!%_v@z3{D@5$OST305KFH9}<ZK5~{TaVp6ci);?zo2P?~aVZVSo%4)xf9)bqXciPkk7!oo(Kuxk3~D zGqg33h~8rsLgWQ(iulA;OuX{^T%jE2%0iNo4Q8P(|KydWxwyM&Jd3gAFcu4 zaLQr;?q&ac*vdmtZurabi{W8q2zdq9R|_AX7ji$p=$6e#HgC5T(zz#~JJ^-1UA7K0_Xx{n3{1AAs9y=m>g3}5 z`r)MZ5z6nE`c0i|{FxROU`XeF1vPh$#+woD1(07pv)<4vEf9H1T7J?DJaK49L zQi68jbiQ+RH7#8aWqL{r|^kemCw7v2m%FtpRR25-7JxV1#H{8Il z@7T+FswfRoRkrbEOQcdUya?GN4YjqkXd#TupwMYm^qgLjCvBmr?C>054FRgqfe{vi|rIO-{(A8#gc z*?GhZY9WORmi_)H9dK&q`hUlNXgE2O&NLPDWGoWbBE6!#k1f|gfj_-gn&+|)rpUK` zRp~bct%9pG_F1wTJUfgGQ9bQHo8QXMWqVv&tK4+%k?)45TvuRgG~aNz*?8R2y*h_{n?GMScQ%Cz5+a21+B~Tu(`eHlE{GZi z_ffH*#T#8`w+SAi7;MSG@;~CyZe1HHj9|2GU$eKL6`mFj4IVa~!Qg}q-UDz&AP!Rb zqxRU4uck*B?pik8ekQ7&#w0v7$f(g_peP`ofQS?-6=;uyB+nb| zaxA0;Dy=VokA0?W09wPNedyEsMt+WBRn*BhdHwevZla*bs|w z=1PFbjL6vEg>=kMO1$0rrtV^j;tPnFq-H@v(wvM%#05|7JN9EFYlcWOTT81q6WmI#ykh0>q|gC!u1#$XMfMz z`43R>{90I~a&EhkN3seuiV&=?QX0Rk@4`!|Fjx&P*f-5@F|tpH;GQNjw;+e18m?1V zVxj4jwak2Fi9xs83`h^c@_@Gj{syF#l4OXo-_y}_ zF;a^MefnWWZUUmh9V{;NA-D?MfKtE~Mnk!`HTy<^lhCLZ4rZUU@rOX7UT%2Vd#EN& z#t?t7JDA_giwyO7aI0%A?aRq&U*Hu&GbL~Ik27Np1ZEBw^*oYIM9wn>Nk!&(9UVgd z;Z+n9+7Cz`6i$xO+hGbq!j;{mmaT!H!QEzavMx6szf+TjIr73VoQdMHMbv{fa>^LB z_X^yAMc1SiB(9M|zLQLADXd+Gz|3*|MKVXRi~7#8_R48$zd>UsXrO_nV2}X?!U4b$(I13H=M2}CIMNSB(g++R4z%9v zQV$8YMTeTQx{`~S5d+Wy{UO>#&ec^z$Rfl;ut^CpH0+O=D@*^Z_U{<}zEd6UwST;SC@bK z8{pOQ9E&o>*EBhOi_88qJss17achQ`AEOJL~0sb)0q0_h5=EQ4v`LbgpSm*Nrc$G1o zD8~!H`_sshaqtdw6ozQCJeEDr`xAF`eV8gN0|B&jY5aE9C*Iy68EW~e#7M>4xAWuu zd~w+aa#Ef3`k^EvG#9cz#WM;W$0s(tSMqeT^jf>otfI7!0YRXIvf!SJ?L8q98^w1D z`1Q=uHE4hY3C`KKM9-=%)&UDwm=8e>kq^|L02YWTe6^kO4*^|(%uKa{N46;4 zOkS}D_1QGl54VC0-x-*tDjTnlhxesTU@yt?0ZS2|RgGEudHjJb*hae(!#cW8 z2V~#IhraZmv0GOBdqp*3mM@cqG@M>zB1?paBM1HPA8@Artd@TPhW!KO`X|Elf0@xt z|3P&A-!h|_82=qC`%lbhMvnifUGm>1_~*3$6R^d^$i(ztfURi_Yg?RF#P1s2dcY`` z+PmtvZzKXU=6`SahPhRiW7L3J|u_kVJSHm1% za@SV}$DawNul7Hg)0|why=9*sxAWgSvz{3l%7vMFoY*kcj80@FCwEr2ANb0SLbE;j zHyoiS$$z*lwTf1GdRxtMu6Zy+cCaWzX{j*JPD9?kw#7sZ`c96_>1|HPv!Tg6GKjHc zi`w&K{<@pR{5ZJfad{jke3Xqik6dP%`ILj*cJc$e8x$}X$+X8g6q@TuVxxN9G^j2W z`?@YVtX;Jua8`CbDnABfDx!B)a~bx}uUjN^RG$Wc<7RwtpgSARZqSEUS?L8?a*Meq zSN`$%K~jC|t$EQtuU<5{ZCm#w5?r3n6kN()oWIbB^IjB7%N$ut-)pjwLRX-ud|AO# zglw{1M>5-`28z~>=lHn?rW)a<1nRGO(Ip$z~&U$i$vBXh3*ZfugEX}O&QcS)J3X~$Zf^{F=M(N2Uc%2^X#WM}`tzg47u{`#3EA%aT$K<#U9@GTzuN)0;PKFS^14#`wqzg4lebtT8p#M6gN5_yLc|JNbRw zCN!n00&nVHIc6ydtW9=211!iRUh)E(-h%EbA>;$xf)+3i{zIf@2E>VFKQund02vDyNcgVHPn6}ySFDe!jOwvt1 zbjH~}BNdm)$B{=H)a{O9t1kN5M5*-Zb4&ymB)af%XlA^E7I!p7JyHVuPeD|V%?GH( zYSPOH2^*5kL+nf9dG~lnG>%eh4pW7C3s^8{$pVJ(WBy}FG?LilV`ppw&)l8T<=%vH zS}D+6Len!W% zV)+az_pdJe2o1R7mktYkIHzfUB)C5;I7aB2ux{u4UJT->OdP>NHl%@}rQK*2q!Rr{ z3L7d4wDZj1>Jpof!~*IT2gM6S_+tUEOb(4XChNb*E6#_QwcP<=i6nbK<|ckExS^7W z$IxG9e`UMVc&6%5C0gb5OR3k$oE&d(i~&VI_9vu&26Ve4w=&+Pq8vqt_fZH$4fN;W zz>N2^m$Qmm2X7!*c7Z2;_~&I2j^;TcNxIN80;Gnwd?mw?oCpx?RqK%G{8Qg#gLzODX5WQxx z2zI(1<*K7vBS%75v1cRjWvzHXgshV)Yff-zJ+N@i-a%5BOh|_4lVhg`cUNB?TD6M~ ztVOCdOJ~qOZ2moVf^)YFgo!wBn}b^@@_Ip`&^en4mKF3|GChZS=0Ok#g@AFFJVq6V zIYt&Ho-brArUcQhMhxMpUafdvyJ!5{gaEy81F0wr+u8M&kJ8@?UzVBf8xE(mQ*sn@ zeD%w-fJ1XJi_UzXtK3V`Yd}XpP+W{tkeG;>NRXIyHb%4=GwcZXAiA(lgFCe@)vN5U zs47+W+I0!yS;rh-XI0meX8XBznEtRdx$XqDv4JR6rsz%CSF9VT4_>j*9Xh7S*^A{3 zRSCn*X7B!o&V$JMKHd~`(6=j5I52#1QB*$sskmh|e);cPVdhHzl`)|3EWas%ArQt# z04bxjBxX`V21GwShky~w&k{oYZ*w*P(z=Wf<-f2x2;+fIA4xownI8apo1zjKZ(d%u zAM}cNT*hk&BLdL3e^fBR(jN6;1U|>OqqoVt@WT8q%$P))-&+y2PIRAu$Rh^LGPb-) zE3;=a_YdtPG1>;58LZC>kUNwJ4>`AfaOcy^mf-D04|aEA(kt4Do&w@*t+xsCxQ(l1 zsVG&q;pvB2siJ61Qn7!V32e3ZcGcK~*k zLae?xi`xhb1)r_HOvW6I{OxNRo=_S@+2|Qt%LlfbN+Te5F62BGE%B6qx*4UR$3aIo(K;ThwDtUbUK*}OAxLO5xZp;!3T#lp+?Vo6Atu+WE^rM%>8jdLg(4_QR7IE?2DZGy{qZpNrRyyhSkcL6rb={G+V*UtGW;vv2Ruas~#dGBFh*|>@l{Xw5<>HdS zTkIo;ckv&>G)odFBuA!yAKkVmO|e5}Gdlm=2i#7g6TiErLU zBDxjjY|41;pJZv*fM6Gp@Iqk|l!hfRz5IH7^kdrv4@06z4 z;2tEr4lFrb#cj-+Yfpv|Y6X^dfLOg}5}14vu_$66%^OQA=!E~Ul?d{iK_hd2YaPI{ zz8kJQ7E_^gz}p=sj-Y)Xi*z3cR?T=k+i7qZc356!PT&u%We!t2!7}V8;t0OR$O`oz209>D{OE@84 z&nOqL@2D#n3!d#Rrr8^g6Ufl6w438M;c;(~*TjAzp{xYjJqP-1+iPYNKQIa3PFw4s zZj8Nea7b1pQwPFmQ^M}-@r$6(?7L8~a|IuU{kh?gDexkRKIHy=zEZ6Q*gg+}E-~LS z0B=Qf=O>7RAFbpPt$Yf<5u9VoC(#+Z#S1 zeNz_jLd(Yuot4)ncZY^t8Yj#BF;kT&xh>T-(k7zwv) zh2;&^*F*Fgqn8b5I$3VoCa0D*p36VNRGz;kN`Z-QS-DseP zZmv&uaI5HGMBJBZT6La?EO)sbN+)iXP{W`wRZre**)eTdPbI$$R-hHaD1x3%dE>KW+OcQ%NtOtXP0tJ_KIxMr zN?{T;JFI&a5Q7cQ;jmJX;@4LQEtO)w3SPtz5)2yDIU^R}50wb*ruX9vvujWafx4E= z80#6jn(eU%HK&D3FVgP<%AuF>E7ht6dB#jk_p__=H=X+E)2g~X3}yEiaU8m5ia{XP z{nJhMH6u{kmI;Q2%A%YJpw@`?MEO9VimdnW7oCm+#~my^aG{%*LZ$Bo$+_2Am=hjP zW8T^T(Lr%nRD4yvq9Wz-j1Wt#1M`-@3Vg5_ZjE?vKH2R5I`?Dn!sc~^jNWI0We)Ld zDgd|ZRsUm@;qbL~DR~l46_!4_L!b}i-S@P2Jztrp>K7UX=m{g@VvWVd#2n0wnig8n z6f00&iPd=~ioEkT$Ov|*fp9bQ^JBTIez0IIFdFgcb*GDvV`*ELj}5+u zuWJbWL)p0Z;M@+g^Ht7w7thqybvnQE>d@{R(GdS*b91rv>eTIw^*(ERHFo;2j>|Va z06+BRYhhG#=?6UcwFO%N;=<@<_C7bX^fnVbIP7%$&D$dptP#=}dE)44*PiAyqstHUYK7Yx&nWbpF zx?1On-dH~0>?2cSjb!yVbEAv)o5S{8UBl}N<;pEqo6FR2U--8$rd?e+za4pk(R|)5 ze`zv573fT-6sPAL%Yv<1$0@anbJE$Uclfp*Hs>&GNI9Y+{X&({BW^>q0>y^Ug0?a4 z>f0(nyJA-bPMtFMr|ORcauc}nX*mH+!hf%Ov}O`m%U%os(c2+N&M! zO|pNWW3UP8)yYel9~v`tv`AhO5j6~ z^pd82!#)WLG5ON`uKb?cg7P(vUsU)hUxrll4R(|KLVa%@=!GaVZubp1ivKdK{(Jeg z{HlU4=y%FHRr;lIuX|Nw@i)|^*|_c``oZ0d#9#y2@zU1D?mf^YxlHAU;`Qgal5{BA1AIXl_ zYqrF%9x|jMpN?mWI^JO1lKZ#To{yF0*K@Yx-UfZuM;yuCIuBVB-*mVPD+NcqsgDXO z1C4PQ`?XO^PeVxfXYzY0P`;-mJ%lOTmImRr;qj&K*I6Ff+>4o+rY4Jl(k17QMj1U~ zCFk0V(z;}ef0P-ed;dn>jMBR#i+?Wv=A=t7l^Cb<63xL9rI^-?G6-c$Fe8jI`bigI zj2Wlp{+=TLM&yeygy|+_iRK|>=_YD_BWuQK4bsIbXU6FblEo@(jVR6^Et2-rBaOl1 z@XqhYm>b$dcJ18<0RlFLo4suWzFy3jP-2sDSzcBfd^$G|3)3G!P-lY7f3alzXR7tT z_o~SnSSUJK|I=uZfQj*M$K1bGIg}ybVEISN!O@9;{lBd!*P7;zn<7YGJ-r6J8_Jda zTcW=*X*L@SRoBBMR2pXXB?L%F41@t;AS7ITzIyQDm}xEuAlGSb;w zRXgcVf{g#G^fY?gF%t&))0{ZRMISfFWhF13)ulP*)L#i zMY;o}#7E6Etptnamv38L-{}VLXXwUP894RllhRy$yFX2BbdX1y%lYuRhJJ0xl$1p zYCVUtcnw!9lRee376|oM_S-O#V1^NYb~XRnZrGr~&#FHM7>5q$6|Z2hzjel>D2~Yu zIEJ>F-BpIYKV7wX^HHijH&o52xg_1-D=Tl;FfdbW2B;3ZceC+OjUJwhScMO*DYqyg zY{_9P=ZFgNqH?p1A#AYM*zE}RWU-I}r=2_MA^F^<9x ztl_FfLi7F82R@fg#!jGpL*pi1;|#iFDROq^+GJ#xE>fa(SWhWA&}$1NvBTW6Op1#g zV5l|~0e>!qPA z@LWj(zt3xgIFS&N>Vc)`n??-?oDZi*4GCtWp{An*ys#x(s&}Gc{_0Otm5l6zKtH%M z>(r)~M$=+d2SuoHj&~^gBCB(=tDNj+cOe-amE7&MtC1%$>|+ z4Bn1DSo;kzqeJxVFNqs}nyQtoy=b{CziAO9zVcD$M-L#5IPVjV@AJ;*%~k6iG)ULd z5>w}8vTbhSGSRv^OR_3}q((O&X2l(v92EJCV`TXmQ#Y`o+AcNjJaj#m7XZJqEM0B_ zmp6ACpAZ#Pv_X9|R5UF&3AkT8aZvCNEi@Ssh_qrL;L<#9+HoTdfnTO`zwc|hF(Jp3 z76OATD=+&7I4vGtyYZ;13dl(Hs)7XEom9hnnP-th$fIc61<{0#`-vxj*@u4b;>YHndI_eSq2^GKYof@Mfwl|bmRN-XIXgk5cds#a z7m(58Ul0O6LvUi$w()}|6LqYuB;h|6d)yTRPl=Fn7O*9c$OF#rVcOvrpOQyf;|nus z_(DXUIZ6(_BNVwr@m4-9XwRE=w{&p455bzR<}VlW9)rY15Qs$uSs~t&a_{XC#XmY0p%-gW9Xp)%US$z;Wfv@dv6G#x7syZPF!<+o)&Ruv*1r1d!QDV z40f-O=gF|k$nS#&;cdyhwhG6TPfU76Q5#ygaUS0*N~nf*H#hu{y-UxP5+;C}wEugIj&0M4>fxM^CsirAl1` z^E%pP1_-T{7Q?4ilT04JQA7^ z%(e{4KCz%|rI%3(SzdBa4s1DVjfGeW5F%@RyL~f%Zo7?%b^mJl`d5T^pJJ?FfCV$Z zlBkkxWS}j8Bn>OGY1!0gTM8FfsT+LKE3U?mcgAPjb)Ix%`d?-Oa?%;^P2;nkYr+c) z*j_@zW2n~Aa^v~tA29MtS4|r38nbZ*@6^f=SvR4#3qu4d?Sj+l_@!jIlK+HF#3T+%wgaB*I zC+8q&T{SSo&Nl`Myj8B_!QJzw7imGn$Pu`we_UYtfU8TaW1kz@cJY5QkH}bcvZ%y% z=X=>h&s-RGr1SS$+U|=e9R#00NFD>fY85u*6rlUGQGI>bOUtmI_sZM?M^&HhOf~`w zOKo47r0lX(l?A*-j+6Fxjobomd`aw(_Iyr6FNmvm8(blRPY=!Omn7XC6qtosC`yuc zxhKhlnJDzMiS2R^m)qFQPg|wkIR~&O8|FVCiAb`I#x9^_mULE}0tmUvO<`+>sIF3v zX|kF9CI{dfN}VJKJcUhx8h|(6W@F5Bc_+=;c(IFMU#?IL8>^%~&el05e@XeRFRHkQ z#JR_(ssEJE!v?!5?RjlMOB^^^~6-KU@&m1PemPd9lpPGpkM%HN_d3Mk5q<^7sk zb=Ey`B-)7ox!a#>LgIUZR~FX?xi2rgeHC{!{rMG2FKf{LIAEz{t)<|s4u38u_AENT zR9Ug)NNTe9fJ-|Jnv{(y{-v%QU`D-8N7XuHV&w{iFx4YsNc zkt{9d@!YHi0_3#{mql2BT=&F)7>m%vB=(#F%3xB~EBlb-u=)H< z$R&M7(&&~=AzMo^TSGCsTf}+4;$<$W^^A6>fU9+Go6gA!q)yl&Rj?Qi;_9(7-#c&F z6{?t#PYL||E%n)()rOw&Er zqllGC{<6eW!TX}V=wgQ^ivP;x#a6%UL!LXjD4Gb<%pZfv0?$*s(_W<(&mlnx?c}^h zJMtyKvkZ?NdPHd{CSbDIbtW-Ofc+x(jCrB0_=5aN<(we2Bv3NazQbpGYLUWFg@>~}Way#zLEZ$|Tr zLwkGP>QVGC3NN`_(@lH@c^Lw6OU{v*=m?bZHm7sNLr$ya)n>rmQe&V?4{|e&)nlX{McEUZT+{Sm)y_iPt6U(rtR5KMQ0~Gt?-YN zpOQr_l2skz#@l;OiWeQgU$XQIV*E)ZRfshr5dIEgLq+=!`6*JIGy%tdOx z9FqyCv>gg$*B1XG9x2=ua%^I27jK6e^GB7?y1{aGWA+J{0AhqL$-sbbDIwy6j_MXT zw!KWWE6XKH^23hD?`=#lvV1;ehLt;nGmvuePB-lP&gW3*RvH4|5NdwkPoUTHJKQ-2 zqyb_PBJw@g&gk)31c}owO%i%(-*=-kl4OZ|<-dxHeO@JM9d`7;)d}QKf`$H+ng_J# z^mLhZ#EH6pD#YY98sU05yxd5?pv$ECL*6lDD+xx(_Hl9mvHsuU>AxA+PpfI#DzPE>+}7R^a}Qmn*Ymgp$3d6_lnWx}E7Kk5 ztEZ3S~UPVWDS`|M98t{tlv(hW5%8uL>vVaj?t2F4<$#4I+46GrC?>_ zHqKwLV{2!{%`b}sWd}q8IuW54&c~@lqood=ohdMIwoi$erY@T~ONgMCuAf~<0AvSJ zxrH(c;1;ecmx-ur(SXW+q77ll(U;jb5E~p)pgrq?rH0r00gR-I>6e1oB~5`ePWyz& zDSV=iV3#N}NF*NW0im{F3qu_STg&5&vXl(qf+k4XY5gifbI>Bqq^FLsAKK4!gMOBz zN4?Y6PtR*CcV=}-9MK5RvIq4WP!s#c%Chwr=XNQb5%8FtiACX6A z!URunWP~H33StmwAacUs6AcD!fvpd-CumP=Mc}D#_&cbuZ;@ztwNnP@VXt@SVtR)HY&fP^NSc-h7WjUU%afbKFoTTJkK_JV;4RNni(sxRA zB%V7*QI5&-aa}E>vi*qIY30c*SbBs{*1F7fr0$g2qo^zkt818!=>f40>zJ`wU%j^` zyH)17u_4Qtmpl9HIbN^!te$-M@L!?!J#MX^P3&2Fv)<@+yuJ$S1b zpI%KKuKCLrEK%6a`0%5H=ilRA^-eA^D=puD%)J`;W<(9gF)dxyqAh`4A&ujnxP@G* z5f21UQ93AB9*ue8U9(TNOQcJ1w4l(s^oz$-Aria#dW@wjm ze6aYq-mxt{cPTkIM4Na;K>@jWoJ{i@4{xF$S4YD_-ok^US3fS@JQ%xhYQkYR5Ojw{ zD0zv8NuZUpYBIb=)axAdx+PSNwDOhP16AbLHKD07hksW`R`7(dwwL9xRYjlyMJtLF3tu{ORp<2Jh9L4$dv`^aLLt?7sX%rYsWNx|2$seqIlbQR z?U^~>p_xLpo3*@Oo=wktJ%6s6a?Rpd^XT^Ak1XmplWNYMR_aR*Jg*S1?`-T8dwmp< zim9*?%JD<=jS}s2xDe%XR=@KxqP4Af$}$2&Fs}#SCXn@9#MDY6t2Qk}luA2k>p6Xw zVS1~{gf&CHMjVRn!_CJ~j#6n@)(3{w!y87*4uf$7fqs5h9{?ez+N`A(fHY!S?#QwB zns?>Fq_ZYr6z^I)-CGw}a>{BWYwP+cdk?4eW5hc&n%el0*2Nx^vCxu4!h0?DwE><& z0DC%%ByBoN;;YdgPDmG?fkk1ea%K0gA)Ci2(%A9)M`X_sOfeK>+mskAid4ee_{(=2 z-r6YEjJ;&rko^HK=iq7W3-=Ps_7t3Flz2=yu(TU7(IXQr^i@B-t*@w&b#Xx*(dQ)+ zGDqBt<0V$mP<_|>AlM;O9}0J6F#sBuzHR9PB;4I<1;c2RaZIz+mzNgGG%W`dIhEgJ zlUKB(Ez4PodIeJ>K4`$ahox6YQSRccE2h3l#+m;po#7b|!?Dn9X6Z}W^=KyOKFBEn zZTUpwd;!sU6waiq1Z+uK?z*$|r!i)+tRZ^ zWc_A{2wmaPrW`P4x#o^@8%I)T;Y^9x;z5gR8;60q`8GgvFDd>X3B%`RHdkT!rE$8t zAAr|!(7yloTlH_gDgTST0R{pF4%WY@o|LhTsgoHa0RuBLJ=;ISf6!L{BSXNz!ph9? zUt0K@U5%A8FI#AbyZQYKNZUF)JGq7>`~dp@4pB?nIwhe|TwPlaYK9}xXtgi1x>t7& zKf8&E-yWpAcPMQ|D8!YQFomVBqXZOOS`b83$k2HG0tzy+VdqkkgXO1YHk8nSn!qvv z*3#6}*yAWEhj9h?A=YM8fFJp*7o$3PittIg}E3L0I5I%D*+%54VYg;QTuSV6m0A{xt9Z&U;I${hkUkj zAKz7299BL$_E$pHl>$t;h+VT|6Hpf~uE0uOFVvxx<-3MV80gt8fG-xX96+X@ zRW`qGR?*eB%0^ep!}7T@+#7oRsv!UOP}BD&&K?NRmnar=-lA>~)URJq0N4a;Xh031 z8yQ_#f_@05!q)(vioEZN_&X|08I2O=kT9pd_BNgz9dU1DoChE%WL|}?l|3Aq9R;s zm+OrK+62BEzL8~>1pVRr1_wa(j*kKAy5P!-s>|E5qo=7X$&dJs69IZxAPr8>WnC^a4 zr6x5tHzUn~2|fBIKlNJsv+7yC{4{;jAzRPk4>0=*ex`z5k$}1Yr&st9lA-yoKHI+S zrVCk5R|^1$sKAk-xzM)svSj$8O}#rO=-LwBytt9qpy8U)f) zd1Ev1J$|Me`mlifWb=qNc%U1a7=Y3P`vUIq$oK(v07#?x2IbH>S8LnU^t1ZWzsK6T z*o}Ki9_4GQs-3_7zW&MBn0ek^*NL39f_DJfO#dD{WGOu4+pB~;(v5p$;T&4N!M|9& zn)wN_s`Y5?-Sc&kI-bRSUe##$>KZ2t*fCX8`WA+WAHzl7i9lS1)F5UtC^o4k~% z@0rlm{rQ;Q<3p1B{@{EKZB0wd&Lk%$r-Jv3|7w=1%BsEg;=yly@qNws^q&20o~>dQ*X!%+$o&;*V_3(&`$Jd@4Azd{?A~H*f0;sM7i5-{aKOhy5zFFa3~-Q z+i0x5{zZwnl5x5qFlXriw4l=q8k261pC*oCmK02dhpn_N&B9~ANtSN}r=+iaPA2b= zJrlIoT%V^6vn!&p%@VJKwU91FAHq6~oaoy=+>Q2|b!pA|x8yx&7Yt@ZzSi zzk;x{`lk>U%loio`kWq~MnWq^*=10#$%9ijFF?rgQfR>dG8u2GVY4pcMcN8aNJKQ-v030|LDnKDF(e)+Gu!e=_EVf1 zQ?ZHcX&|LMd;Xk29=m%d(xvl+^tSAIdO4CW%cDR17k~zwO-4Rr;v2O}QDrpRz;w%B zOK;R*El7|bEE4$`BecAcX`n6?6TDv9&0zj1n^GYc&o{B8oA=px{KG+)R_#qN7urNv zT0R5htWyxA1zVl1nV8l+dj7)tZP}O`u1c;%s7@F}1u5YNK*G`y&ouWzC7|ryLJAcP zW64{kR2c~wVp4Zy=;!wL@MqQ@7_PlG)*522$eBvqoZI77ILMdk4>oHEn9Sm#5=Chx zHN|Jgjfm+Ma<_bz<7VCxp+68sDBW$Y)JvnqptK*+YJw1Md{u9_F3u8ky)+kKDkg$z zjK(}<`EG3ka~?|WTGTsd^>AJKxZG5W4Ck@LSi|YiQ@*T2iUh{8o&xB%ym+-jbz|;J z$sRCp2U4%6a!C8cESE=P-N^H>zgk7#|>ZB_v4<*;AKy)>|L`9}8 zYg;OSCpfddaAx7t>fc!=sGcALWEyk_71KTwyd4O%t;JPWE z6~_h*Q(}6&jb>lYzHK0}th9yY&w%SSa*{X3X3TPGn)yShtHknUE|-P%D3a{AKLn|S z8pmWqs;yMtFRLxhT`U;{)>+izzdieuo0*;EvfN)Z>p4+-_@?IlB5KL@#)aG9<4QXq z$v5N0EzPFxF3q;d+>9;K$3IPos;is3-&0^@53}^(n{2akLR~hk3v4Z#j^{=e?TSSo zOc}0&(V81Kh@*c0HSpS^ST*N{qcc35(!y@tQvek~+?JOaN3Pivt+(l@GP;qiJC5s; z5$SQ+Kb&XB$3@xp33XHl_!{Dh`{zpq4eG>3we>(}z=mvv?s>*6p#T}G%wS>-usu=y zZrl>~#*89l`aFcXcr>bX*ArtB#g+m^ZbJnsM(@tF-eJyD>}QHX>7Bv)Xv!8U@J>F1 zVc$`X0kI?tt^Kz&w|XAd)-;p;_b0}o<8S_1Kbwe-+!d=4LdqaG%LtJ8*^`?j*6yti z4vb-kJ5ll9uq|}~=|-0J>gEN|x1Z6U^1$htnj9!Chiva0@wEyut)B|FHIoIr**LJ@ zFxW6c@{QUB3Q8-~aqYEXJWtFjY1=$t44orwk^2=My;f08WNGEp!RtO~F1F83o-h-t z`h{I_9enx7W9dQLE6(G#&&&m&!(n#*zPOLm$GnmT9(fTeO%a_F;Rbf{6Nn`WRmRn< z>^CxLH#^2h^9x`lzt{7&x(4S?;*}pSVvvoW&!!dU_OIMtVlkXP^!i`Ep)e`rBXHP_{*+s~&+Fi? zaFV7Br9s-a=_5MF1UILK95&}(5nZftBk7!|3cI7&Ic+y+Vhri{RHmwaQ1`cpSmnU6cdlMw`|Jmr~y5cWiiD*CL(>>*Wmc(Tz1~*>K8UiY|YU)eIyM zyzLBe{D@#pESyq9`divF^x&z+s!mCB_jE`KX01v|8-9k%uuJlAz%Ju$DtiwebE%qT z0cL%X#)coAjIUBN(D#(c=+f4{J!Do$Kd6{OC85*|sPq>0*W(0B z($UkA7owr%4c+qP}nwr$(CZQK0!4PM0Wh#qu}s)jYp+&folv-$G6ZZakxzfdq$ zA{=oJoDaQr=%o%|c!+V-JR2`~I8(8j!1}3^;zVr8je5%~Nh1}LClri)ji0$ zatVtgdS7XO-t{a5eS0xllSDi?_kle2T@On}+2HzKVBQbTY$coWCP{Z{89&kX5cZBe zdn-?^m!8)~c}O&pIjCir$>kUp@nVACSIg& z5r>XjO#nBYC<|e1>cP|{Sfbt+Lo9p@XayNXC;X&TAip6;w#UKR?wKx#KB|Q_OK%98 z_V|z{?5{;op@yQlbDM*d%22`x&BTx}^2@jNJuKF7pskgt5PW1#TiR@BRp1VW8r*&` zFrXOYOdbW1f-e?WWu(NjmyS~N^$3aadVt8Ul|+EySZsp$gC~DJ%7KHaeaeP8EnoHX zq1A%4-vIm0pfWNf9Tn}$O>Gi z3UVZyt1C{{|2$NYCQgh6I53HtepxB^U5Sx5EiyfGf1uWjRyM8RNk$cJ_F**leARb` zR7w2~yQ%sF6|OjYq%%QSEWdxk{-EacT2)SmIq>rC>3mRTA-x%YG90cO1p9h-J06TX zO6J$#l3&m1>0tO2BO2wzK%4mwuRxF+4YNgle z{zx^wqm2-ABtlnD2gtfqb5WB}_k@Y9uB-u#aSWYom5Mm4&TUE$lIZU<3pqJPh?mxQZ+u3=f~qt-29|NQ>)iR2^hUm&>{m1F)c%kUKQ|E~OJ4XAhSs zU~V9IyLL=}xOAsGlh4NB+u+C&;_T%SWsU#(%P70^OaP@U293g1#lv2?Ja%5^ypSks zr&-Z+2v3DO<#rmCRaH!qo}@u)FVP>(0Mzod)<}KfHA_ZE@2SpC=Yr z7EQaTGqIG|Q9)DzSLvO!5aaoB2?JWi;Sv4$pBZJ=nkI1B3(h-8ErFTvfr^hDpUUeJ z^4X*Q7V`V1-yl#9eQIDK=J?j=Qv_X<`Z5GOfd{XZC<7#8n!_Wc4eo2m6;=E;770N74orP{;!yj4M|T)5)>%^OZ=PS`utFw2l7uG<-e z(d=fV-mC^u(5uwhZNAozysZ%#ZSj*=d{e%;R7+6OFck8QO#21iT4yl1av<-S1-#Xs zsS7K09Qu?hM@R-OTc|~yECsxw%1B1)YIp64zt8T=M$XT*K7I*?Fpe>RiJk ztxcIKLxUmWAPU!e<2=jm9YO9&2HRI!zlwtA)*m((D}+Z{M8@!Wc&s=Y!LfosBdmU9 zICWu--}|`BJaV)G7}CNJ_%ZJx{q4TUg$|-(cPGRJ$2d)Xm-!IM{JsC2x6fj>IO_FB z*(b}fpLa9ei!2`Fz2!JFqBN(UP^>o)G5R*5O@X;bKLtHU(qm0b6Rv<$_ZdKy$`sdi zSXyD2lPi-m8z}Z;xA`E?W0@meEv4qv?Qi)0x@km8bX|dSzP9l4Y2n?EnQT!iOsY*2 zh<)1PMrMXr>+5lWA^U|3UM9;v`6pwc2%_k;e%Pt-3j}HQBMIiO@WajiCB)1bs-;Q{ zwh$2R0wI@VcFdjAB9^fJz)w)G$XG<^+|APxezDFf{qLs-@O5BCaMgdr{lT&5QTy{iHBEsHoN9{whsKM_%9yAWM{oL0(ngQKe0dnwkm-8$2 zpMmGg6MDXD;%0)8?h+0gVf#C#&a*}o%e0z$>ck=D+fY`fiHgNq6obN10oyb)k&DAK z2w}m?y#Trq7D@n6=Zr*?rj>}Y;~6S?32YyyLQ{}wb783)UWo4HQ<=l}%cBaacF%h8 zCN7G_o{(xyDEP13@6Y=SuV~z(4VAT?RNJ1$6t0yV)bEm7X3N0*B5YY9U|CN7h`*nN zNtQ=_?TM;OvuVNDqtOb8=Nyu{5hR+E?;$qLb>&@eCu!w^hoSCbBUJ%aAfFgMowNew z!np>C7SaQuq_@Q)zuHE4fFVP(==hzA@NJ4WUlt8SJ4YA>rHlBll;G)}8Ou>dd}b6D z{afTf^c*yTKNT8RwXYIv#2bT#4US=OvjDk!kKW7=ut zfv7%TP_IHX= zdeWm+?6gJJ#o|`jbk{@jgpB+4sZ`&5nrUh1%|`hV4q#-%6t@;fC818FY4XUS&h~}l z7#GBDZY$R_ci;a^xfu4cpE{}Boz>hqk2NM91Dr&&FKguIzrv#{5C|!~Vt6ES9;f2r zRhyL7fV^na zNAex%3VsZFb;*5$Z3CgK^Lm`R8FSl#;cVn$PQs}`UUep@&Za1c-~xiFWKKW22im+j zk?4886voRU!wP$%&l5pF1cp?O5SiuBex2m*o|5O4*uF=zF4L27dUAc>L1z}{ zyj>Z3wqA%5`0X^I?LQabZ_HP$`SdRxb$9%V=DTqy$v?X%g_Sc-rpza$`|6+-iI~PX zB<5z+Q7lTCHitFwin)?QM58dihGK56W2Ml8V+9Np@;Tw%OF4mvbWeq5D0kh&t9y0) z6q9&@y@iRpDa0{k7)siRfAfbq(oJ~Y;4p8Ga~MYuG*}6}=%uU@QVGmwle<8|TvP_) zz61li;*$A0d>&Ekas(t_Sp?+Q?;CS6$=cVOdtH@hopB{M;fB~xntswlBmIG zMPxj^HQ@%1ifhAgn}x?!S{a0_JF;1?N6+}0Wd9JwD}hS=&rugsRCyb>tp1*~8+{ez z1VsF69J#V)3SsPxYo46H?T@(F$4l!s9oN{1kk~eR>Deg0+2MT83B_$#Ppb@JZ1z~W z8f*YbJdk3@0TixfJmq-4JzRu>`-_vjHjc`BY4_7~ecn=^cLy8kN-)G*$_7Z#Yd? zzz4e7Vq_vpS1JEe$sS$?51DrK?zqCMmLL=#j?bWKMw`iou-i*iAJK|n#Fbm^OdP&m z18-qlX(<{UgExS*N7nk&3%x+kS&p<1K!h%Shz_c^_6rAxlUt!mB8NwH$v+oAMUDE7 z!=oseZI=U;`3oj?dQLO|fAVK}g{1_&K>LD5FE}f^!0YHWCc)tPV~} z3G+<{9w#po)~Cf2g@NIPLhU0*%aD&46vMDxk8O-04tw@`IktiQYh>S9owFA$(e>vV zak?1Tqn!Yw7+c8?{)r@Zr8(cjIcf9p84^AhK8XdpD- z3@Ep*AZDn00+~MnhHGc^titUs!6l?wBkJ=ue(L^Wdb{7;-!9-B-O7gq#AiXOlSEjo zu-_1bo%#{)>gXSYlUXDO;WG8;NB~||Yk`!2<)F9@i$*fNcMZuirHBlU9N`uPh3oB| z?NGBXBlhGHJjS70d$Y{pL=bL8ym@T-37m; zw5j+=sHt(B(xS3T$wl|7=u{x9wAFVujSszcXDJg|6XHxXaH0-Kg0tX67Olv&vXEKK zsPbxuFKXMMRS!P^1?FU#dr?8Sns*~;st?L+cg&W==Tm#&lrx&J$v}^DJ$N&VoA-L_ z6E=@5sGWV?{$%^^u-{i5n7oU-=kk-wE{!BMX#XDJ=Ow^gF+%~|HPB$W2Px#&LfFc zNh(yyRPZ;%_#Ek)&g-_*^sdLT6QWX;T1?$XCxa?Xy>!Oo-5+a*Ehc#-?dcO8-k~k3 z4Gvi-I9_1*%>?FIxZjW3v_WnZCoVlF)Epd+L zswqd_Ln97v5#a_a5eXt!Ejw`Bh4G~%Fi{FOnCBQ#>N>=GsYh=D>BK?Ot2ZK3=mLv3 zkga7eymC=3YW{MWiXccT9w$Jb%6w@Oz6vhWugY^q@E1vAJ}uF&@9FvjrQD!`1^vh)|5K!gjGf zH*tUQe)hpO$TVr3P$0ZLpK8kEhHcS)L4bU)&~;%3Yi=u6UbXsNpo*_nJVVX_J7(u= zEr?%&1-gtF!!6W)a>`;)BuEi=`Kw3Sa+C{ibgy&~1eFrR(VhL!HWD6D=vRIY-0nek z+soAFkySshRS{kRA-8<2f9{w{t~`s=x|b_+Y?QQqwCiX56N?qMt}sF8qjRi0F}+=@ zC0XpWM)_DNAnc-eM6w<5pWM<9FW1)zC=r^T<+v8;DQOb_#^4tX zO>rhkWBnG6LcD)0t)+;%jWwO01rhdX>ujFNypxqb%zQ%z?8z#a!EP{Wh>kuHCBbgl zLTC4(FmcsC5|jggapV%=;5yAc-=F8eZwtEU3KDta62CvdbgKxFf)(z? zF_a=|#UiniN#RZkN~t;P7inrEJAEJy?U7pKiI*rJ$UE#9<|xYOYo%W!H{QX_mfYf+ zVk4?6H%MgNeNAuf34BLoHmFHtws#bpmJG1C0VbEaUW#Si8@%{k&v^HlYR2v0lH*AN zUUa-F#kfKD^|4`Nsd6&>?GPnbmO~K$tl*Bh^9tPl6GQnqhlM*XVJ|38rsTAQ*unx& zNPwolvRk^dO?t6;^y#rs+HL#*M?x2TYg^;UbYNrw{7q+6RPj2P>DWLN7uDnSQ`87OF&TbER{ol%tLJH=NRBFv9hbJxa>$TNf^3K+P4R`P zC=~vSCMoRPK50dd$Fk9@)X=r`rj18J)B)sAY zVEmmnqWaFUcpKSrmH(N*9o8TQvXx98*GGf5$tgk#Ec)PL$~Dp^iuM!ZqYlnVf{sHr zhdsZ;S(fC0Ytiy??%`DyleR|1K~^-{Ektbsg<*3WO@4#2J=oNTaEwL8GrGZNA3MeS z$u+ogJ}7!fwopQ}8e1MJlvuq~ozIanv$W@rpRaue$t%L((G$k zL18xI5a_Ui!ji^~uVGVMQn}%s3cN)uQbtbjlE-6XgCp2HpUuLwUUE-WMjS1E$$xt! zbOVGyz=?vyTLab{q)^4qKW+^@u#tM z?}K5dkR!)&|HLHv3@o>t+#bF|OnSc0c>I?6gVY-=;|(t!3(#5OL*FbVEp>4&7M$(W z_)53=Zb}M%8u~aTI61^8D+~jc{V&@FWvz=&{E+WglO<8kn~vh{-b()my17H4BnGi& zUI)N>LbR8Q*UMy<_igwhGOhv~HL|)`YN3vf??PJZ4E$=X>y6n2$Al(nDD-L-cVCA{ z#|%9X@HDPezDW#(OA#o_*p1 z)Xx8kOdt>&FFU+!!cH*Trqkpmep9?-qzs*hn9LT@Y+S|AHyFA2P3A>eI<{8``Cup- zO7^dMc{ZsTD{@;Yb5sVQouHoVK*HH3&YU&cm@UmiWnmWWnL*;wNTA}^3zQk^Jntb z4@yGO;U)F(EYAr|H_l=pVAir1QPbnQRN8EQqml4UHa54+2Qzg8FG^`cAMs>-HyE#6 z3T_ZK$>uTaV8&uEtSP{g20D4e7^rmwNDlW^Xz%`dJT( zwTq~RiL$q<(_IzoxZe+@n*_z$(6HXsJY_a~A!RJ!qu1|6=j=Ftdd^||+_43j(K6o= zgEem1wbA`?%01!%y{gp!Jw5{Yx3ar%pmLv=cS!tuFwByIcImtj@TTuIBBtSSl@Ty%U0_oR2TFCbnyh zD(*l#<)>-jHy#id_nm*T9;QMU9_K0DU8+&{v{&3ues59yI;b-Fe9D=`%eoe-Em@8K zP|A>*JN1gDL@q+SL$N^k>gXgbm<_dQktD|<5go3j_#me1%av+}YRs5pC{eGdUYkq0 zT|Pk^TkssSCIVg_v&4Vfuvnc-MP}_E?gM5M69`*kz`$a&tHo2G%zF;?`|8JvNs))>GNUv%GDw*|VpGNw#7`O0-ZpED_ z^j)^6g`>E9h}6wr63ay-7XoTK1IY}_YL5bp32d{;RTo~pmpo>1o?RAYsbr5@R2R33 z-QQ$$P>Quq?L9}dFL2n`SXB5IBiCK50HT$$xsRk^YDGFLLjTgq$wreHT1vKj8!i8) z0ToBr&)0g(WT>As*iC-{Te+ML#hqMt(aR2@MxOxnwA(v0v{2CCF2_=ijy0Yq5SWtg zrP@j|bfr-L=XsjP_HMuwts&Af6H4|Y+XM?4Qxy0$+}`CySma`z)WQOhh_ z=VNc#&qyJ5Bs{qsHkAg~Y_$|Z`l@rIRhP32ejVO|U!ziQZhsBAy%7vRL-C!xwBha7 zkK9v{aoeWdF0X2g?czdMIz5pChkte{8H(Pj*1kA~cEs14v#nczOycR$P>5kQ*8nYg zxz(a89`+pT+s3E6CfNIAIoR%8h%?7;x7e}^m%A+tTc;35SXp~Z7TKP4xcu%O+ni^P zGe~&1FRx&Tyk~+;dSp}DI@H3E|BA8qaV+sb#=h!2bolScPkVonc%VP2qQEdwrz|Lz zqPNC7wh7n}ae)7cLClAKoSl1y6VxL2CqGUF#MA*B^*@=D5IP?~TcqEm(3D8fIM@X=$CgX&6BA1szM^g)%gafPYOt2%;|_t?Zq!mfx-BU1Ye;7r%9oEHPFi^}NJ zbB08FvsMPdv}}Omq|>K6!I2TmTgep(Rmu3vaQ!0l`)HhTG;Yq>Wg&5oeQ^--{zje{ zB%)&dou(4D3ykx!C};l0NZwy@_14ZCT}1*KtL9~mYWtoShVCbiZqjufZ1Rlp`?@}1*2bF zn3z6>B?!h9ZXs-irN3!du{$=If-Y->L5oa9W9RH~>%FJTJ zh%>>vwD0&x5RT#inY@zeWYpri>#J!GWNvc#je)mnTi zE74%`Tl+8(3PW4Ci!P9tYbTTtA$D+^AA0jk6%nGhOB&di+kF`)cpX)hsq_4 zs!R^dyOg4@Y#bqB7t))!uJNH!Tae4;96SLKwFp_qEs^2cS z5!w+WzJtJ>JX_24Xigd_VR%h~b!(=H-fdiYALjN%?7o?b)RGp#9k9vaS!Q*bn_}Fi zvaK?i28+ilaztA=LB3;=hFL_$oCFl?<*!6Vk&;x_M?Mg%NA&|C!3TWpQXL;v3HXM-VW;@hn?_# zrG|+ToaZV8swcNx)1a8>yKH+4QoX5)YMumyHf31ayPgBb$)4y{?)EBH6^)(?s(@R& zvDuFCG0B#nqT#6xIba#1i?WC_eUtX{84$j#eQj;NDZjt!NikIq@v4GLaRl)}DR+0- z-t47-nT>R*vvE)kqwpMUV@V3>##NzRMX7%x#?4S=&HZ$iqXec7T@$ zuJXG=-hu8h6D`e^8&O0XJ%RI$dV|DzVL;^wl_mbswJ-{mzNyC!a4U&8g{g`6g#LPy z3wB40L6~_OHZYS-E6c-q^A`FE?&>1`;z&=h)hAs1K2@Eo+5vXPlC0G8d&eXkWS~c` zf7}QFM{fZ7vlvB+(t2A>E&zb$8SV&jUQGF{V8qIqEd(HH2p%x6!}TtbjQ~B78ctr= z)zP{XU(<1EZgHWOCEF-B5cN5In9Ya4(cuWuCI0Ij8wLn@T&tH_GQo;h>kFIB-P@he z?5?`+Ib8j2xJWkJb05RxIcAV}(5e}Q_dM~hm#Dcwbb0*bdjIXaR};d9SStuUuW zQ^?-d|6BuQXuB80a(pSwZA%V$ht)m&P6j=kjIHA1-XuMhg$5jn?do2+ z#{QrmMFykr*ZW-T)L=@c<;-?LP=g%?Jps=8o4+FCsYiZu zEALi&Bgj*IgPpj~ZlUb_STk#wSQj~j$%yMb%#LkpUt=d7n$R$ur-o&M5$P(n4u%s8 zU6Sdk82cXJipo>jTPbVjA-_RP)U8*>`?5V8j9YxFW#Gcq^V6Fjvue#ITB3)^vZ_6L z<-oZ!X@uhOyAl!@=VNX2YDN}a>NOBpOHM+CK$5h!-5rOru-$V$9r)K}&d!Rjqu|Fk zD%)0>L_wJYae>*9Nkb`Am%hK8y*V6S!62kN42m{Ti9AQL`zRc56=^DHWSg>vCI%}S*Wy>4fGyR#uy641i%4wrrQnLnV1(MgR^hNMoZYSPJ zDg)gYAbS+OXM_=Z|I2`%*rzpo#ZS)UG@G8ka2!?Tj(|dwco38>=0f5FZr74eJMstF zjuHQP<^&3nWrP{z^YGzB%@y*CN~U(@rO*@b%d^b8i8Bk}vKrYlBUGX)%-Qc=NS%uhCggk~W^;&1p z{?XHpy%f#~gQ2ADvUBOeHdrm%ERO94@EGJl&%d)EqZ1A)`Zk*vjF4ZfXG=NVa62d_4Vm0j@oJdh7~@oNF6$zBtq7<7+~g z$fTcu7#&8sFgD1S%6|+5l-&Fj;pos19`#>+Lw5J*y3LBy3vbGNrS{(Y(BTGF=Dc}8diWxK>JeBaRL$qW-V<2B6ZkAYS6T4eEI9HJb?;FbDNInjDmjlm+Py-g5%xsOasno_TG}V4d5I|nQbj+Lp`(L9*H2}r?7p_!elx9B zt6cZ}YPiCsS&A-au{foCH1edptTHi+CL^yZ`%IFSAYKVGpm-cz9pXi3^wt&`#i~dv znOd}S?75aIMtKo7Q5P<;7S@R)YZ19yZrzQlUDDCErNA(}{0=LT)3{`vAyYwtUS&mE z**E{hY1V!_DQ*v^%M8k4vMqVASHV|EdBRy`q*te!&>^<=_gwuZzm&ci)JqQ49=xzt zti*)Wc-!_3c0!WufMo!~sK||<2fPc1!XGIR;jcmR(!69Tqm;fN&xP$BOK+-~P3~%@ zLIzt|SL10+(6Y~hJ2y_C4hj6K_YiSJTgD$I$YP0jixwK27j@Gg1Ba+8vL=CTf?!A* zz+o<1=0QIXQGX!4GlQgWt9LZfvKwt^-@+iYoMzn zfIha1C0AU3VDNmnXF|z+7c|V51B8P2a0be&H*LDOs|Fr>Fd510yAzn&fnLl_pRMyQ z;zpy5WWzAN?@&7rXynMzO66_kQ!NGPSQko`o`06n(y6}TvSa$5c|J@S7+wpv{pbW} z1%?8XtsST8+9$%K%YEV<2rk2d*nVsaJH-#@46JS&xN|bbl~V`v`G=^J@PJH;3-8;J~bj*cBq_9@b(!%ld&$gN9|Q z@~?8`@jySQcYn|B%8yFt&As9H;|&mJ^#dyPmFWY^b4v|!wd;9_VXAr_hRmq+?jY8l zV9;@f)re98(bod83ox)$6<%eKa+AS7I~hTxP-%>K1n!5}#7brr_q*G?bmLwGou=Im zTl*CvG&IG~9Il=my;&VYE8gY%aFr7;GRRVN-yLJ5ol`tz9fOXdrYt4V?bBieb6tB( z^h8TY)i*wCdP5DI2qkh?4-Y=0r#74XYl8YA)Pkw>rF3#Iq%^qdeg3uF7C9Z2w1|2! zhH|#rqu~CbzAr99Tw*nBvfFF-*V$wLXwAW1oz&7>YkHWmn?pBw(7P5?vwz%Cm$8Rq z$SV#2D?T!q0o(;5F%WIi6pB*?W@wX9oNg&7YV9K8qo@jwn%NZu-PuVxJf0dcP@7*^ zW650~0XVdIS<}r}mbbmR$FH)N{bUNYSV%3DNZBCw7jV0v@GQ*m(Aq#z# zplTb{+6l~Grzt~{W#XHd0n>9o?vBoAjFA^;K2{?#1A3%g=1udia2^2h-pRoZ4)m)) zhEQLphfuRK^O*h2Gr04gCw(pxjpbnA2Q<1#6+Uzkh?l-?srv@aa`A2k76_0DnEW$i zI4jA)NITaV%M?CEk*&PJH1?_;kVaDJ_5nJB zhHg!X8iT5va+`0|#X`J>e_8nX?kXQ?)w^@{n=r!tTuuY$u8XUc8EtRlaXGNCRVJ>I z)tGI?KKP)%DUst!WZnXs-cU%3ERB99pkuMsb<+@@hj698kB zL_D6YPp=`H12{B7!)cCo-wo-T2_s$h(m4j@E-%eiZJWp;9(AyZ5?mzpt5H^B?E_64 zXnYj^A{{v5J^0qbwsq;IVS(#1S(%FEz$;-~j%lkHc6i}%Fxb3%G^t&=SEbzn`%GK< zyHE@0I%l3OwVCr`{@~M3BgOoN>ASsekBlg3P}l?&`;DI5TK?+N86NE@H?o{Q+J>|X zl9>DF8n+a0h7jOwF-!dchYjRtXppL$o_D~IVaV1n$`;I$Xt+PXkzT+)K<$B_mI&nq z@VKBDB{?EIM|NTnRg;+1S%e>v2An3li>06HDl&yUO`W@eo5r_kTaa2;^BZDWUv?U!++-aM0CsBReLgem*X{b5?5~jTjT`PZt0An$`v~4 zl}PCWfaRGOhjL!41>={wOO^G}XBoId9%G#tL*HRU3^Fja^}Z}{KF61fKg-oFC+A2q zNI!@%u-7PzDreKHu8OSPSu};4?YsMpeHlL~tdob7~hmauU IDl>TT_y5G>pb{(3pivisF zGgwrmP=gwatqs)6t+mw7idocRgB-3VEcuBmYYi|NY|0luUXBqxK5tD?bJNm9e}T$T z>9DJj+i@z`W-{uU;DJd9N>98><$btAh9zonOQ47G`hJPoyA~W_(I@OQF*db{=JNOn zSVC?P>o zuAyXoj8!?>0)(<%4v?u)n2?CUEA^_@Nv4u=`WQ3N4IykxJ_*;D9~|W4}Ru!r*uQXC(ZU!Z4 zguDSdt*rDOADc-3$ih}S9R8n3|4h8azaO{I1nj?e!+o3Rtx}hJF$;EJ+pF_zW1+Gn zQm+7I_+Us8dgkl7ZI}mBeQXwPN%Q9Yx4u6yB*G2(ZU|%fQ%zkWP>1 z+JO^7eB{H#k|Yd!yuZ2mlcPoHAq)tvwGoXB3=!7JhU~)+F=XIF0GigzlUMh@Ss?vr zpHMlIq3o*D^AW_Oem;iRBh;IHyLP|ZZ(5-s_7h4o$0Gu1rNK+AM;4ia5iv&XpIz@u zntmX4+g%d~gGIQ46X@HgO=)J#&JRa|QCp_~Q2vNJONz12m?@*oXb8u(%z}=52IUX9 zJ@i6wBMBJB2l+NS=m!lZ=KzUO<9`y^HDNs1y(0w*1=*^x2LVF)6#$})p2Qay8&AX! zC{skW5;F++K>?%=qYH%7qYvl?fgn>%+)on?Hi|B1|Sinh6Dn?+6lRXl2f< zdy^=Qj-ABn&CokPCT-UIF&adjj1k6lsZ6_I8% z|caMJgJVE|v$x zOfb+*QT6-t{<@F=H0DLIS_{yF(*S@pKrROMJ+>*vxx$#N*kgS(=}o4|#6W-r3EKDP ztL1kFbT-CxHC=>{6x8$(XjEsz#*5AwqEX)M$ZH7Xs%0C&D*oa4{7Rtv@G9r?wvEOh*zwWvSv%xfjXzh7NmzvUmVS;>=}>PR6*F<9I8IyHRb zwo-6~#*5>X9;Dkj?Vti=Dcn=#YfAahUpLR}n_BJv`1(~gUK{)+I(Op{rtgV{u`CL7 z(+Cfxqs?Y#$}{Hf@)WVr@Uv^#BNYx-+DmOU_~s`(v$`N`HRdkkr8tflGAuU}`CSU) zhoHB(wGB#4JjgOrIzOVI^)=siONG&HyMr-#6lA}rUAy8qf4vvLY@aA_j%IdH;x4i{ z1f%hiV$(@m?Q-%zv|lW5O`TaJwUdJ)d#K=H5)w~wKdf*zZ8R^j?urVLhN^CT5-{4z zwgH~CoIMZqyOlrNTXV0mJ#%pRZm%lkHs~g1In*By(9>5chMM6VNiY9`^S*H-$xP-< zG|OAdZfZ;M^)OpYPP;+88aq;bCj5#LJvXF<9(FK%7jn`-)o5o#tCBVCZ>@GqM%)y> zrq!Q@i=rd74gZ~PJSKDa=c4Pqw;h@Uw1G@;<5aq za=c0svCz>}j9&#$*KTS0#0~w`^`4-Yoi}&BrOasgB*0Q2I9rb3fr0)6 zJIn#70^h#(s@j*_w-Ea@oX(fcMsJ(?V>N<#4A|*jI{K1^t=+YnTE|LV*TcQ+c1lR! zIbXc#bzL8F6b<6(gM3W9>*{@xioT;Yj%haJ=G2R z`=SQEOyoM%GfipHBY5lA;BQAjhr4mC;{MEB)F*s(;@g(x?a5jL*jM*WYZ#Dx z#`s3}B^vGZMs4a&T~kR~=Xct>;3yT;u%Dx%O%aj9)YzQtyvm~45i!HNL7-;VisyBb zO!ut{w%mC}6M9GgaPw!*Eq9~R^q97T)nuW(Ceg6?h@67mT7JZ&6*vHXkB$MzDzsQ=>ZOwS)p6BbG6bp*EBo>C(5bC`G~6e z&ZoGu!{T*yjlWEH<0IMAO-;*s3yQjwG7RPdo=_%Se|)_bgO%6A(6%L4E_d?~y}Lb_ z7q=vMKYx0Pjl~wXUSfCQ--pV-=1O-mSbBq;*Dubh?plQeKmOqtw_zN1otA?RD?)O$ z-%ej#YnR4*wHbEoR9$z)os_GaOelG;MqIn^va<45s?XAThd8SrVp4PCUWL+f zUx?*w0?&UE(Rzqbnu6sJECoL>tQc8NmH9< z+n07;stUf_NkYCVCp6C#9Q_I}ZOpaw+<0zgbUwvDGz5woW;AJ>w*2@NybXS8jD;SX zyi;tiJ8tcENxSoS9g%1apq$fu71jM4LRA;vcVS1^I)E@3HStzXusS1VCF*N=zgIu( z%pnyKOFM1~S*47W6vn2vaE|I&CAWbt>O>eUJx4V2{i|dc-(WY;Oi(Jm6N6&epL_pU zCEKlAHBtaqrVzSsuNCw~-wBi2kKIj{oTHx zuN7T|RWU_-Hij{3R>@d$%dPz9mub+K>(D=c3@-Q5I{~%2MpMm>EC^01O{vUQq zXau!SCNhGe#^44rmXk-?1 zXk>Q!*=vT~v84t6at-& zIzknIB@1%p!p+XH=@fchmM5j?d~GvgG>KD<_0E%6BJQBhhPv^_YO3PnDM+#LSEbY+ ziIJQ4ZmEULV@T0wwHIyXPkHmhoN!KxD7kK&7EhIqh?&8$y;dtSvbGd#Ldr7;Tan$! zMm8?qO7RO^a`lTXlZSjcQ)f|GeKqs!gE{)XL~RGwbc#pi`_jx<68dH)r+52As^MyX zrd(!DL=G7r#{I4dSG$E}HFn0)oar<;P)g?9B^~#c6-KF#X~|c5&-d~WO0^{yU9xJ5 zgQ~RRIen2yPMt?mb~$II#j1IY$gSR`j`t0sw0?W%sK3zb)dXziO0Wckb(28qe3L~? zcs0F}+R_{Cx9@v4Tf+Y$UjEYq`F{dRF*30;{#UO2N4(H8{IBYT@gHjHf6A984`byl z%>|YlT1eT$o0}V({%kPLHcdPD8(BYm|2+`b|D0A3UWePMqnWPW-r)d~)SC(==bW47 zi~wQvXn~=r84z-F)61#8fw3v*xWDqvjSMZu`X+JqtSw;aKx&xj>FbE(*9HtpG{+r3@_{{jxduamqmF3cv`9qU_L9ly}g z8CaZQDLnkz!U{S-0N(s@@r(0Iem-kC=U$?FL3#KUzGZ&7W;6F~T|pt85uL4#A<;^9 zSpd`kYWT&-=5CvQ2csXNG56q`+5S3;ab@-*0hG~XXJ5Ogr)_O*VbEy(`^WVk8jS1e z--A*cNC%*{bq)={9UUD1mA+oALj%i~6bJScu#Bd8{4pdXinrcX`t$CU}T zQnvO-->`0$fp=Hw%id`~S^hoW;(BJ6kIBC}>gu?9hA<7zU>jJQ89kVs*j%0LfKfjQ z-QPgdWj~}4AYtw7oIf-9cecrgz6)QO9c9~^#vUs=J0mu~o(3Z=w6ZzBXVbo^))uCh zAPx>LA6$e8dTk2{ZGCp8xt!t(<_(16Vn1M zCMUgIwkHO&Mf%VVc7PLMzmp#_L3d&%fKFiazd`GLyCpleYJo*j?RM z*H?fiBP+7}ws9PH*P#8A<7=o8b}mkU9-iMy_q!1O!O(hOjLtsTPp!rRZ&V#o)$KTb zr?&UliN2LTEI>=XG?deuDH-eQ8lC`CekTyp5iT>fzyjaDrn5g}sYQ{^%?0(aaUZ^# zAH6OktoW9(zb#*Q$TEAj4NDO&F{~pizlFdY5&>KOCKLFkN|U`@pRHfFTK^9>K*+x$ zwiebv00Rp)#rU-)m+4HotFo5a{<6_FK>SKf1TkG_|q?IynK@ng3D&-`ep% z#NW=~H*EluoV=#Cya?U@nVrAPBy3IX%s{pl09Fo8fU%>au?GV4+d!~#Z~(km-e%hj z=>FHl0GJqU?VR6S0QN4<-T-quM}*&}%E68SCq8*u@c#QuZ0m;p?Ze-ImhN#+mY z1Te|}L2rHve-Jl-N%0Tj0Wc~32XVi}RQ`il0ZgiY&|3zzKZqT`r2YrJ@znT(-ZE(Y z2l2dRH~xd(@|pezalBcY+F8Gy!oMZ#?7t;8Hh(-=nBPd5{R6T9n1FvGyjA+wfc(SY zjkft8gE!jdAlH9*{8rn!IR3-o&BWp#@Qs4yA3|@XS$f!80&V|c@g@WP1HM&g^$++~ zl=VO0Tg5hiAj?}de?oD*x!S%>+&}bh<=Oqo{^o4=U+OpZ_J7oG)!PFdL3aP>56c_< ze-y{^#?|RhzTXJw`j28b-prjq?*DLji}|OU9B*`-EggaX=-*ob&Te-9aCoc1ky8v0<WiPyZ_-WDr)EM#lZeHmkg{tZ=1@-&H20M z9Nz!sYx?)Y@UP4F?P~Zp{{18Z0DK*MXrUzlhGcYb}uiitGt5?Q33Hp&ZmH!KTd-`S4210Go*^fQ8b zCxiI_j1~*Ant44D@?H10sHT@InJ42~UAYgvsWX(e826PIFjztwH|G91`*xc+yl#MZ z6{a8QnAtb!Fq)6~NxjFK15-bODYk_(#If^SY=WB>Xbzcc{@~c9L_ZJ~q2cDkv(!A1 zlc9X>%iEU^)r*EUp59}DxZibTk1{3K#ug((p&6V^-EDw?g@x$MhW+l4=E zuFUkbFGQCg7{EewM?f^?B1K9=TrjCI*_%mF@8#YHfB%*3mv2J@77*IIiza-!^aii` zSI$dQ$y!&A6HI&+rD|967u#ebd_4Dz4H3iO;m<(+L-71iq+n}K5W+1-H#>a$lpDRfgS}9YDjnb8}8Yr-={g~_N5QEuIXn(CO^+*|g zyO2#8j4C(SLR-}y0zX}-Kp@s!75bCw$GQr(UAC)D>zPI#aY6dwM}=CL^RZ*ffh;QOTWPl90(GGiuv~<-!i4 z-=!Bg(a(y)E@|X@accyaCf}gR%H)vxg(Wntha3UTg&TiG%}-vkOk;G7J9~3H2d~M? zsiLf7SNzOTGxt;3gFdvRaEg>k9%h4H&`Z>0J#OrPsQ+TA4KcN!0oHhyB|=?5s#@7{cs*R4{*-MGx`RXx^Xw~>B?Ud+CfnbD`ZySXUkb17h5FMm;e82h;k7==!=^Q? zC#%#}7uTco2+V5EvD$5?(5RHvBta2#6tE-i+waIsr_@Z#F=)?JfAuKGlbN>`nY>2UaMBUvy?ZYw$)qdx01^pU}frlFWe2tu|RK zD+E`~vH1a?rp}nMS{2UFDZrR|o9YwJlK(`QO7iM98xh&*zESfL37Ff^3{z7iz8+{L zO=PWlvsI@5ohlVG)~a+Nc15`mKLMgeXwSDZ6M1!Tv9E}f7!xd>;-(sL$SKR-tQB-Fq*ur=eeFaMp;9uwywn zqUGsCd(*4U>)Qr~Qy2VIoHA{KUQbP&h_u|wzV7kj+>ejVO~w&Thz)q3wk)^( z#*9&HvTz+M8h}Yug1G#C8&cG$XeT_YUAHu!zmgp%&%ai_q=pU>yw+!Vpk44dS?VwP zLR!rsE3&@#WghSnd~NaEJGj(M;p{)NfL&tR9FDQH#>H*Um4s+gcYgH=IP4uWCLYY1 z!~smZ94Bzh51Uw=V{VR*(?Gw!pAN#*Xutc?Y7^($0E+fz z3rNbuDTZ8kx%oQlN>4*$Acl6r9vLFI#&YPnj*SZsR+LpJ1sbG zg>@JIOnMnVyDbgT&EgI(HrZ!!G?;^L>{{k}HIuFq$$bj;Q_l@dfV4aJfG%`}o?Zp{ zwda|;eDo-e;@x$?=S07`_sOveFJsmqdUmh} zESVLILHlE}%S?HSE!jTr!fxPE+pn85$!^WK7d~7ezBL>(V13*?$vYqPJyOkO|)pjXx1mhv1^-!Ox>6A%H1COD1D* zh0OW5rmo}{wgK~+wBbw8sW3oaGvC|Ptz9IVvB|4H7G z`6-4|S2|Puhr7aKxD{3GSmF8AU~nMkEfrbxXoWZWv~7|j%P(cSUFv-YR7143D_FjL zX530=SHj6yojQR8US&KN$`_cX$%)>^h(q>Um&C=!!u;GG5H?5#HX_ZVP0kW=eed3XL4?>nE z(=W^QbeB_I_nu_YT|6uTL-u>k^zy85e|Uzrz7K-F&>^?m;gmSJbF+e@)Tk;gWMI7? zKiU=9BimVH;)qlh#lbG|G<`~r|5)Uj?FPZDckX-wJG|td@EXtgu~AOgX28upk0N^M zaU_6t6Y2rjHWi!L8wAHC&+!6M2^0qQ#sHU-#!!^ zbB5cm0?hT|RT_JYG#XCS_jeeD3{^0}i7VKQTK60&M} zF&@S*q@zf~coko-Wq3F`bmz$E5S^50{K}G+8QGYZ;17~jSmMg3IlF5A25>WR+!hDijdOrao zE*ULU!@2Z|ZTpz@JE(5=cgH{46Ih`ZcekQcDP{DVkObLnUG+Zfa5fx<%9qN5BuuRp zih1U`)NUpXN#bz>bdO*H;}1{JaA&`;ZzCg<-()FSm6x-8NXKVgWHBpIWAxQrKITW# zByT!)F23s#(e9yQ&-{{R79<{cvC#+a;>W3xw#rP>ZMPagHVO*CK9^C65FA!JVMQpj zKAA9kv_NikC&)v_NsJ=u;<~)qM2UAVgK>Ic!pqJGy{hDl@7KGPVF=c|PE#otOHes%JP$q%RVU3UASPoQgtk7XMVk-OaOenl5!0a( zY}BzsA}F0vV7`p?{t~oEOw?&h1wW@P-OoLHNk&EcflS&P!BFcR&GlRKJr=Y=zZ@)V)=l8eXMTCCoa;+6wAz6dfINQ5LXe1PbF&;Cz zY{ZHDaW*oXNO9}g5A{|4~ zRW0zFT+^=+Jigf(RmcqCcSxc87^FWJ1!k6uC(EZO!7Wh2Jthf6e+}f@a-iDm3C9g1 zA^n6mR@-*CT^SQLm`vo)#g{Az<|Zlc-3CnZ7y?~==4NWs7?sD90C-rYfW=pMCb zly*vEfkobS#&*5GtGYsFMKVhsOBR(DR~k}S`}V82A>h$w_&jlFaP)w!Sf`b{7jwkL z`gzek=f1(KG?Oj=XkS@2SDnF&Y}(qYB#^t5A$3iidlm2#$u%IzP*BvoewvDf@!1Xj z-E%S0GW`m$)YO#1;q(2GRHF4OY}X3vZNbnWQbLX5#wTKY4*CAN!mtl{8!z8=!G49u zR49{*@b;t^)1Bq%)_2T8+^y%+2^PobEJ^E_$+C~4+Sis9tqg89A6D7(ye0&=l?TO; zlZWD0+Zv{s-oFb9=R@a-_TnaJ-Vk&nfz2tp8hEE9Ds@3zd1>g)74m|2|7oM+<7P}` z$iDi^nip57?R33lokdl-4dlvFHZ~yzWwyscu<+A|hxfCcwVH#SgbYO9(P<+E{do{i zd$|sFGuX0Sndyi8EkvvWD0U~P1f+ue5t9ytlJA`jcC$MlZd?Y3kh&=EfC0wdo~APb zq^fKp`L<)W?y=@t|3J(o-|%&S=*}aMZ*2!$gd^;$$&|M%jX0CNli90 zANYCHjO@UhkuGog(6P}R_~}+1ZRMww`MBAB6Y_4@ZrfM#@=cN#pa3K;b?x!3Y(QU7 z4nRNg;w!Y7IXIuWr;yaSN!DoP%p~mBGhedFebafPN^*1vGn+jf*!Jg(k)n3FvWA&* z5m_v>Fx%HCb$4o-Z13mJnnLFI_9Uool=^V93Qx+PMIIC08SGEkmcH(q^b_!?C(ad` zY_ZqEvqzxi}Gaqbg>mmIoeQnwpTw%C7!~ao6X<78?T&IRwh2uL6rtjqO&+ z;f>T*Kbtl5x4eW4FSmsuNpSCAe^q=zb!N5QV{@N}SJZrW3$A54bK&IWRyZ&fc>r6U zD4U&lh14^17QGNnq>h~oDKL)6!h*UcAIG{55#mzG{{1U;$Oq6>u6z(V8_Hh)sij?R zGS!{^G7MVO%1Ba2;V;;DbLw$cj!HyHh0ox7t`-av$O|H8;j$Yk&a=Trn zQQ~O5Q4#n%UXn+7^q2ncsIacN#f6K;5cc8L(lV$JxHD)#)I3THgn{sNiW@FU9>#TI z>GzPxTR&p6Kb0s0KOLzUX9aQTy1A=n+eJ#|2UnWlnrrqk(!yo~3-c(;?;4w&Mzg8SZO@ije>k&-Q)R*Ls;))augVpxy=-&Fx;dvA`Tz z^Mvq<@G0|a*SZ!2mr^eaS7`@BAm_v4BOZGFagjQ@HpW0tHd#-Z@m?e|SzzGFoTCIh z);4-yVM%)|7gLqQd$_BifpS(h4!RVLl=6vCJygfNm8B-js< z*q5>^_;44a#KY7<$kL5=S%PKga}6A42jl|KevL+`e(mj%9K-Sy>g&5(qK@Ay_7UD5Asswf;|RvoISBB=YO>vG~yJg z_Hpn}H2@u`p?jM*7}{5Jqgd5d7Ccp8L zZPCXXBRlFy<$cWnJ$jy$7Q;oc>&Ot=&8np4KnQ*w0{luim!Blbc zbRxNG>hA~KK_=q?2lXxwBr`iARZSsrdSIRz;ihZPVARO)sw7$|Tp-6veF{p<)nCbq z0;dl~((A)}`0Q)4 zDSaV9bL-5nHU)p{vYY5e(pUaHV;h{xai&*<#8_b7z; ziF_*}(|x3+=oE_vx2tfWcK*CLpbNVFSDC-YR#amDC{T4gE)5mywG2+ z*`mIAV*^S(i2{VN`U~mqL8;Ud=Uedt5kC;S*Fhh>00oo^R%Ton&T!@U6)f=AC!E$7 z4n0m10{M=)+%<5uSA2mg53#6!JqBpx^D-!0-Z>JY_EUoMA-t~j}L;z=2_dfn6OJ3WUK zjDw*s}A!p!ToMifd?y_2C8G@>2F|;tmd&!>C+|#!hL8B)bp@qv(6S>b;wnf&W z2%7Nc$uLF})cCW- zNf{m4&VB4~{D+QF`7t3sqk6Ca#wyL8wy15J_v3I2!R!Jy=W3ct{fPXNQ&2G*{DI%c zRVjA4C~Y3M9KXOp@}e~OZW}8dtp{sDR}81;L!!_M?|pZMpbyzq6iVWD>2{m$?WgoQ z-&Ea&#d(f~j})$+ zw6Ar4yyQKGe5@lREH=L0CDfEzWWI_ys-~Iwv;#iC!Xx*YG3H}nx4-#0{EZcKq;3Lf2rj z(rDEfJW7jT%$(21D!4|9IhEE=yiC`{soATDNCZBDdoO)iB6w5>W^HITn&s18|_iS;kvP&V{I2DazTaxRSp5TYkQI1Lh(Lv72&^~7EK%NN$FSDCljX-90s`mkq{zi;?^@b%v2VIKQNK8!E z(eDhU{P1GputsjPM~*`^&j za1z-B!>6l5z<YJ#t_TGC734&1|aOYb5;(;*%`Q|X!oLMoPPQw9o{b+tvu*ek(gz=o7 zn8#Mw8o#?an@WCij-(+c^H_lefBvhHZ;wx2dUm^7^B=f{JeLXPISJokNVDViDD#La$&&oZw~a1q2rh?S)G z;VdaE#qP*KsnSOLlLmL zBteWALQ>-qD)!ao2`;wY^#g+8w^du;U0B}!daUr7VKj|gWXWA2Yk_X~81Cc1v$J(+_SsNiI? zuV-4wsf?WIdof>9q-Gs4%|7EbR$r!@Q*|q#pWndj^-&PU8B4Al{fNv=}TqW%qTojnSLB{BqTb(=D!Q1Pl z(h&@h?=k2E7K)G0BCrdR#-pfwGTX~r#8|8M_)g?Ohw#+}9wA|EM9a*9(&`7-bJc;u z+E}hpbU(km)FTxH7vb(q;iX65EYhIC8EuzJXnjImBX1Bi(#-r3w1rz#hewPYc5S-G zSXv>D(`iP05^~RD+5&pb{y}7ka-xk%5B9XX!WR+u?_8+qkq@4N(w~?rOn;84F0$%H zw4R=-8b_CyOdDHuKJ$Gt6g;?~f|YmYI(OeVOh!(Uou6QLF8sEdXYOW*FYygiDsiju zLI4&ECX(LhYd4#G7hiC(4HRT86iVF-;<5=6RzaDK+nK(L)d2Hk%B-!GKFq6!b*ZD; zoNq56H0Is{KTpvIcT>5cDc3J^6nBfv$wTx@#KE@h#t({;%MCM_a@hzvDN4WE-o&iv z%stTKg5`nds5Zb5XQ)kcCO6F)T>NsKl+XiT(N#3zNt+x>r+TDGQcz9rHm(^gwkPvw zs?*81_?S_TD)ym@5Qt6jX_8FlR7uVLuv5q%9g5zDwqyroT3n7dM_?nzc;Z4vdausK>eg?cEYQn(da)v(TI#bW+R&nCZJnzj^Q%%X?y~>cfeLf?nzZ znOrdTSDmckpogem2K`fp82#%TM?wwgS4s%$nu|U+25c*vytsw*ubRx6?vBO)_w}@u z2_jPUA_m|hF(p^a>zs`T^M+aE_>B0IUR~ky9$~!1xU2xGHPuk(&`2Q7@_EPRyYKuE z6uZ4uO0Dn~OWvMOJHgxSCzE=fo&5y5087j8o>%{9w323o8vobm_VMx^pU6IjL6 zC&fsDX4n-vman)H-T6dd(3JXRQw)l-Nq9THFCMvK9+el@(#63-h}#Xvs;sR4{+wWX z1AWIS#G+Jh7M1LLZM=S-AIxpd{2uhUv03c(fuwz5C9c2XBRvk8NQ1za< zQr2eyEM6NjaT2d9aEnrp)nr#){wq#)p+#p%!d=jI%~)hMvo0#S$Kw{INvBp2pWGT# z$k%iu$~gRCS@qM*M`L*iR|f{_RTpiaXvIJY6Z+~&H(a3NG*XXIz@5ysOkm@ga_Ai% z=5=X(;<4Cf(YcL8Et3T`FSm8^a1=0~lZm*xLt@NQ(U^!!ce6QaMK)W$j z@eul8sI{J%P(O!lpv6l^6tq_XzR>t|why86i5C9T@}SI(P{T;+sOoFxfbZs+#*t^Y zDrp3<0z7LlT%JM~CnwvsD2Txe=w9jr9)IX8HEW^B-RfK4CyT@ANn%D&qYt~+Lb0y$ z(n3LojA>y)3n`+36uD(9omjR$XyvN*C5;HS5^WY@G1~h5OQu!O)JqaAcF6mm1NNbl zAXT_8(Y2nhWNIUR`f~sTRZxf0Uerwf=sk4ng5qnIuN59YfqD^ixfp4kUi1@@q+nl% zkgOe(e^21;i9izJC~2s$6&|y+z{@Dn<=4Dw-%QE8^*&al8f&T)mBwpsm;&u6uI-oIMGHb_%EY}lH@BT37~)C4PyMz6lwxb7CV@ZZYW)Z zBpTAP5_yW3>#+D)O~d;d2f6S|r%fTyN zJhx;z5j-^!f+@H{{qQi0Y;Q(4Wu!}Zuh2Tkj?*G#h*&hUa;IiegMnoXmu`ECFRdXx z)s2v&6BWh}441Y&3u=SJ+iJR@h6;{?jsDnU1tkzZ+yP}YX|J)%z%fKK%$lT04z2Ra*RpBfu%qLsg(PjS$1`4b0Ise@?wZoW$ zpImh2n(C(WveH5I7V!>`Xh};$F+(=9x&DI9ce+l4Q=UmB{`~m9wuhNH+DiQLtb%1b zRbHca-?h|IcMsQ*yZl0GNms;@>%y<(LXIF*gV1r4ram_}pq^x-R6cuW?~3gEGKtJ@ zEMGb`?l{a;q!dIrd*&H_w8)$H=!&NZi!mTiWA_L^#|H>Mn{#0yn!s2Nvpk$sJ(&5fdXa(23^(9;5_$nX@S??6Isl&X~{OrC-X;-(wWt53QCsGmg{ZnF=_(pgwZ z(Y)zie#u3~&tPUHxf8etT`>8}AmeDoosTBF>M zTpndI({(4l!=}IBu`p@H@ITK5(bc&T5sQxam1*OGm>A>>_P)7gfX-<)7LIn~YYVf+ z%9Lj&{I=a{T>xJ8Df2!YI_A=S*laT17n>Wg?+@6=ihHCS4`|OvZn`@qCeik!Gxlk<2gbqc3U1^CEspHjC8IUl?B;ll?~ep7xW2 z7&;Hwu8y}=2#=<&LI-Gh9wbRsdO_BnGas~`vGmrgF67vkf1P~No)gaAE3KP&!Z4eH zz>?3DX)p5O5@)Xab$294v7Fj5E*G^p=IGND*&AA*j|M(Rae;DN_cXV7T^Y%LmH7cD zO?5+&1017`6KUR&3=46CYL;SMiArsUrV(14ob-_-F1q@GWIFpk=rEZ*)qNU`%U^O_ z6xocOxv);tz&<{cZc#XLToD4FCE-h2)I*eHC~j~iVQZ(kk(MZmoC_gAon16uD3 z*1A3@`&4Se=(|bA94Y3T1FiK9O5=AkNw}ITm7RUOnLKWZBEOG-g)6)y^=6%G)XoWg zzuJg2B}*g2>XYJz3dO&Hj9V8ZrpW!cK+=+Otd?>m*rnb66NDRI={ooB6^ZLwe9Hl~ zqTyVs#B`EOCqLa5vb0*o2C?-Q=GSUy$=BM}q?*+5chRd3v^&mjuAX_*Y)|i6@`J6K z>tmD;&)baENn*Tsh}#qPmUf;ykv-LXiC9+ev10!HH5;(-?W9f=P|vcdU8~E zsAo2Z6lU$#G;p}-AiJHga+g>Cgmz$p6`|epWO41baP{OUEs)0^dyw>2S z`I0`O+=Oy)-(x%&7QpSX#EaY*p!jb~a2GN}SrWTd8*5G{Xhd0q1+8<+KW3Bpmqj_U zK3+rWp!Z~s!+Ts_n=_rFl0(AqYLD^>kqy#%CZ|}f7&R;ZM6b??ZfN6c-(VQlh>hlY zy2)=uBTkr^Eu7`)L=RnaZzcJFMN#J#rpoy3y6?NLEolSzdAyzRT`Zr-E`-Qf(xM;A z3NCkT#dm^YWOEOSnKm@7EIjS=N4Cej=mOPEt9k^rQ}(Onw$=F$U&}&qjBly)+Zat9 z$>WBU6zUjUR7jxebu!ac*eIH3jYhJs%CdXh66XquK5HgwNvT!)+F2 zeNHpsQCfg&X*4pB?)GDu+e>>aDJ6C_U1e8?d;3_%r5h6>T%_q_8ZNe$+|YA$LLu8( zZU>_jhO*pgO~Fy{8wleLj5ncEaqlKvd(QjV|Kes>BKswAq@;dhhv`_C5(k- zjFF0<*gO-LQ5~M796?e_5x&eD{NcjpqXLuE@3Y*#J0hk?xfz~?2f^Ha1{q(!~i%p+w>VM!veC24q3OoRcmSKHJV|#ek_>v$-cUk2=qvp zp`((`G-&0k2Z<{dTR@5{pCMbfv?2_v0qC?d?LQl-xe}z?3?UKpU>rk})r%4HobHhX55kqVFc#-=+Yc+1*w0tpe?OIdkzGMIdGD0)IWht*3uZZ z+CW{q@SdHB6}~uKdTeIKI;K%G2uy(<1LoS9nOiP35~S5biDMj7qe`dJ=W)bVjo6l( z=ggB`b6{BuAXlWwPxg^j-lfCA(*ZGVdUi0u#rc1ARLZ z@13FayYfKJ2<^+5S3bVE5jMLCR3Es+dBN>i2_xS-<4zfW)5PC?_e$2&FLM#B%?PmAo}1ZbVAu0U*~k@ z$YwLeAZQ>yALKcO*Me1*eqz58uinkDZP1q2!2?9w%(QxckHH+^Cs=2#wr9NJ7z zeLxqE-J1TA!HCJ8i$o7U8eBXxPXyT#5gtZ33%^+iK%(loGF}@dkW)NU?<<~E7)0Qy z+3sdT>%W@fu0PEklAVF3vgN>BhV8;J9TBsoJNKOk|8GO+DekMM>`6b#AiGBJDf=T3R9Hk=B7K_TQI_tWvF*PRy&e}M z=rU9OWb-XN#=W6*;~ffB8m|@_bq8n0X=N_Yf%e`6A165KmN*rDhh^1VvUpK(wtmZN zy=ei2u|UeR0xXT-1I4^E^c|hqrFV$dDqawPz>72d2;Y=I`)7IfEM&=X^m2v!pNX7c z2TTwHNI`l{*FVa$ybCAo%(};eCgml?E!*ceoZTe85wB1{?>lY8>ss<5fX8KNT3?HJ zV|jU^S3GR=Es17&Bb-y0LJ}UQ-A1f9F=#N8^O) z7mav%ut%nD@@qjT!clgPQWZ#dKJ!z6uY1jl{=lhD%JsRdh4RQ^t2t1IA4h=M<6vnY zc^)DQq}NDGF3aN?XE*thCO&xpqdF02i+Z!h<)`7FX3;-Z*y+`zlWPlmxn&1MoI;4> zR|FN1l;>7PC`{VimQyS(cyP3+0p+lL51l6X)dwk0g<{IdaWah|+)`V|s5H5p>u#L( zsMq!is-+Ne_-OXW>`$mGoG>i0A29OY-WPqFWLt4k5VO6g4u-96Mz}}l|2ATBd(4FX zxjpy$O_jH#;9dUmWns6k172_3HW3C7etUb)+PM;|yfvJqk_?KCOEzE78o0DSS06P{N|PU;G-uYp zw6!$8hh=efq31br1)Mq)k4iULiLSsqA? z;2DrSs44^Y?FHTkM0w++&Dx`;%iEJt$r1J=-`6=@h!b^cuGnvH=iV!!m;yZ91~c-r z3qm-<(z!CV0{zkYU`l;V`lbH?9S!1voJG1E!@0=?mClhAu81`eP;)*&#Kt_iG(=(( zb=7Z{1pdba>?V+7oTsmQBM2A*>%+adj(;=IMOLDRev52}XN0c9qvck$qcUy>nIRQN zj?@GF4P|V_0v*Ddk$j3Op7r<(BJ}BH?d7$B@bHIKPje}YgXh6Y_7m^-&yx)>u7*n7 ztd#yyJs*{|O{cSaGOGKW-oI0hb}v=dnSa0c?e>Mnnvwt82LSX2{*UVReP>FDs3KI+ z_jhw7(CiR-@e9dOdZhZj5ALR!N84N3{CwQoRWsdWlsfmicF`|2?dR_rYN{!*6kCs$ z9C0%3kt{PBXElY7doxjr`5Y^wB@$aBMC9Cov6YwuSKFCCPE=#EZ`+~XaULYzvF5M6 zQjK;duRW@8S49O|@TLwseeik>LwrT*R3kPJs_{#*z@G#PM%{Her7zx`%>RU*#9_Wz zypLYWb}2I3pp^vGb!0hvF)Uw=7L+8rm@^SZ9zKeyx-m0)hC&uWzaM-dFHAt_#tKE~ zStg%33g~jIAXsak(?0mZFyc_ka}$yVxwfcC0+T&+EX};T*`~OUt1GE&Y5hLXh#{H5 zS(!75pp8k|Sp$5$M9ge3Nn;yFgQztlMmC3g-9yXI^hpoH{WAtMKo0qn13jh%oE=Kw zV4i3fMM84ShuN~}nfq3~@cTop+8~;!zCcu#t-xHrqduzBvRa}+{f7;;tK|muo_5Y0 zrc_YR(IGz$-pn*!LPfi6(c~t{o~46bD^5NbXeHVD|B(4n)%13Bn#9S_?5X^>SHx4@Kh&<2IghRWS0YdQA*N@&;LjtC_C)Z$ryF(sx}0`NZ&}Lv@%m%jUbBo+P?(g2 zvidIRvS^Dz>mYI`x>+s756g5is0T9x9KAhkS5Hy(g8TLugZpaNvJVX}wu{3eF6kN9 zBbkd>{!0Awfq{5N6P2Xys4qHyCVm3C<(mGcL!XQVd?cvZatDEWimf#v>w9 zC?PVm$b?y10UO}PM@X(b6Vo4t&SE9DY2F@bOk9iO+iBv0D*M4cor(HQN@ zEQ*wuziin{?*{BFTkC9_TG)}qi%NNfPY(-AaYS?g=<7QzCwPC^l9@)dy&xGj*sH&- zT@13;X%Hd#_gY6=6_|G-bY7-AB&GC0%)hYtgYQ@8yL){MC@nAK-|$7swRc@e#*aqF>7(c9ZIb67uo;=5{i;Y@y~#NK9MmG-lV^>yT0( z!;P)bH0(H%?3%bAnkusiQd3_1P!m*+~IUXKY|(AW4i?+x}kwB@){0 z#3aMN&j{hF*%$gjmSWjjSDL<7W_p_#b;ufWcnw~HwJkiPVTag_aMY{f5Qkd$yE1tR zoBd^Wtn$tC2JY$SV-Bmy(AgkiCq6-=KSkeWgwx!;%2hY3$ilTY*4M`Q(=5Ls$)$)k zg$yaUFx8{v&6@ShNPbqS0C~!Np^YW;XdTcK97f?y#sL>Ek!*>`hcQj8Wxd{Q*5wn< zMIYoZ-m6U2;DB{v&|FJLRAZaLS34bKr0t)9t%DF%3R&#TD~^oolZ&_w=L~FWv|br8 z-tKy~z9&9KEaG|wUc3@)folN&TJZvqL1->4Rq0awD!}VHd_9tsly4f#h^9W%FOFQF z18Z%f$|T?*p#9vyNsV+_O5?1s>X~tMSMc8Q#lFC27>+pL?OU!{-BrgndPIG*NB&80 zjPCP(k)C?rKRL#U?Ay-BRVqivb=n&`O)-cf1NZKyN00USf}B9oyW$wy%<9B;IHCh6CwUbg!yMH(5jsxjiM$r_&E#7bk+J@EUd_n4Jzp z&*VtcGVi|BN-cThA4OiyNWYxESBZ@-WF4L2B0o&L-2%$geO=yuVLsmX*R=HNS2m5Y z^EPBOF=6d}bxgqnHUwNuepL)Byp2w^CauPd=7b|$q|K}vNzN$!F6Rm+v?bKdA z!P*}!KX27Uc&C!DRLzD`m&6g6znP;B6@oJ&FO({(Cd5`BC-)pSKpm0P)dLrpH|b1? zvR7378o1FP8xIyY@*A<1=uvmw@*@+8!LE~E{HRAU*K>YtXy!om^&y4WUue$N{eiZ} z_r?laX$e{rob7Vi37WA5CaW?L+C=W~z$Z-`iE}OMH0O7B{s5@EcU6AZG|Bh7C<@5K zIBwu0Pjb>Ef_%`7Vu%&ZNrH=6{BC4eu+rN0SOSx7g`R?y1+rj2M-wjT6VtBN-tNJ$74d}IRr6e^uce01eR zslnZqD*Y8F0Z=d){B-fEd3JmMW2l!;jF3=WQ50eQ^4pSmQ;$2XiQL?94z*@r^6eF- zj?-sn5=Jy>oh1ruab5C!OZT5Fu_CY$u#OeU$Tab%WQv2X4sPt6(+|2&rtgK^YDrqU z5jItDt6y<=_aYOMjc!po`ZrPPo13EIAoai&_);S*2OLiF`7)O{$bQ|(*RXC z>N|c9igUawh*l}RSD@4LfFM65B#8}}^NJlHpb+E488fgq6?iRf^RWY)en?62a6VW@ z2e{&SQO&YQjypb<;zWU;q3EpEksGP%kvNM^E$9Bzkz5?2ap2}vf>VmWz8%@>P;q4& z_|BFU`@I&8z^$YWi{g?hC0Yg+WF@Km=o_mHuk+z}Z^W(}E0Z2T(~9N5>YlNEr#Bpb z48wQLwYGyt31l7I!YAJfPHYDoW%h!-{2x)*a21+>@E%%Td`vks!PHIqG{W)lPDd#kbX;20#$T5D z_W*>KE7-er?Jme_;mAx@OO+S#WtudQvN*jsEA7I!C+%>}@TO?pm$UL2oov9S`IwZ`G?j+YIf-?cjcPXMOYZn>r)PM@1Wv%AbYweQif)of2o)=AEJXSzvZv`iDnr}FU9 zU%^Qa%kw+2JBgGY;9k^c($vt9N=PoT}<&-zJ;QcwPG!t4%_Hm z7)#=Cz8}xhnZ12EK_>4~CJc?JJ!(==7b`u{V0U{ilX&(Z5_jJ~uh9)jP$|Lj6TW>H z>SJ@elm(p~lqk z*Fz7W@^sIns z#mf6fB=hJY;Rg_M>WNvk@J-2GOxZRSBrrPBv|K+;_+#G)S0>+s>R-IBFLU*wSp;$n zsQz@jM<3S{n8{Q)Fyj>$blUoZ5lkQLJw&} z)Dt8%3Lzw(EAShu{W){!*Dt8>0hyHffzRFI(Gb3wwOgS~KXKm{4mu{K#KttTdy4(a zKE(l4DDEd=WxSJe(s%|kVGl09TEP|!EFDIUPMD^KnuCIyjwa8M%kkeq=DD$efqft3 zN0@RHzC;up8tR5bXGH~Vb@HwrB~)tTvzwgH*$3m+aySxjv8&z>TZ_IS$WNlXJ*j$d z4<*S)bty=2-a^A4M!Tt(2QHHoXnZ*u^V!(0nn> z51dLJ8Lh?e?U{X~g!vmz>IiVccPZk9;O^yOwKRz>?O-yolH0&U&BrtCRd3?-$KjE> z%}B4UN?KKbgx@`Ry}BzqjV-G3^01lNDUQ>1C=W&lOmNX@dm(!I$|odSm8@SMYemOJ zMBho2`HJTwm2bj7wPx1@|L}QrV~#7G&M?!Z?!_*fm<|pbf%a_IqQ5t%Oapx5=5nOW zUPsJVNW$Z;`wAD&ora+2#%@~Jxr8CM-rPGpvMT>qtu8uMXDt;rIeIWEH*gSW?Q@j3 z;GBPURT?CGA-%!ErZRsiAMq@4%P|(R-12zFq9z|}jv&-R!J=1o` zUVJ-C3R8)ps1OIHK3!THd*CRwxNlCXU}dNONi5`4WEigR=a;g4_|3s)AjQR`ft2wN z;WAwgZd6K!wN|u6?X^+XnyqEnyXhmF5~Y(kz}eWQqH3^v-UvcGE(CyW-aSjHZX$z6 z74Xsre<0u3*it6l(_HWA&IG}UdAVdHzb3K>60$w6F^aHZ@j1#Um*V{iW;8GhmFWR~ z2{-1ycToUtEJ;>*RZWoMYML3C7#7>Ic~re;z6=-Edg^&8QVhyq$rzt1Zf82UO%_5k zIP)2ys@WetP~^On@f7sNB^fVfL76bwr?5c&h^REa;B+0?r-*`|n09j%S^K_>2r4d? zw*Kw{KDnVr4JdqHQ=5t`9K=9CMCD2cE*^PdEX45kTzemxd>%X)b(~V`OZ*@@a=tYR z{^?kT>=qNOGl@O%U`uP{ko<+J{mQ!Km9*!u@M|zp#m#5y`&An$cs@dkY)Q&Jkv5)2T zv#WpG6t%SHsh*F_OpS;T{m3PQNH#Q^1@XsyAVpW@{q@R?e+4-h;5c=14-bC`T?Wq0 z?VO{z_7;S1%xkE^1UAj+_(lDwvYmtR6%mrIKsQknTJ)Fc+?bX<3%JvLlwt2_2Q#AB* zy(N>#m=xWLHeUG__pY@UX6WLMi0&96JN)9d3t`l3!HE8)s4VlxF4Yo)6ieCOf+(Aa z@QKmYnzT=#>oWCdw!2;pRes>HMzrOe+P=r}!j4S46BYqe2ul5LWm< z=t5Q`E3an&eM5BZn%?^C+_t)BA96mI)Q~aS`wCCnah7*y1@j^-A--~v}(`;gtB7m?G`lGgxuRQw0&fv{2Eq~$QvJh6HoQ| z5VoRi&4fgNJr$(z1WSkMr6h-xWjFSc#>s<6ZFi%7!hThN!};Z>Gwx z!~)@22#x4scC z%iMWD(#^bleLX&Vx)eKlz}&b_r27r=5B89qGO`Sdz51Fa;V^+a|8^JGUmtv+*SfO8 z-KZ%);Yi+oe>%U<%RagGm_@>&wqyLTDxCQBF^7MW4c^G%yK2m4MkYamVgdNs8e50@ zW#hPb#Nw-(Soo_8G~K@0&=0GjqK)YNZd($+;iv%-VI|UW7h$na0fzFS&c`Yd4sl1+ zQq<<~7H|bl$k5IE(})|#cVk1EJ_4&xw0IgUs^&RmVHx9)XX`VDhBxY`WHhLWsAu5R zUH=!UB$Oxjcbcx%?--Hk%lNzMGIIzgkqPc9 z{3-{Cs2cCaD!JWyKV1~p<2|p5Wy?J6Cs8T|1r*xUXR_u;+AXFR*E=l2W?Y_p*Czuk zbj&$oaaUDOTLu2oYu5x$#tzg44^z=+Ztp46$Fnk4JSG46kP0LuE{`{Bjgys~=x zZ#}=KxSKdmcfL#WrB?7Vse{JXre8#|;@lp%`_P#%n4>Zs{R^4O+k0jplQl-hpk zVX~|{hnzGa1njgo2rBo(sCvbImh=5lG_`NS5rZ3lHz8jl!G!)JwMTV~)@ni@*XunV zV^AG8T+=qwN|i0k;i+z~Nu#VV4~>#Wk8UiPyTWrV;?}L>de5m*y}mNJ8~sg;fHrDDd`q)@mVkRwg~@i_z?&2#U-K`*c&j z{6dh#f!Yzz6A}Pj*|$wiLV8?+-;9$RVG6SX-4rcJ3#Jld z+(KWbiLUo^!N&S6){tCUunnVSQuS&A^P(L|wOwiz*OZR+h z)id+kPUL|vFFOyG5~(%$&ZfQ8ZYJ=U&nH0YL}D)e3>6hA&M3gzfgyQ1pfEYf73nS5&ZAOp)ERZx(6T0dJV>Owe?!pq{2WvBf7LH z#>AGgnqzT7gJ7$vOd$(mLj_HF7ThMrwfEsivJ2fY(sbc%(Af~C*@Jh3r;d-^_y4YX zVEra|u>5KIIv{^m-IE$TVar7T4gP?FlsM0aL>?`Twxfmy(F7wAHN?6i{`J8?L+-P< z_3)(UOXFgVgod+PjA z$WW$hWTMPivr2ppv`{wjC`7vwhIB5GH2B9y+V@mX#BnpJGb?cDOIrT$U@-vz91|O< zSt7kTEMsOO8npzv6|DA5!9XEA7X>i13q~NNg%!2Y)6>hH+SWp2l#Vh7)BR;?I7Qm4 zc^AlkLMS;0 z9mcbg)A8&?(0Nr+xd+xnE;fwKBQ44*apdmxqrNyLQU3tBI9m>P0o5|q4xE1`9!jAz z))>qo`k=+^j2q3%RtP-!BaGY_k)^9-Q8H#X3}{xfShkY<(b?RF8c?J*5+dA(*k3U4 zpw)ien!v%40Fw&$1&|~Iek_hbLELtQE$}6I3gVp%N097G8Qn&)0Q5@czD7KzcOu4Z zOr+uvN^ZZdaHGYJ68urmDQGxFZ*B(=ZFBe_zzPq+gfF{?zYEpB1{HVw@FKIvCE1N~ zOAMzTu<_K_r$fkEBz)p&bK^ydLqUR}3~MP0h)H)Sj3ce*C9rD2R=FyQ(Crh&O!52D^-=H2RX zg!D9Vn9Z=l;r)oTK;ft!x`FmIGk}DkC>PC%+}2?#!Y4e5P=B1XyQ3rdiI1C8#e*cHfuFudF?9AzSwEXBlLY)PTtP-+$zMgcuUVa@=F1nv7>sCmKc6DMzf`M& z-^b0oLM~#9MVKax#!(NrnBz>jqU_LxjVm0;p9U~i4Jc6sXyIsIW2 zo3uA<_6L&=UBTaA#KWLPN}+z9kx?kNwm}Ko*0}r8m*h(bE69c}tBw~mDMbS#Jdq6W zB2}9~bg3knFg>C{;}{blffE$pE=LkF10)VSz;eU590j=s5*}}Bz%wKUOE}!nYcg}gsPnT`$Ysf~4^nn*Rge?EORNxQbj9SDY(^p1*nQTbCCq8kPueljbQ6)m@a zbhygdPylD+PgPA-%g(ly#1%&g+Xsl7vD?0s?JHM_z!CrO#Nkl;m?e>ObIN8!1cMS0 z)Pwu?-fWYd8@8g_JeIcZ`aF$o9%yD-kg?i1F6OHc_>^a_c%>^eXjk~5|K)c8$uYW_ zoQOrkSlZv(0b^3`UY;JT`P}ZA$AngkRXjyHZcW>cmdCo2JpHyKjGey$^qzJrC3nld zl!q9dJoOBwIlG+&k}|9qY&ANij242o${yg&aL z19We$NyTM~Kln>__Ov(reZ7eYB4CuUe66uuM8|g_qj?YYx9Hw>xh#O>VeV*J?T;E6 z=MX5JpV4|>$|Tr#W4k2#AK+zp`jw5*9%4|b*|u!e!d^A+G1jU~_jChlS}Gg#+87R0 z@WdSCH#=tC>o(%OHEx5Oa7NAdv%!mXce!ytmPx^DBCYB-Kyf*jSaXNnadwtJh87=Y;_K` z8DbmWn$`Qz#c#*M#gQw5k3UoFt67rUdXJdb=YyDnHYv_b$ zri}ZgCpar?`00hlBT`)YR1;YdO|fyA1U6I?X-SUMADfPoNHI-osoL%&bAxyJIRQ%G z`(yu;o7EiY8RZdy(rMnidwRk}Q`@sX{d_`?y|M%*fiBbFnYt=0>3T0Sw0t%bdNMku zj3Yk2ZvY?mNUHm~FX}B67z66yYWF|Opcf%a_4z=oBir(q+{ahCU*Yn)HH4_JvnxRZ-=aw%w=lAK=~kx!W}W< zn_NP?*VJnINQaizdNcJPzKW;)#3y>Xza1a~49zV4Jz|${&ozo`hgq~jxoHR+P@}+^ zA2`drf4x12?>`%I_Ge;WdDl#_*JEaN4-?+vOp*!{yn?x;_am|dTLFVFtU%@^K;xY;169K~1d%bRUC>8Q$%TXlvP#9(*_xzD@&i zu|Sv899BwsJ6$vL7K>xV0Y=T(+cATmzP-9V2P{(0vMXjw(f(;k6jQ;+<8;e`H&FIg zrVDdezED27U)&i?TFfkHnP~4ZYG8WdGAve7fIjThH#_nS%L@3elY&HAd#_yowm;>E z2u6ssT*qcbe{f!8y8OHblwemOom72!|Nt*2LMNrgXo`biLQ407JGLy-n*-i56^MEk|J z#7zpQ{qbsknzJ{gjp$jqyv}pmz%IFoSlVmZsd{%dSvJ*my26(qiw+}u)r9B7jE$w~ zozGV)omxT>0@_`_zZ(`sNqDX#7>@A6{hVO?$*ob5(MZ&^w<1W@fl<|+QvxCT-(TPA7gm;s%`C=K6b-L3 zN{3&qOVJwyo;P*Y%2-Mi9|7|-gnIaMXBsI))&2)F7~-mK=ElY&so8HF0Lv#Uzm{o9 z7(#+_1RRVvqPlAemmOGBa#(b~Zebb-H1G%6uU0g-><`AxDMj4ZI7}Pm%6r8!o2Mo? z5l^bQRNP$2uiaG+^Psd8=&hwV3H4k#fg93LCppm@kh_L*KF)a+WONag% z4gt5I*(#N6UR1GDWm)g~K_2HwIsK9K$n05a9ntYb2z#4c>3z@UAH{NYYN^0#aD37B z?nOIs@vO4a)4N%hf_krh?NFovgnKmE4lr$3uOPX!!BU{fY0cu~Y&kiV4g6ZKHrC4U z%`<&FYn%17u{|X0`!dzjkIVRgiEW)yoocn!3O3cKTvfw4s7+;^!&t9Qs|Ff(>k+c4=@s&+g~pjjM}DsddDLjq1aswsb2kXBB&$ z=nb*X>EGHG1&hk73E73#&T=txvLmh(I|R-p3^fqt`!$^m#T0VP6c&+9g5RAFFm2CO zjCen1aA%0!iY_xI5D_{!XKM$FPMnUUqE z9?{EExhdHI^e!X7XZr6|-|#MYS<^%XGkf?KQm%zgtqQc;#lXp8H~OK;yd@1WpQX0X zhLCFKPQ3|t{51?`-PF(44g(v=$3vpruoHE44FsTUD}^(^H(V_zcE*^FYO*QT#Tjf~ z{mM?((7vI*u-L8jf8kR8(^(sKAg3op;@mlO>J5D-ePp4ls zg-CF4k55Z^dIYP>o`?cH#>=G5nVYL#K0o!wsm!`2w=p5$wy>+$e|(uW-C*DF+$En1 z$aL7d7suANMQ;>nVuh>aDdV~7*FXf?Fu*z)$)JE%IZnpzu+_+>Pu;3Se-m)*qFY?+ z`_^c8=3Sqrz_JF6=f+}1Hoiq#tng7LEMs|NPUWyxR_!Atg8pn(LNB#T+9qOgzqF{* zER#V@{pkNo?jEwy?e!#~fM<1IrqyNaF5Yw{(_0rN18cJc}?!L>YVmi|E zEjKR@UU>e6B3uE9Y>G8$$Nez_%Hf_R#<8OCML|xM@dZA+vCrXuDAsuYOR*;D>0#sS z24rFea!R>Dz#g{FZ{NQ>AV4N*VGtLHj~m3z&&31c2k{#GXODk+D}%Km`ao}64;!G3 zuM5P@4Py1ytOW7(c6PIL|95LD&X)goL+aWx`W&*J4i4sECnw08L2Vmbci`LVVrlhe znGL85adUqQ1<1p}&CAUv1mfr8VFz*WvvYIsa&dqJfJRLJ1fXo|1$mR70Pz_y1MOYF z9yaa}FwpCNUCPhF%fbB?mAad=rKiO|PX5FAf7*C!((*L7gIIX{d%W7V9u9A5zn#@~ z1G~6DEPxy3{U zxXi&k7J`<%ycXObkcA+Zr8&PjuMiKH5Vxhd5QtmM_`m51TA0JYGRUSsf5%XkSWo~k zd!b8%HV{(c1}4q!v;fRk5K^Bd=w7t}--^*` zvTplIIdG%t%wfDj>#@hY9_`J3n1rgR!mKV~(gOy$I{j98&3=xh0^NmWI-IHJ_2zmQmF_q vI}$*P|MPAJ{%iRc0{=qbUkLnvg8-Vl2iVQS=WWhKdz+on=;>wDWYPW&&I4}W literal 0 HcmV?d00001 diff --git a/docs/releases/nemotron-math-v2/training.md b/docs/releases/nemotron-math-v2/training.md new file mode 100644 index 0000000000..9b7f61e7ea --- /dev/null +++ b/docs/releases/nemotron-math-v2/training.md @@ -0,0 +1,101 @@ +# Model training + +We assume you have `/workspace` defined in your [cluster config](../../basics/cluster-configs.md) and +that data and models will be downloaded to that folder, and you already follow [dataset.md](dataset.md) to get all SFT data ready. + + +## Prepare base model + +Download the base model. + +* [Qwen3-30B-A3B](https://huggingface.co/Qwen/Qwen3-30B-A3B) +* [Qwen3-8B](https://huggingface.co/Qwen/Qwen3-8B) + +Here is an example of commands for Qwen3-30B-A3B +```bash +pip install -U "huggingface_hub[cli]" +hf download Qwen/Qwen3-30B-A3B --local-dir Qwen3-30B-A3B +``` + + +## Run training + +Run the training (assuming slurm configuration here with the same folder structure). If your cluster has strict +timeout policy, you can run multiple dependent jobs with `dependent_jobs=N`. + +The following example shows the training script for Qwen3-30B-A3B. You can modify it accordingly for Qwen3-8B. + +```python +from nemo_skills.pipeline.cli import sft_nemo_rl, wrap_arguments +cluster = 'slurm' +tp = 8 +cp = 8 +pp = 1 +etp = 1 +emp = 8 +save_period=600 +max_steps = 7200 +batch_size=2048 +num_training_jobs=10 +warmup=0 +partition = 'interactive' +backend='megatron' +lr=2e-4 +min_lr=2e-4 +sft_nemo_rl( + ctx=wrap_arguments( + '++sft.max_num_epochs=2000 ' + f'++sft.max_num_steps={max_steps} ' + '++data.force_reprocess=false ' + '++data.num_workers=10 ' + f'++policy.megatron_cfg.tensor_model_parallel_size={tp} ' + f'++policy.megatron_cfg.context_parallel_size={cp} ' + f'++policy.megatron_cfg.expert_model_parallel_size={emp} ' + f'++policy.megatron_cfg.expert_tensor_parallel_size={etp} ' + f'++policy.megatron_cfg.pipeline_model_parallel_size={pp} ' + f'++policy.sequence_parallel=True ' + f'++policy.megatron_cfg.bias_activation_fusion=True ' + f'++policy.megatron_cfg.apply_rope_fusion=True ' + f'++checkpointing.save_period={save_period} ' + f'++policy.train_global_batch_size={batch_size} ' + f'++policy.max_total_sequence_length=131072 ' + f'++policy.megatron_cfg.optimizer.lr={lr} ' + '++policy.megatron_cfg.optimizer.bf16=True ' + f'++policy.megatron_cfg.optimizer.min_lr={min_lr} ' + f'++policy.megatron_cfg.scheduler.lr_warmup_iters={warmup} ' + f'++policy.megatron_cfg.scheduler.lr_decay_iters={max_steps} ' + '++policy.megatron_cfg.scheduler.lr_warmup_init=1e-7 ' + '++policy.megatron_cfg.scheduler.lr_decay_style=cosine ' + '++logger.swanlab_enabled=false ' + '++checkpointing.checkpoint_must_save_by=00:03:35:00 ' + ), + cluster=cluster, + wandb_project='sft-Qwen3-30B-A3B', + expname='nemo-rl-sft-Qwen3-30B-A3B', + backend='megatron', + output_dir='/workspace/final_sft_model', + hf_model='/workspace/Qwen3-30B-A3B', + training_data='/workspace/sft.jsonl', + num_gpus=8, + num_nodes=32, + dependent_jobs=num_training_jobs, +) + +``` + +### Training configuration by model and bucket length + + +| Model | Context length | TP | CP | PP | ETP | EMP | +|---------------|----------------|----|----|----|-----|-----| +| Qwen3-30B-A3B | 16k | 4 | 2 | 1 | 1 | 4 | +| Qwen3-30B-A3B | 32k | 4 | 4 | 1 | 1 | 8 | +| Qwen3-30B-A3B | 64k | 4 | 8 | 1 | 1 | 8 | +| Qwen3-30B-A3B | 128k | 4 | 8 | 1 | 1 | 8 | +| Qwen3-8B | 16k | 2 | 2 | 1 | - | - | +| Qwen3-8B | 32k | 2 | 4 | 1 | - | - | +| Qwen3-8B | 64k | 4 | 4 | 1 | - | - | +| Qwen3-8B | 128k | 8 | 8 | 1 | - | - | + + + diff --git a/mkdocs.yml b/mkdocs.yml index 899e8f3826..6ef479a929 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -91,23 +91,28 @@ nav: - tutorials/index.md - Papers & Releases: - releases/index.md + - Nemotron-Math-v2: + - releases/nemotron-math-v2/index.md + - Model evaluation: releases/nemotron-math-v2/evaluation.md + - Dataset construction: releases/nemotron-math-v2/dataset.md + - Model training: releases/nemotron-math-v2/training.md - Nemotron-Math-Proofs: releases/nemotronmathproofs/index.md - OpenReasoning: - releases/openreasoning/index.md - - Model Evaluation: releases/openreasoning/evaluation.md + - Model evaluation: releases/openreasoning/evaluation.md - Dataset construction: releases/openreasoning/dataset.md - Model training: releases/openreasoning/training.md - OpenCodeReasoning: - releases/opencodereasoning/index.md - - Model Evaluation: releases/opencodereasoning/evaluation.md + - Model evaluation: releases/opencodereasoning/evaluation.md - Dataset construction: releases/opencodereasoning/dataset.md - OpenMathReasoning: - releases/openmathreasoning/index.md - - Model Evaluation: releases/openmathreasoning/evaluation.md + - Model evaluation: releases/openmathreasoning/evaluation.md - Dataset construction: releases/openmathreasoning/dataset.md - Model training: releases/openmathreasoning/training.md - OpenMathInstruct-2: - releases/openmathinstruct2/index.md - - Model Evaluation: releases/openmathinstruct2/evaluation.md + - Model evaluation: releases/openmathinstruct2/evaluation.md - Dataset construction: releases/openmathinstruct2/dataset.md - Model training: releases/openmathinstruct2/training.md From 81198bc8a6fc099409582917a928b9a95152cf59 Mon Sep 17 00:00:00 2001 From: Nick Ludwig Date: Tue, 16 Dec 2025 20:49:55 +0400 Subject: [PATCH 53/88] SWE-bench: don't pass external environment variables into Apptainer containers (#1116) Signed-off-by: Nikolai Ludwig Signed-off-by: Cheng-Ping Hsieh --- nemo_skills/inference/eval/swebench.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nemo_skills/inference/eval/swebench.py b/nemo_skills/inference/eval/swebench.py index f3610091e8..5d761c9be7 100644 --- a/nemo_skills/inference/eval/swebench.py +++ b/nemo_skills/inference/eval/swebench.py @@ -355,7 +355,7 @@ async def _execute_container_command(self, data_point, command, expected_file_pa # Launch Apptainer container and execute the command apptainer_cmd = ( - f"apptainer exec --writable-tmpfs --no-mount home,tmp,bind-paths " + f"apptainer exec --writable-tmpfs --cleanenv --no-mount home,tmp,bind-paths " f"--mount type=bind,src=/nemo_run/code,dst=/nemo_run/code " f"--mount type=bind,src=/root,dst=/root_mount,ro " f"--mount type=bind,src={self.output_dir},dst=/trajectories_mount " From 5992dc7bcf3cfb14447477c83bd7a075f1a25dbb Mon Sep 17 00:00:00 2001 From: George <37293288+Jorjeous@users.noreply.github.com> Date: Wed, 17 Dec 2025 01:20:31 +0400 Subject: [PATCH 54/88] Adding clan PR with AudioBench and Librispeech PC. (#1103) Signed-off-by: George Zelenfroind Signed-off-by: George <37293288+Jorjeous@users.noreply.github.com> Signed-off-by: Cheng-Ping Hsieh --- docs/evaluation/speech-audio.md | 80 ++- nemo_skills/dataset/audiobench/__init__.py | 36 ++ .../dataset/audiobench/judge/__init__.py | 40 ++ .../dataset/audiobench/nonjudge/__init__.py | 31 + nemo_skills/dataset/audiobench/prepare.py | 606 ++++++++++++++++++ .../dataset/librispeech-pc/__init__.py | 29 + nemo_skills/dataset/librispeech-pc/prepare.py | 215 +++++++ nemo_skills/evaluation/evaluator/audio.py | 13 +- .../evaluation/metrics/audio_metrics.py | 60 +- nemo_skills/pipeline/prepare_data.py | 2 +- .../prompt/config/judge/audiobench.yaml | 28 + .../config/judge/audiobench_binary.yaml | 29 + tests/gpu-tests/test_eval.py | 2 + tests/test_datasets.py | 2 + 14 files changed, 1147 insertions(+), 26 deletions(-) create mode 100644 nemo_skills/dataset/audiobench/__init__.py create mode 100644 nemo_skills/dataset/audiobench/judge/__init__.py create mode 100644 nemo_skills/dataset/audiobench/nonjudge/__init__.py create mode 100644 nemo_skills/dataset/audiobench/prepare.py create mode 100644 nemo_skills/dataset/librispeech-pc/__init__.py create mode 100644 nemo_skills/dataset/librispeech-pc/prepare.py create mode 100644 nemo_skills/prompt/config/judge/audiobench.yaml create mode 100644 nemo_skills/prompt/config/judge/audiobench_binary.yaml diff --git a/docs/evaluation/speech-audio.md b/docs/evaluation/speech-audio.md index 9a5f7c5251..2e170891b3 100644 --- a/docs/evaluation/speech-audio.md +++ b/docs/evaluation/speech-audio.md @@ -2,8 +2,10 @@ This section details how to evaluate speech and audio benchmarks, including understanding tasks that test models' ability to reason about audio content (speech, music, environmental sounds) and ASR tasks for transcription. -!!! note - Currently supports only Megatron server type (`--server_type=megatron`). +!!! warning "Running without audio files" + If you want to evaluation without audio files (not recommended) use + `--no-audio` flag. In this case you can also set `--skip_data_dir_check` + as data is very lightweight when audio files aren't being used. ## Supported benchmarks @@ -35,12 +37,9 @@ MMAU-Pro (Multimodal Audio Understanding - Pro) is a comprehensive benchmark for These benchmarks require audio files for meaningful evaluation. **Audio files are downloaded by default** to ensure proper evaluation. -!!! warning "Running without audio files" - If you want to evaluate without audio files (not recommended) use - `--no-audio` flag. In this case you can also set `--skip_data_dir_check` - as data is very lightweight when audio files aren't being used. +### Data Preparation -### ASR Leaderboard +To prepare the dataset with audio files: ```bash ns prepare_data asr-leaderboard --data_dir=/path/to/data --cluster= @@ -55,7 +54,7 @@ ns prepare_data asr-leaderboard --datasets librispeech_clean ami ### MMAU-Pro ```bash -ns prepare_data mmau-pro --data_dir=/path/to/data --cluster= +ns prepare_data mmau-pro --no-audio --skip_data_dir_check ``` ## Running Evaluation @@ -344,3 +343,68 @@ pass@1 | 0 | 6580 | 55.52% | 0.00% | 290 evaluation_mode | avg_tokens | gen_seconds | success_rate | no_answer | num_entries pass@1 | 11 | 6879 | 31.44% | 0.00% | 5305 ``` + +## AudioBench + +AudioBench is a comprehensive benchmark for evaluating speech and audio language models across multiple tasks including ASR, translation, speech QA, and audio understanding. + +### Dataset Location + +- Benchmark is defined in [`nemo_skills/dataset/audiobench/__init__.py`](https://github.com/NVIDIA-NeMo/Skills/blob/main/nemo_skills/dataset/audiobench/__init__.py) +- External source repository is [AudioBench](https://github.com/AudioLLMs/AudioBench) + +### Data Preparation + +AudioBench can be prepared via the NeMo-Skills data preparation entrypoint. By default it will download/copy audio files into the prepared dataset directory. + +```bash +ns prepare_data audiobench --data_dir=/path/to/data --cluster= +``` + +To prepare without saving audio files (not recommended): + +```bash +ns prepare_data audiobench --no-audio --skip_data_dir_check +``` + +## LibriSpeech-PC + +LibriSpeech-PC is an Automatic Speech Recognition (ASR) benchmark that evaluates models' ability to transcribe speech with proper punctuation and capitalization. It builds upon the original LibriSpeech corpus with enhanced reference transcripts. + +### Dataset Location + +- Benchmark is defined in [`nemo_skills/dataset/librispeech-pc/__init__.py`](https://github.com/NVIDIA-NeMo/Skills/blob/main/nemo_skills/dataset/librispeech-pc/__init__.py) +- Manifests (with punctuation/capitalization) from [OpenSLR-145](https://www.openslr.org/145/) +- Audio files from original [LibriSpeech OpenSLR-12](https://www.openslr.org/12/) + +### Available Splits + +- `test-clean`: Clean speech recordings (easier subset) +- `test-other`: More challenging recordings with varied acoustic conditions + +## Preparing LibriSpeech-PC Data + +LibriSpeech-PC requires audio files for ASR evaluation. **Audio files are downloaded by default**. + +### Data Preparation + +To prepare the dataset with audio files: + +```bash +ns prepare_data librispeech-pc --data_dir=/path/to/data --cluster= +``` + +### Preparing Specific Splits + +To prepare only one split: + +```bash +ns prepare_data librispeech-pc --split test-clean --data_dir=/path/to/data +``` + +or + +```bash +ns prepare_data librispeech-pc --split test-other --data_dir=/path/to/data +``` + diff --git a/nemo_skills/dataset/audiobench/__init__.py b/nemo_skills/dataset/audiobench/__init__.py new file mode 100644 index 0000000000..152d2ac721 --- /dev/null +++ b/nemo_skills/dataset/audiobench/__init__.py @@ -0,0 +1,36 @@ +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""AudioBench: A comprehensive benchmark for speech and audio language models. + +AudioBench evaluates models across multiple tasks: +- ASR (Automatic Speech Recognition) +- Translation (speech-to-text translation) +- Speech QA (question answering based on audio) +- Audio understanding (emotion, gender, accent recognition, etc.) + +The benchmark is organized into two main categories: +- nonjudge: Tasks evaluated with automatic metrics (WER, BLEU) +- judge: Tasks requiring LLM-as-a-judge evaluation +""" + +DATASET_GROUP = "speechlm" +IS_BENCHMARK_GROUP = True +SCORE_MODULE = "nemo_skills.evaluation.metrics.audio_metrics" + +# Top-level benchmarks: evaluate all judge or all nonjudge datasets +BENCHMARKS = { + "audiobench.nonjudge": {}, + "audiobench.judge": {}, +} diff --git a/nemo_skills/dataset/audiobench/judge/__init__.py b/nemo_skills/dataset/audiobench/judge/__init__.py new file mode 100644 index 0000000000..62e48d4ec6 --- /dev/null +++ b/nemo_skills/dataset/audiobench/judge/__init__.py @@ -0,0 +1,40 @@ +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""AudioBench judge tasks dataset configuration. + +This dataset includes tasks that require LLM-based evaluation such as: +- Audio captioning +- Spoken question answering +- Audio understanding and reasoning + +These tasks require an LLM judge for evaluation, matching MMAU-Pro evaluation setup. +""" + +# Dataset configuration - CRITICAL: needed for audio to work +DATASET_GROUP = "speechlm" +METRICS_TYPE = "audio" +DEFAULT_SPLIT = "test" +GENERATION_ARGS = "++prompt_format=openai " +EVAL_ARGS = "++eval_type=audio " + +# Judge configuration matching AudioBench official implementation +# Using Llama-3.1-70B with vllm (can be overridden in run scripts) +JUDGE_PIPELINE_ARGS = { + "model": "meta-llama/Meta-Llama-3.1-70B-Instruct", + "server_type": "vllm", + "server_gpus": 8, + "server_args": "--max-model-len 8192 --gpu-memory-utilization 0.95", +} +JUDGE_ARGS = "++prompt_config=judge/audiobench ++generation_key=judgement" diff --git a/nemo_skills/dataset/audiobench/nonjudge/__init__.py b/nemo_skills/dataset/audiobench/nonjudge/__init__.py new file mode 100644 index 0000000000..d26668ce8f --- /dev/null +++ b/nemo_skills/dataset/audiobench/nonjudge/__init__.py @@ -0,0 +1,31 @@ +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""AudioBench non-judge tasks dataset configuration. + +This dataset includes ASR, translation, and other tasks that use +automatic metrics (WER, BLEU, WER-PC) instead of judge evaluation. + +NO JUDGE REQUIRED - Metrics computed automatically from model outputs. +""" + +# Dataset configuration - CRITICAL: needed for audio to work +DATASET_GROUP = "speechlm" +METRICS_TYPE = "audio" + +# Evaluation settings +EVAL_ARGS = "++eval_type=audio " + +# Generation settings - OpenAI format for audio-language models +GENERATION_ARGS = "++prompt_format=openai " diff --git a/nemo_skills/dataset/audiobench/prepare.py b/nemo_skills/dataset/audiobench/prepare.py new file mode 100644 index 0000000000..40fb75acd7 --- /dev/null +++ b/nemo_skills/dataset/audiobench/prepare.py @@ -0,0 +1,606 @@ +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""AudioBench Dataset Preparation for nemo-skills + +This script prepares AudioBench datasets for evaluation with nemo-skills. +AudioBench is a comprehensive benchmark for evaluating speech and audio models +across multiple tasks including ASR, translation, speech QA, and more. + +Usage: + python -m nemo_skills.dataset.audiobench.prepare --split test + python -m nemo_skills.dataset.audiobench.prepare --datasets librispeech_test_clean earnings21_test + python -m nemo_skills.dataset.audiobench.prepare --category nonjudge +""" + +import argparse +import json +import os +import shutil +from pathlib import Path +from typing import Dict, List + +import numpy as np +import soundfile as sf +from tqdm import tqdm + +# AudioBench datasets categorized by evaluation type +JUDGE_DATASETS = [ + "alpaca_audio_test", + "audiocaps_qa_test", + "audiocaps_test", + "clotho_aqa_test", + "cn_college_listen_mcq_test", + "dream_tts_mcq_test", + "iemocap_emotion_test", + "iemocap_gender_test", + "imda_ar_dialogue", + "imda_ar_sentence", + "imda_gr_dialogue", + "imda_gr_sentence", + "imda_part3_30s_ds_human_test", + "imda_part4_30s_ds_human_test", + "imda_part5_30s_ds_human_test", + "imda_part6_30s_ds_human_test", + "imda_part3_30s_sqa_human_test", + "imda_part4_30s_sqa_human_test", + "imda_part5_30s_sqa_human_test", + "imda_part6_30s_sqa_human_test", + "meld_emotion_test", + "meld_sentiment_test", + "mmau_mini", + "muchomusic_test", + "openhermes_audio_test", + "public_sg_speech_qa_test", + "slue_p2_sqa5_test", + "spoken_squad_test", + "voxceleb_accent_test", + "voxceleb_gender_test", + "wavcaps_qa_test", + "wavcaps_test", +] + +NONJUDGE_DATASETS = [ + "aishell_asr_zh_test", + "common_voice_15_en_test", + "covost2_en_id_test", + "covost2_en_ta_test", + "covost2_en_zh_test", + "covost2_id_en_test", + "covost2_ta_en_test", + "covost2_zh_en_test", + "earnings21_test", + "earnings22_test", + "gigaspeech_test", + "gigaspeech2_indo", + "gigaspeech2_thai", + "gigaspeech2_viet", + "imda_part1_asr_test", + "imda_part2_asr_test", + "imda_part3_30s_asr_test", + "imda_part4_30s_asr_test", + "imda_part5_30s_asr_test", + "imda_part6_30s_asr_test", + "librispeech_test_clean", + "librispeech_test_other", + "peoples_speech_test", + "seame_dev_man", + "seame_dev_sge", + "spoken-mqa_long_digit", + "spoken-mqa_multi_step_reasoning", + "spoken-mqa_short_digit", + "spoken-mqa_single_step_reasoning", + "tedlium3_test", + "tedlium3_long_form_test", +] + + +def get_audio_duration(audio_array: np.ndarray, sampling_rate: int) -> float: + """Compute audio duration in seconds from array and sampling rate.""" + if audio_array is None or len(audio_array) == 0: + return 0.0 + return float(len(audio_array) / sampling_rate) + + +def save_audio_file(audio_array: np.ndarray, sampling_rate: int, output_path: str): + """Save audio array to WAV file.""" + os.makedirs(os.path.dirname(output_path), exist_ok=True) + sf.write(output_path, audio_array, sampling_rate) + + +def extract_audio_dict(sample: Dict) -> Dict | None: + """Extract an Audio feature dict from a HuggingFace sample. + + AudioLLMs-hosted AudioBench datasets commonly store audio under the `context` + column (HF Audio feature), while other sources may use `audio`. + """ + # Prefer official HF Audio feature columns if present + for key in ("context", "audio"): + audio_dict = sample.get(key) + if isinstance(audio_dict, dict): + return audio_dict + return None + + +def create_manifest_entry( + sample: Dict, + audio_filename: str, + duration: float, + dataset_name: str, + sample_id: int, + category: str, +) -> Dict: + """Create a nemo-skills compatible manifest entry. + + Args: + sample: Raw sample from AudioBench dataset + audio_filename: Audio filename (relative path within audiobench directory) + duration: Audio duration in seconds + dataset_name: Name of the dataset + sample_id: Sample index + category: Category (judge/nonjudge) + + Returns: + Manifest entry dict with proper format for nemo-skills + """ + instruction = sample.get("instruction", sample.get("text", "Process the audio")) + reference = sample.get("reference", sample.get("answer", "")) + task_type = sample.get("task_type", "unknown") + + # Create absolute audio path with /data/ prefix for cluster deployment + # Format: /data/audiobench/{category}/audio/{dataset_name}/{filename} + audio_rel_path = f"/data/audiobench/{category}/audio/{dataset_name}/{audio_filename}" + + # Create audio metadata (both singular and plural forms for compatibility) + audio_metadata = {"path": audio_rel_path, "duration": duration} + + entry = { + "expected_answer": reference, + "audio_path": [audio_rel_path], + # Used by audio metrics to decide whether to parse LLM-as-a-judge results. + # AudioBench "judge" datasets are open-ended (judged), while "nonjudge" datasets are closed-form. + "category": "open" if category == "judge" else ("closed" if category == "nonjudge" else category), + "messages": [ + {"role": "system", "content": "You are a helpful assistant. /no_think"}, + { + "role": "user", + "content": instruction, + "audio": audio_metadata, + "audios": [audio_metadata], + }, + ], + "dataset": dataset_name, + "subset_for_metrics": dataset_name, + "sample_id": sample_id, + "task_type": task_type, + "question": instruction, + } + + for key in [ + "choices", + "options", + "audio_text_instruction", + "audio_gt", + "dimension", + "rule_type", + "rule_target", + "task", + ]: + if key in sample: + entry[key] = sample[key] + + return entry + + +def process_dataset( + dataset_name: str, + output_dir: Path, + save_audio: bool = True, + split: str = "test", + max_samples: int = -1, +) -> tuple[int, List[Dict]]: + """Process a single AudioBench dataset. + + Args: + dataset_name: Name of the dataset to process + output_dir: Base output directory + save_audio: Whether to save audio files + split: Dataset split (default: "test") + max_samples: Max number of samples to process (-1 for all) + + Returns: + Tuple of (num_samples, manifest_entries) + """ + print(f"\n{'=' * 60}") + print(f"Processing: {dataset_name}") + print(f"{'=' * 60}") + + try: + from datasets import load_dataset + except Exception as e: + raise ImportError( + f"Failed to import HuggingFace 'datasets'. Please ensure it is installed.\nOriginal error: {e}" + ) + + # Upstream reference: https://github.com/AudioLLMs/AudioBench + try: + # AudioBench mapping for datasets that are not 1:1 AudioLLMs/. + hf_map = { + # AudioLLMs org aliases + "aishell_asr_zh_test": {"repo": "AudioLLMs/aishell_1_zh_test", "split": "test"}, + "muchomusic_test": {"repo": "AudioLLMs/mu_chomusic_test", "split": "test"}, + "openhermes_audio_test": {"repo": "AudioLLMs/openhermes_instruction_test", "split": "test"}, + "iemocap_emotion_test": {"repo": "AudioLLMs/iemocap_emotion_recognition", "split": "test"}, + "iemocap_gender_test": {"repo": "AudioLLMs/iemocap_gender_recognition", "split": "test"}, + "mmau_mini": { + "repo": "AudioLLMs/MMAU-mini", + "split": "test", + "fallback_repo": "AudioLLMs/MMAU-mini-do-not-use", + }, + # GigaSpeech2 variants (one repo with data_dir selector) + "gigaspeech2_thai": {"repo": "AudioLLMs/gigaspeech2-test", "split": "train", "data_dir": "th-test"}, + "gigaspeech2_indo": {"repo": "AudioLLMs/gigaspeech2-test", "split": "train", "data_dir": "id-test"}, + "gigaspeech2_viet": {"repo": "AudioLLMs/gigaspeech2-test", "split": "train", "data_dir": "vi-test"}, + "spoken-mqa_short_digit": {"repo": "amao0o0/spoken-mqa", "split": "short_digit"}, + "spoken-mqa_long_digit": {"repo": "amao0o0/spoken-mqa", "split": "long_digit"}, + "spoken-mqa_single_step_reasoning": {"repo": "amao0o0/spoken-mqa", "split": "single_step_reasoning"}, + "spoken-mqa_multi_step_reasoning": {"repo": "amao0o0/spoken-mqa", "split": "multi_step_reasoning"}, + "imda_part1_asr_test": { + "repo": "MERaLiON/Multitask-National-Speech-Corpus-v1", + "split": "train", + "data_dir": "ASR-PART1-Test", + }, + "imda_part2_asr_test": { + "repo": "MERaLiON/Multitask-National-Speech-Corpus-v1", + "split": "train", + "data_dir": "ASR-PART2-Test", + }, + "imda_part3_30s_asr_test": { + "repo": "MERaLiON/Multitask-National-Speech-Corpus-v1", + "split": "train", + "data_dir": "ASR-PART3-Test", + }, + "imda_part4_30s_asr_test": { + "repo": "MERaLiON/Multitask-National-Speech-Corpus-v1", + "split": "train", + "data_dir": "ASR-PART4-Test", + }, + "imda_part5_30s_asr_test": { + "repo": "MERaLiON/Multitask-National-Speech-Corpus-v1", + "split": "train", + "data_dir": "ASR-PART5-Test", + }, + "imda_part6_30s_asr_test": { + "repo": "MERaLiON/Multitask-National-Speech-Corpus-v1", + "split": "train", + "data_dir": "ASR-PART6-Test", + }, + "imda_part3_30s_sqa_human_test": { + "repo": "MERaLiON/Multitask-National-Speech-Corpus-v1", + "split": "train", + "data_dir": "SQA-PART3-Test", + }, + "imda_part4_30s_sqa_human_test": { + "repo": "MERaLiON/Multitask-National-Speech-Corpus-v1", + "split": "train", + "data_dir": "SQA-PART4-Test", + }, + "imda_part5_30s_sqa_human_test": { + "repo": "MERaLiON/Multitask-National-Speech-Corpus-v1", + "split": "train", + "data_dir": "SQA-PART5-Test", + }, + "imda_part6_30s_sqa_human_test": { + "repo": "MERaLiON/Multitask-National-Speech-Corpus-v1", + "split": "train", + "data_dir": "SQA-PART6-Test", + }, + "imda_part3_30s_ds_human_test": { + "repo": "MERaLiON/Multitask-National-Speech-Corpus-v1", + "split": "train", + "data_dir": "SDS-PART3-Test", + }, + "imda_part4_30s_ds_human_test": { + "repo": "MERaLiON/Multitask-National-Speech-Corpus-v1", + "split": "train", + "data_dir": "SDS-PART4-Test", + }, + "imda_part5_30s_ds_human_test": { + "repo": "MERaLiON/Multitask-National-Speech-Corpus-v1", + "split": "train", + "data_dir": "SDS-PART5-Test", + }, + "imda_part6_30s_ds_human_test": { + "repo": "MERaLiON/Multitask-National-Speech-Corpus-v1", + "split": "train", + "data_dir": "SDS-PART6-Test", + }, + "imda_ar_sentence": { + "repo": "MERaLiON/Multitask-National-Speech-Corpus-v1", + "split": "train", + "data_dir": "PQA-AR-Sentence-Test", + }, + "imda_ar_dialogue": { + "repo": "MERaLiON/Multitask-National-Speech-Corpus-v1", + "split": "train", + "data_dir": "PQA-AR-Dialogue-Test", + }, + "imda_gr_sentence": { + "repo": "MERaLiON/Multitask-National-Speech-Corpus-v1", + "split": "train", + "data_dir": "PQA-GR-Sentence-Test", + }, + "imda_gr_dialogue": { + "repo": "MERaLiON/Multitask-National-Speech-Corpus-v1", + "split": "train", + "data_dir": "PQA-GR-Dialogue-Test", + }, + } + + spec = hf_map.get(dataset_name) + if spec is None: + hf_repo = f"AudioLLMs/{dataset_name}" + hf_split = split + hf_ds = load_dataset(hf_repo, split=hf_split) + else: + hf_repo = spec["repo"] + hf_split = spec.get("split", split) + data_dir = spec.get("data_dir") + if data_dir: + hf_ds = load_dataset(hf_repo, data_dir=data_dir, split=hf_split) + else: + hf_ds = load_dataset(hf_repo, split=hf_split) + + fallback_repo = spec.get("fallback_repo") + if fallback_repo: + # Only try fallback if the primary repo is missing/inaccessible. + # (Keep behavior deterministic and close to upstream mapping.) + try: + _ = len(hf_ds) + except Exception: + hf_repo = fallback_repo + hf_ds = load_dataset(hf_repo, split=hf_split) + + if max_samples is not None and int(max_samples) > 0: + hf_ds = hf_ds.select(range(min(int(max_samples), len(hf_ds)))) + data_samples = hf_ds + print(f"Loaded {len(hf_ds)} samples via HuggingFace datasets: {hf_repo} (split={hf_split})") + except Exception as e: + raise Exception( + "Failed to load AudioBench dataset via HuggingFace.\n" + f"- Requested dataset_name: {dataset_name}\n" + f"- HuggingFace dataset repo attempted: {locals().get('hf_repo', 'UNKNOWN')}\n" + f"- Split: {locals().get('hf_split', split)}\n" + "Please verify the dataset exists under the AudioLLMs org:\n" + " https://huggingface.co/AudioLLMs/datasets\n" + f"Original error: {e}" + ) + + # Determine category + dataset_base = dataset_name.replace("_test", "") + if dataset_name in JUDGE_DATASETS or dataset_base in JUDGE_DATASETS: + category = "judge" + elif dataset_name in NONJUDGE_DATASETS or dataset_base in NONJUDGE_DATASETS: + category = "nonjudge" + else: + category = "unknown" + + # Output directories + audio_dir = output_dir / category / "audio" / dataset_name + dataset_dir = output_dir / category / dataset_name + os.makedirs(audio_dir, exist_ok=True) + os.makedirs(dataset_dir, exist_ok=True) + + # Copy __init__.py from category folder to dataset folder + category_init = output_dir / category / "__init__.py" + dataset_init = dataset_dir / "__init__.py" + if category_init.exists() and not dataset_init.exists(): + shutil.copy2(category_init, dataset_init) + print(f"✓ Copied __init__.py to {dataset_dir}") + + manifest_entries = [] + successful = 0 + failed = 0 + + for idx, sample in enumerate(tqdm(data_samples, desc=f"Processing {dataset_name}")): + try: + # Get audio data + audio_dict = extract_audio_dict(sample) + if audio_dict is None: + print(f"Warning: Sample {idx} has no audio, skipping") + failed += 1 + continue + + # Extract audio array and sampling rate + audio_array = audio_dict.get("array") + sampling_rate = audio_dict.get("sampling_rate", 16000) + + if audio_array is None or len(audio_array) == 0: + print(f"Warning: Empty audio at sample {idx}, skipping") + failed += 1 + continue + + # Convert to numpy array if needed + if isinstance(audio_array, list): + audio_array = np.array(audio_array) + + # Compute duration + duration = get_audio_duration(audio_array, sampling_rate) + + # Define audio file paths + audio_filename = f"{dataset_name}_{idx:06d}.wav" + local_audio_path = audio_dir / audio_filename + + # Save audio file + if save_audio: + try: + save_audio_file(audio_array, sampling_rate, str(local_audio_path)) + except Exception as e: + print(f"Warning: Failed to save audio for sample {idx}: {e}") + failed += 1 + continue + + # Create manifest entry with relative path + entry = create_manifest_entry( + sample=sample, + audio_filename=audio_filename, + duration=duration, + dataset_name=dataset_name, + sample_id=idx, + category=category, + ) + + manifest_entries.append(entry) + successful += 1 + + except Exception as e: + print(f"Error processing sample {idx}: {e}") + failed += 1 + continue + + # Save dataset-specific manifest to dataset directory + manifest_path = dataset_dir / f"{split}.jsonl" + with open(manifest_path, "w", encoding="utf-8") as f: + for entry in manifest_entries: + f.write(json.dumps(entry, ensure_ascii=False) + "\n") + + print(f"✓ Saved {successful} samples to {manifest_path}") + if failed > 0: + print(f"✗ Failed to process {failed} samples") + + return successful, manifest_entries + + +def main(): + parser = argparse.ArgumentParser(description="Prepare AudioBench datasets for nemo-skills evaluation") + parser.add_argument( + "--split", + default="test", + choices=["train", "validation", "test"], + help="Dataset split to prepare", + ) + parser.add_argument( + "--output_dir", + type=str, + default=None, + help="Output directory (defaults to $NEMO_SKILLS_DATA_DIR/audiobench)", + ) + parser.add_argument( + "--datasets", + nargs="+", + help="Specific dataset(s) to process (e.g., librispeech_test_clean earnings21)", + ) + parser.add_argument( + "--category", + choices=["judge", "nonjudge", "all"], + default="all", + help="Process only judge, nonjudge, or all datasets", + ) + parser.add_argument( + "--no-audio", + dest="save_audio", + action="store_false", + help="Skip saving audio files (only create manifests)", + ) + parser.add_argument( + "--max-samples", + type=int, + default=-1, + help="Maximum number of samples to process per dataset (-1 for all)", + ) + parser.set_defaults(save_audio=True) + + args = parser.parse_args() + + # Determine output directory + if args.output_dir: + output_dir = Path(args.output_dir) + else: + # Use dataset directory as output (files will be in nemo_skills/dataset/audiobench/) + output_dir = Path(__file__).parent + + output_dir.mkdir(parents=True, exist_ok=True) + + print("\n" + "=" * 60) + print("AudioBench Dataset Preparation") + print("=" * 60) + print("AudioBench source: HuggingFace datasets (AudioLLMs/AudioBench)") + print(f"Output directory: {output_dir}") + print(f"Save audio files: {args.save_audio}") + print(f"Split: {args.split}") + print("=" * 60 + "\n") + + # Determine which datasets to process + if args.datasets: + target_datasets = args.datasets + else: + all_datasets = JUDGE_DATASETS + NONJUDGE_DATASETS + if args.category == "judge": + target_datasets = JUDGE_DATASETS + elif args.category == "nonjudge": + target_datasets = NONJUDGE_DATASETS + else: # all + target_datasets = all_datasets + + # Initialize category folders with __init__.py for nemo-skills to find dataset defaults + for category in ["judge", "nonjudge"]: + category_dir = output_dir / category + category_dir.mkdir(exist_ok=True) + + # Copy category __init__.py + init_file = category_dir / "__init__.py" + template_init = output_dir / category / "__init__.py" + if not init_file.exists() and template_init.exists(): + shutil.copy2(template_init, init_file) + + total_samples = 0 + total_datasets = 0 + + for name in target_datasets: + # Normalize dataset name: allow passing without _test suffix + dataset_name = name + if dataset_name not in JUDGE_DATASETS and dataset_name not in NONJUDGE_DATASETS: + # Try adding _test suffix (AudioBench uses mixed naming) + if f"{dataset_name}_test" in JUDGE_DATASETS or f"{dataset_name}_test" in NONJUDGE_DATASETS: + dataset_name = f"{dataset_name}_test" + + # Determine category for logging + category = "judge" if name in JUDGE_DATASETS else "nonjudge" + + try: + num_samples, _ = process_dataset( + dataset_name=dataset_name, + output_dir=output_dir, + save_audio=args.save_audio, + split=args.split, + max_samples=args.max_samples, + ) + total_samples += num_samples + total_datasets += 1 + print(f"✓ Completed {dataset_name}: {num_samples} samples") + except Exception as e: + print(f"✗ Failed {dataset_name}: {e}") + continue + + print("\n" + "=" * 60) + print("AudioBench Preparation Summary") + print("=" * 60) + print(f"Datasets processed: {total_datasets}/{len(target_datasets)}") + print(f"Total samples: {total_samples}") + print(f"Output directory: {output_dir}") + print("=" * 60) + + +if __name__ == "__main__": + main() diff --git a/nemo_skills/dataset/librispeech-pc/__init__.py b/nemo_skills/dataset/librispeech-pc/__init__.py new file mode 100644 index 0000000000..28b02d9656 --- /dev/null +++ b/nemo_skills/dataset/librispeech-pc/__init__.py @@ -0,0 +1,29 @@ +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""LibriSpeech-PC: ASR evaluation with Punctuation and Capitalization. + +Test sets (evaluation only): +- test-clean: Clean speech recordings (~2.6k samples) +- test-other: More challenging speech with various acoustic conditions (~2.9k samples) +""" + +DATASET_GROUP = "speechlm" +METRICS_TYPE = "audio" +DEFAULT_SPLIT = "test-clean" + + +EVAL_SPLIT = "test-clean" +EVAL_ARGS = "++eval_type=audio " +GENERATION_ARGS = "++prompt_format=openai " diff --git a/nemo_skills/dataset/librispeech-pc/prepare.py b/nemo_skills/dataset/librispeech-pc/prepare.py new file mode 100644 index 0000000000..a260d864c3 --- /dev/null +++ b/nemo_skills/dataset/librispeech-pc/prepare.py @@ -0,0 +1,215 @@ +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Prepare LibriSpeech-PC for ASR evaluation with punctuation and capitalization. + +LibriSpeech-PC provides manifests with punctuation/capitalization from OpenSLR-145. +Audio files are downloaded from original LibriSpeech at OpenSLR-12. + +Usage: + ns prepare_data librispeech-pc --data_dir + ns prepare_data librispeech-pc --split test-clean (or test-other) --data_dir +""" + +import argparse +import json +import os +import shutil +import sys +import tarfile +import urllib.request +from pathlib import Path + +from tqdm import tqdm + + +def download_with_progress(url: str, output_path: Path, desc: str): + """Download file with tqdm progress bar.""" + with tqdm(unit="B", unit_scale=True, unit_divisor=1024, desc=desc) as pbar: + + def reporthook(blocknum, blocksize, totalsize): + if pbar.total != totalsize: + pbar.total = totalsize + downloaded = blocknum * blocksize + pbar.update(max(0, downloaded - pbar.n)) + + urllib.request.urlretrieve(url, output_path, reporthook) + + +# LibriSpeech-PC manifests (with punctuation and capitalization) +MANIFESTS_URL = "https://www.openslr.org/resources/145/manifests.tar.gz" + +# Original LibriSpeech audio files +AUDIO_URLS = { + "test-clean": "https://www.openslr.org/resources/12/test-clean.tar.gz", + "test-other": "https://www.openslr.org/resources/12/test-other.tar.gz", +} + + +def download_manifests(output_dir: Path) -> Path: + """Download LibriSpeech-PC manifests if not already present.""" + if (output_dir / "test-clean.json").exists() and (output_dir / "test-other.json").exists(): + return output_dir + + tar_path = output_dir / "manifests.tar.gz" + download_with_progress(MANIFESTS_URL, tar_path, "Downloading manifests") + + with tarfile.open(tar_path, "r:gz") as tar: + wanted = {"test-clean.json", "test-other.json"} + for member in tar.getmembers(): + name = Path(member.name).name + if name not in wanted: + continue + fobj = tar.extractfile(member) + if fobj is None: + continue + out_path = output_dir / name + with open(out_path, "wb") as fout: + shutil.copyfileobj(fobj, fout) + os.remove(tar_path) + + print("✓ Manifests ready\n") + return output_dir + + +def download_audio(split: str, audio_dir: Path): + """Download LibriSpeech audio files if not already present.""" + split_dir = audio_dir / "LibriSpeech" / split.replace("-", "_") + if split_dir.exists(): + return + + tar_path = audio_dir / f"{split}.tar.gz" + download_with_progress(AUDIO_URLS[split], tar_path, f"Downloading {split}") + + with tarfile.open(tar_path, "r:gz") as tar: + if sys.version_info >= (3, 11, 4): + tar.extractall(audio_dir, filter="data") + else: + tar.extractall(audio_dir) + os.remove(tar_path) + + +def process_split(split: str, data_dir: Path, audio_dir: Path, with_audio: bool) -> int: + """Process one LibriSpeech-PC split into nemo-skills format.""" + + output_file = data_dir / f"{split}.jsonl" + manifest_file = data_dir / f"{split}.json" + if not manifest_file.exists(): + print(f"✗ Manifest not found: {manifest_file}") + return 0 + + if with_audio: + download_audio(split, audio_dir) + + with open(manifest_file, "r") as f: + entries = [json.loads(line) for line in f if line.strip()] + + processed = 0 + skipped = 0 + + with open(output_file, "w") as fout: + for entry in entries: + audio_filepath = entry.get("audio_filepath", "") + text = entry.get("text", "") + + if not audio_filepath or not text: + skipped += 1 + continue + + audio_id = Path(audio_filepath).stem + + audio_root = os.getenv("NEMO_SKILLS_AUDIO_ROOT", "/data") + rel_audio_path = audio_filepath.lstrip("/") + if rel_audio_path.startswith("LibriSpeech/"): + rel_audio_path = rel_audio_path[len("LibriSpeech/") :] + container_path = f"{audio_root}/librispeech-pc/LibriSpeech/{rel_audio_path}" + + user_message = { + "role": "user", + "content": "Transcribe the audio with proper punctuation and capitalization.", + "audio": {"path": container_path}, + } + + output_entry = { + "audio_filepath": container_path, + "text": text, + "expected_answer": text, + "task_type": "ASR-PC", + "sample_id": audio_id, + "split": split, + "messages": [{"role": "system", "content": "You are a helpful assistant. /no_think"}, user_message], + } + + fout.write(json.dumps(output_entry, ensure_ascii=False) + "\n") + processed += 1 + + print(f"✓ {split}: {processed} samples" + (f" ({skipped} skipped)" if skipped > 0 else "")) + + if processed > 0 and manifest_file.exists(): + os.remove(manifest_file) + + return processed + + +def main(): + parser = argparse.ArgumentParser(description="Prepare LibriSpeech-PC for ASR evaluation") + parser.add_argument( + "--data_dir", + type=str, + default=os.getenv("NEMO_SKILLS_DATA_DIR"), + help=( + "Base data dir (defaults to $NEMO_SKILLS_DATA_DIR). " + "If provided, output goes under /librispeech-pc. " + "If omitted, writes into this package's dataset directory (only allowed outside site-packages)." + ), + ) + parser.add_argument( + "--split", + default="all", + choices=["all", "test-clean", "test-other"], + help="Which split to prepare (default: all)", + ) + parser.add_argument( + "--no-audio", + action="store_true", + help="Skip audio download", + ) + args = parser.parse_args() + + if args.data_dir: + data_dir = Path(args.data_dir) / "librispeech-pc" + else: + pkg_dir = Path(__file__).parent + pkg_dir_str = str(pkg_dir) + if "site-packages" in pkg_dir_str or "dist-packages" in pkg_dir_str: + raise SystemExit( + "Missing --data_dir and NEMO_SKILLS_DATA_DIR is not set. " + "Refusing to write into the installed package directory; please set NEMO_SKILLS_DATA_DIR " + "or pass --data_dir." + ) + data_dir = pkg_dir + + audio_dir = data_dir + audio_dir.mkdir(parents=True, exist_ok=True) + + download_manifests(data_dir) + + splits = ["test-clean", "test-other"] if args.split == "all" else [args.split] + total = sum(process_split(split, data_dir, audio_dir, not args.no_audio) for split in splits) + + print(f"\n✓ Complete: {total} samples") + + +if __name__ == "__main__": + main() diff --git a/nemo_skills/evaluation/evaluator/audio.py b/nemo_skills/evaluation/evaluator/audio.py index 7a087831a8..c212666311 100644 --- a/nemo_skills/evaluation/evaluator/audio.py +++ b/nemo_skills/evaluation/evaluator/audio.py @@ -325,13 +325,18 @@ def evaluate_sample(sample: dict[str, Any], config: AudioEvaluatorConfig) -> dic generation = sample.get("generation", "").strip() expected_answer = sample.get("expected_answer", "").strip() - if task_type in ["ASR", "ASR-PC", "AST", "CER", "ASR_LEADERBOARD"] and not generation: - return { + if task_type in ["ASR", "ASR-PC", "ASR_LEADERBOARD", "AST", "Translation", "CER"] and not generation: + base = { "is_correct": False, - "wer": 1.0, "error": "missing_generation", "predicted_answer": "", } + if task_type in ["AST", "Translation"]: + return {**base, "bleu": 0.0} + if task_type == "CER": + return {**base, "cer": 1.0} + # ASR / ASR-PC / ASR_LEADERBOARD + return {**base, "wer": 1.0} if task_type == "ASR-PC": metrics = evaluate_asr_pc( @@ -350,7 +355,7 @@ def evaluate_sample(sample: dict[str, Any], config: AudioEvaluatorConfig) -> dic updates.update(metrics) updates["predicted_answer"] = generation - elif task_type == "AST": + elif task_type in ["AST", "Translation"]: metrics = evaluate_translation(expected_answer, generation) updates.update(metrics) updates["predicted_answer"] = generation diff --git a/nemo_skills/evaluation/metrics/audio_metrics.py b/nemo_skills/evaluation/metrics/audio_metrics.py index a00a53f938..95a133833d 100644 --- a/nemo_skills/evaluation/metrics/audio_metrics.py +++ b/nemo_skills/evaluation/metrics/audio_metrics.py @@ -74,25 +74,43 @@ def __init__(self, compute_no_answer: bool = True, max_k: int = 1): self.cap_accuracy_scores = [] self.char_rate_scores = [] - def _extract_judge_result(self, judgement_text: str) -> bool: - """Extract judge result from judgement text. + # Judge scores (AudioBench-style rating 0-5, or legacy binary Yes/No mapped to 1/0) + self.judge_ratings = [] - Parses LLM judge output to determine if the response is correct. + def _extract_judge_result(self, judgement_text: str) -> tuple[bool, float]: + """Extract judge result from judgement text. - Args: - judgement_text: Text output from LLM judge + Supports two formats: + 1. AudioBench format: 'Rating: X' where X is 0-5 (returns rating as float) + 2. Legacy/binary format: 'Judgement: Yes/No' (mapped to 5.0/0.0 for consistent 0-100 scaling) Returns: - True if judge indicates correct, False otherwise + Tuple of (is_correct, rating_score) + - is_correct: True if rating >= 3 (or Yes for legacy) + - rating_score: 0-5 rating (or 0/5 for legacy binary) """ import re + # Try AudioBench format first: 'Rating: X' + rating_match = re.search(r"Rating:\s*([0-9]+(?:\.[0-9]+)?)", judgement_text, re.IGNORECASE) + if rating_match: + rating = float(rating_match.group(1)) + rating = max(0.0, min(5.0, rating)) + return rating >= 3.0, rating + + # Try explicit Judgement: Yes/No format + judgement_match = re.search(r"Judgement:\s*(Yes|No)", judgement_text, re.IGNORECASE) + if judgement_match: + is_yes = judgement_match.group(1).lower() == "yes" + return is_yes, 5.0 if is_yes else 0.0 + + # Last-resort: accept plain 'yes'/'no' anywhere in text if re.search(r"\byes\b", judgement_text, re.IGNORECASE): - return True - elif re.search(r"\bno\b", judgement_text, re.IGNORECASE): - return False - else: - return False + return True, 5.0 + if re.search(r"\bno\b", judgement_text, re.IGNORECASE): + return False, 0.0 + + return False, 0.0 def _get_score_dict(self, prediction: dict) -> dict[str, bool | int | float]: """Extract correctness scores from prediction. @@ -111,8 +129,9 @@ def _get_score_dict(self, prediction: dict) -> dict[str, bool | int | float]: category = prediction.get("category", "unknown") if "judgement" in prediction and category == "open": - judge_result = self._extract_judge_result(prediction["judgement"]) - score_dict["judge_correct"] = judge_result + judge_correct, judge_rating = self._extract_judge_result(prediction["judgement"]) + score_dict["judge_correct"] = judge_correct + score_dict["judge_rating"] = judge_rating if category == "open" and "judge_correct" in score_dict: score_dict["correct"] = score_dict["judge_correct"] @@ -194,6 +213,11 @@ def update(self, predictions): if "char_rate" in pred and pred["char_rate"] is not None: self.char_rate_scores.append(pred["char_rate"]) + # Collect judge ratings (0-5) from judge datasets if available + score_dict = self._get_score_dict(pred) + if "judge_rating" in score_dict: + self.judge_ratings.append(score_dict["judge_rating"]) + self._compute_pass_at_k(predictions=predictions, predicted_answers=predicted_answers) self._compute_majority_at_k(predictions=predictions, predicted_answers=predicted_answers) @@ -219,6 +243,12 @@ def get_metrics(self): elif "judge_correct" in agg_metrics: agg_metrics["success_rate"] = agg_metrics["judge_correct"] + # Add AudioBench-style judge_score if rating outputs were used. + # Formula: judge_score = mean(ratings) * 20 (converts 0-5 scale to 0-100) + if self.judge_ratings: + avg_rating = sum(self.judge_ratings) / len(self.judge_ratings) + agg_metrics["judge_score"] = avg_rating * 20 + # Add existing metrics: WER, PnC, and BLEU if available (convert to percentages and round to 2 decimals) if self.wer_scores: agg_metrics["wer"] = round(100.0 * sum(self.wer_scores) / len(self.wer_scores), 2) @@ -280,6 +310,10 @@ def metrics_to_print(self): if self.compute_no_answer: base_metrics["no_answer"] = as_percentage + # AudioBench-style judge_score (0-100, not a percent) + if self.judge_ratings: + base_metrics["judge_score"] = lambda _k, v, _all: f"{v:.2f}" + # Add existing metrics if they were computed if self.wer_scores: base_metrics["wer"] = as_percentage diff --git a/nemo_skills/pipeline/prepare_data.py b/nemo_skills/pipeline/prepare_data.py index 36820c7337..8c3a58a8ba 100644 --- a/nemo_skills/pipeline/prepare_data.py +++ b/nemo_skills/pipeline/prepare_data.py @@ -31,7 +31,7 @@ # TODO: read this from init.py -DATASETS_REQUIRE_DATA_DIR = ["ruler", "ioi24", "mmau-pro", "asr-leaderboard"] +DATASETS_REQUIRE_DATA_DIR = ["ruler", "ioi24", "mmau-pro", "librispeech-pc", "audiobench", "asr-leaderboard"] @app.command(context_settings={"allow_extra_args": True, "ignore_unknown_options": True}) diff --git a/nemo_skills/prompt/config/judge/audiobench.yaml b/nemo_skills/prompt/config/judge/audiobench.yaml new file mode 100644 index 0000000000..62faa1acb2 --- /dev/null +++ b/nemo_skills/prompt/config/judge/audiobench.yaml @@ -0,0 +1,28 @@ +# Judge prompt configuration for AudioBench evaluation +# Based on AudioBench's official production system with 0-5 rating scale + + +user: |- + [Reference Answer] + {expected_answer} + + [Model Answer] + {generation} + + [Question] + {question} + + [Task] + Rate the model's answer based on its alignment with the reference answer, focusing on accuracy and relevance to the reference provided. Please be critical on the details. If the model response is something like 'cannot decide', please rate as 0. + Criteria: Assess if the model's response mirrors the reference in terms of content, accuracy, and relevance. + Score0: The answer is refusing to give concrete results, providing something like 'cannot decide'. + Score0: The answer is completely misaligned, providing incorrect or irrelevant information compared to the reference. + Score1: The answer shows minimal alignment, often misunderstanding or providing irrelevant details unrelated to the reference. + Score2: The answer recognizes the topic but diverges significantly from the reference in accuracy or relevance. + Score3: The answer aligns with the reference generally but lacks detail or precise accuracy in some aspects. + Score4: The answer is mostly accurate and relevant, closely following the reference but could be clearer or more detailed. + Score5: The answer is highly accurate, detailed, and matches the reference answer perfectly, capturing its essence and detail. + + Your response should be formatted as follows: + Explanation: (Provide a concise explanation of your rating, comparing the reference answer with the model's response. "The reference answer is [XXX], while the model's answer is [YYY]. I think ...") + Rating: (int) diff --git a/nemo_skills/prompt/config/judge/audiobench_binary.yaml b/nemo_skills/prompt/config/judge/audiobench_binary.yaml new file mode 100644 index 0000000000..d121607a48 --- /dev/null +++ b/nemo_skills/prompt/config/judge/audiobench_binary.yaml @@ -0,0 +1,29 @@ +# Judge prompt configuration for AudioBench evaluation +# Based on AudioBench's official llama3_70b_as_judge_binary prompt +# (Adapted to nemo-skills Yes/No format (instead of 0/1 Rating)) + +user: |- + [Reference Answer] + {expected_answer} + + [Model Answer] + {generation} + + [Question] + {question} + + [Task] + Rate the model's answer based on its alignment with the reference answer, focusing on accuracy and relevance to the reference provided. Please be critical on the details. + + Criteria: Assess if the model's response mirrors the reference in terms of content, accuracy, and relevance. + + The answer is INCORRECT if: + - The answer is refusing to give concrete results, providing something like 'cannot decide' + - The answer is wrong, providing incorrect or irrelevant information compared to the reference + + The answer is CORRECT if: + - The answer is correct, capturing or covering the meaning from the reference + + Your response should be formatted as follows: + Reasoning: (Provide a concise explanation of your rating, comparing the reference answer with the model's response. "The reference answer is [XXX], while the model's answer is [YYY]. I think ...") + Judgement: [Yes or No] diff --git a/tests/gpu-tests/test_eval.py b/tests/gpu-tests/test_eval.py index aa5df51035..31c8f2cccf 100644 --- a/tests/gpu-tests/test_eval.py +++ b/tests/gpu-tests/test_eval.py @@ -45,6 +45,8 @@ "mmau-pro", "asr-leaderboard", "aalcr", # Has tokenization mismatch issues + "audiobench", + "librispeech-pc", } diff --git a/tests/test_datasets.py b/tests/test_datasets.py index 39d4b0398a..86fd152df2 100644 --- a/tests/test_datasets.py +++ b/tests/test_datasets.py @@ -57,6 +57,8 @@ ("college_math", ["test"]), ("comp-math-24-25", ["test"]), ("mmau-pro", ["test"]), + ("audiobench", ["test"]), + ("librispeech-pc", ["test"]), ] From 7965d5114adbc8ba38d8a6ebf1d35ce9aba4ab8d Mon Sep 17 00:00:00 2001 From: George Armstrong Date: Tue, 16 Dec 2025 14:13:05 -0800 Subject: [PATCH 55/88] Schema overrides for tool-calling (#1118) Signed-off-by: George Armstrong Signed-off-by: Cheng-Ping Hsieh --- nemo_skills/inference/generate.py | 20 ++++ nemo_skills/inference/model/__init__.py | 2 + nemo_skills/inference/model/tool_call.py | 22 +++- nemo_skills/mcp/adapters.py | 129 ++++++++++++++++++++++- tests/test_mcp_clients.py | 49 +++++++++ 5 files changed, 212 insertions(+), 10 deletions(-) diff --git a/nemo_skills/inference/generate.py b/nemo_skills/inference/generate.py index aae36c7351..136375db46 100644 --- a/nemo_skills/inference/generate.py +++ b/nemo_skills/inference/generate.py @@ -177,6 +177,25 @@ class GenerateSolutionsConfig: # - Set an ExampleTool server-only arg: # ++tool_overrides.ExampleTool.foo_argument='[TEST] ' tool_overrides: dict | None = field(default_factory=dict) + # + # Schema overrides allow customizing tool schemas shown to the model. + # Dict keyed by provider class name (like tool_overrides), then tool name. + # Format: ProviderClassName -> tool_name -> (name, description, parameters) + # + # Example YAML configuration (config.yaml): + # schema_overrides: + # PythonTool: + # stateful_python_code_exec: + # name: "python_executor" + # description: "Evaluate Python code interactively" + # parameters: + # code: + # name: "script" + # description: "Python code to execute" + # + # To use this config with Hydra, launch your script with: + # --config-path /path/to/configs --config-name config + schema_overrides: dict | None = field(default_factory=dict) # if True, will move full generation to _full_generation key and keep cfg.generation_key without thinking tokens # IMPORTANT: do not set this for non-reasoning models as it will make the generations empty! @@ -387,6 +406,7 @@ def setup_llm(self): **self.cfg.server, tool_modules=self.cfg.tool_modules, tool_overrides=self.cfg.tool_overrides, + schema_overrides=self.cfg.schema_overrides, tokenizer=self.tokenizer, additional_config={"sandbox": self.cfg.sandbox}, ) diff --git a/nemo_skills/inference/model/__init__.py b/nemo_skills/inference/model/__init__.py index bd2e246499..164d92fcc8 100644 --- a/nemo_skills/inference/model/__init__.py +++ b/nemo_skills/inference/model/__init__.py @@ -122,6 +122,7 @@ def get_tool_calling_model( additional_config=None, tool_modules: list[str] | None = None, tool_overrides: dict | None = None, + schema_overrides: dict | None = None, **kwargs, ): if isinstance(model, str): @@ -131,6 +132,7 @@ def get_tool_calling_model( tool_modules=tool_modules, tool_overrides=tool_overrides, additional_config=additional_config, + schema_overrides=schema_overrides, ) diff --git a/nemo_skills/inference/model/tool_call.py b/nemo_skills/inference/model/tool_call.py index ffbd1a9921..2891389bd8 100644 --- a/nemo_skills/inference/model/tool_call.py +++ b/nemo_skills/inference/model/tool_call.py @@ -24,6 +24,8 @@ format_tool_list_by_endpoint_type, format_tool_response_by_endpoint_type, get_tool_details_by_endpoint_type, + load_schema_overrides, + remap_tool_call, ) from nemo_skills.mcp.tool_manager import ToolManager from nemo_skills.utils import get_logger_name @@ -46,13 +48,11 @@ def __init__( tool_modules: list[str] | None = None, tool_overrides: dict | None = None, additional_config: dict | None = None, + schema_overrides: dict | None = None, ): self.model = model additional_config = additional_config or {} - self.tool_manager = None - - # Module-based tool loading only assert tool_modules, "tool_modules must be provided for tool calling" self.tool_manager = ToolManager( module_specs=tool_modules, @@ -60,6 +60,9 @@ def __init__( context=additional_config, ) + self.schema_overrides = load_schema_overrides(schema_overrides) + self.schema_mappings = {} # Built when tools are listed + async def _execute_tool_call(self, tool_call, request_id: str, endpoint_type: EndpointType): ## TODO(sanyamk): The correct key format needs to be cohesive with other formatters. tool_name, tool_args = get_tool_details_by_endpoint_type(tool_call, endpoint_type) @@ -67,6 +70,7 @@ async def _execute_tool_call(self, tool_call, request_id: str, endpoint_type: En ## # TODO(sanyamk): Not all tool arguments might necessarily be in JSON format. # Kept here to handle errors for now. + try: tool_args = json.loads(tool_args) except json.decoder.JSONDecodeError as e: @@ -75,9 +79,14 @@ async def _execute_tool_call(self, tool_call, request_id: str, endpoint_type: En return {"error": "Tool argument parsing failed."} ## TODO(sanyamk): Only exceptions related to tool execution here, all others must fail. + # Remap model's tool name/args back to original schema + original_tool_name, tool_args = remap_tool_call(tool_name, tool_args, self.schema_mappings) + try: # Allow providers to specify extra_args behavior internally if needed in the future - result = await self.tool_manager.execute_tool(tool_name, tool_args, extra_args={"request_id": request_id}) + result = await self.tool_manager.execute_tool( + original_tool_name, tool_args, extra_args={"request_id": request_id} + ) except Exception as e: LOG.exception(e) return {"error": "Tool execution failed."} @@ -109,7 +118,9 @@ async def generate_async( # This assumes that the available tools do not change during the generation. raw_tools = await self.tool_manager.list_all_tools(use_cache=True) - tools = format_tool_list_by_endpoint_type(raw_tools, endpoint_type) + tools, self.schema_mappings = format_tool_list_by_endpoint_type( + raw_tools, endpoint_type, schema_overrides=self.schema_overrides + ) LOG.info("Available Tools: %s", tools) result_steps = defaultdict(list) @@ -156,5 +167,6 @@ async def generate_async( result_steps["num_generated_tokens"] = sum(result_steps["num_generated_tokens"]) result_steps["num_tool_calls"] = sum(result_steps["num_tool_calls"]) result_steps["conversation"] = conversation + result_steps["tools"] = tools # Schema sent to model (with overrides applied) return result_steps diff --git a/nemo_skills/mcp/adapters.py b/nemo_skills/mcp/adapters.py index a927000403..2619c783f2 100644 --- a/nemo_skills/mcp/adapters.py +++ b/nemo_skills/mcp/adapters.py @@ -12,10 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +import copy import json from abc import ABC, abstractmethod +from typing import Any, Dict from litellm.types.utils import ChatCompletionMessageToolCall +from omegaconf import DictConfig, OmegaConf from nemo_skills.inference.model.base import EndpointType @@ -48,9 +51,123 @@ def format(self, tool_call: ChatCompletionMessageToolCall, result: dict) -> dict # ============================== -def format_tool_list_by_endpoint_type(tools, endpoint_type: EndpointType): +def load_schema_overrides(schema_overrides: dict | None) -> Dict[str, Dict[str, Dict[str, Any]]]: + """ + Normalize schema overrides dict from Hydra/OmegaConf. + + Args: + schema_overrides: Dict keyed by provider class name, then tool name, or None. + Format: ProviderClassName -> tool_name -> (name, description, parameters) + + Returns: + Normalized dict ready for use with format_tool_list_by_endpoint_type + """ + if schema_overrides is None: + return {} + + if isinstance(schema_overrides, DictConfig): + schema_overrides = OmegaConf.to_container(schema_overrides, resolve=True) + + if not isinstance(schema_overrides, dict): + raise ValueError(f"schema_overrides must be dict or None, got {type(schema_overrides)}") + + normalized = {} + for provider_class, provider_overrides in schema_overrides.items(): + if not isinstance(provider_overrides, dict): + raise ValueError(f"Override for provider '{provider_class}' must be a dict") + + normalized[provider_class] = {} + for tool_name, cfg in provider_overrides.items(): + if not isinstance(cfg, dict): + raise ValueError(f"Override for tool '{tool_name}' in '{provider_class}' must be a dict") + normalized[provider_class][tool_name] = { + "name": cfg.get("name"), + "description": cfg.get("description"), + "parameters": cfg.get("parameters"), + } + + return normalized + + +def apply_schema_overrides( + tool: Dict[str, Any], override_config: Dict[str, Any] | None +) -> tuple[Dict[str, Any], Dict[str, str]]: + """Apply schema overrides to a tool. Returns (transformed_tool, {new_param: orig_param}).""" + if not override_config: + return tool, {} + + transformed = copy.deepcopy(tool) + for key in ("name", "description"): + if override_config.get(key) is not None: + transformed[key] = override_config[key] + + param_overrides = override_config.get("parameters", {}) + if not param_overrides: + return transformed, {} + + schema = transformed.get("input_schema", {}) + props, required = schema.get("properties", {}), set(schema.get("required", [])) + + for name, cfg in param_overrides.items(): + if name not in props: + raise ValueError(f"Parameter '{name}' not in schema") + if not isinstance(cfg, dict): + raise ValueError(f"Override for '{name}' must be a dict") + + new_props, new_required, mapping = {}, [], {} + for orig, param in props.items(): + ovr = param_overrides.get(orig, {}) + new = ovr.get("name", orig) + new_props[new] = {**param, **{k: v for k, v in ovr.items() if k != "name"}} + if new != orig: + mapping[new] = orig + if orig in required: + new_required.append(new) + + transformed["input_schema"] = {**schema, "properties": new_props, "required": new_required} + return transformed, mapping + + +def remap_tool_call(tool_name: str, args: dict, mappings: dict) -> tuple[str, dict]: + """Remap a tool call from model names back to original tool schema names.""" + original_tool = mappings.get("tool_names", {}).get(tool_name, tool_name) + param_mapping = mappings.get("parameters", {}).get(tool_name, {}) + original_args = {param_mapping.get(k, k): v for k, v in args.items()} + return original_tool, original_args + + +def format_tool_list_by_endpoint_type( + tools, endpoint_type: EndpointType, schema_overrides: Dict[str, Dict[str, Dict[str, Any]]] | None = None +) -> tuple[list[Dict[str, Any]], Dict[str, Any]]: + """ + Format tool list for the given endpoint type, applying schema overrides. + + Returns: + Tuple of (formatted_tools, mappings_dict) where mappings_dict has: + - "tool_names": {model_name: original_name} + - "parameters": {tool_name: {model_param: original_param}} + """ + schema_overrides = schema_overrides or {} + mappings = {"tool_names": {}, "parameters": {}} + transformed_tools = [] + + for tool in tools: + original_name = tool["name"] + provider = schema_overrides.get(tool.get("server")) or {} + override = provider.get(original_name) + + transformed, param_mapping = apply_schema_overrides(tool, override) + transformed_tools.append(transformed) + + new_name = transformed["name"] + if new_name != original_name: + mappings["tool_names"][new_name] = original_name + if param_mapping: + mappings["parameters"][new_name] = param_mapping + + # Format for endpoint type if endpoint_type == EndpointType.chat: - return [ + formatted = [ { "type": "function", "function": { @@ -59,10 +176,10 @@ def format_tool_list_by_endpoint_type(tools, endpoint_type: EndpointType): "parameters": t["input_schema"], }, } - for t in tools + for t in transformed_tools ] elif endpoint_type == EndpointType.responses: - return [ + formatted = [ { "type": "function", "name": t["name"], @@ -70,11 +187,13 @@ def format_tool_list_by_endpoint_type(tools, endpoint_type: EndpointType): "parameters": t["input_schema"], "strict": True, # Less vllm errors through structured output } - for t in tools + for t in transformed_tools ] else: raise ValueError(f"Unsupported completion type for tool list: {endpoint_type}") + return formatted, mappings + class OpenAICallInterpreter(ToolCallInterpreter): def parse(self, tool_call): diff --git a/tests/test_mcp_clients.py b/tests/test_mcp_clients.py index bf893e1e19..a26c6df277 100644 --- a/tests/test_mcp_clients.py +++ b/tests/test_mcp_clients.py @@ -556,3 +556,52 @@ async def __aexit__(self, exc_type, exc, tb): client = MCPStreamableHttpClient(base_url="https://example.com/mcp", enabled_tools=["only_t2"]) # not including t1 with pytest.raises(PermissionError): await client.call_tool("t1", {}) + + +@pytest.mark.asyncio +async def test_tool_manager_with_schema_overrides(): + """Test ToolManager integration with schema overrides.""" + from nemo_skills.inference.model.base import EndpointType + from nemo_skills.mcp.adapters import format_tool_list_by_endpoint_type, load_schema_overrides + + tm = ToolManager(module_specs=[f"{__name__}::DummyTool"], overrides={}, context={}) + tools = await tm.list_all_tools(use_cache=False) + + schema_overrides = { + "DummyTool": { + "execute": { + "name": "renamed_execute", + "parameters": {"code": {"name": "script"}}, # rename 'code' -> 'script' for model + } + } + } + loaded_overrides = load_schema_overrides(schema_overrides) + formatted_tools, mappings = format_tool_list_by_endpoint_type( + tools, EndpointType.chat, schema_overrides=loaded_overrides + ) + + renamed_tool = next((t for t in formatted_tools if t["function"]["name"] == "renamed_execute"), None) + assert renamed_tool is not None + assert "script" in renamed_tool["function"]["parameters"]["properties"] + assert "code" not in renamed_tool["function"]["parameters"]["properties"] + assert mappings["parameters"]["renamed_execute"] == {"script": "code"} + assert mappings["tool_names"]["renamed_execute"] == "execute" + + +def test_schema_override_nonexistent_param_fails(): + """Overriding a parameter that doesn't exist in the schema must fail early. + + This also covers the hidden-arg case: when hide_args removes a param from the + schema before overrides are applied, attempting to override that (now-missing) + param will trigger the same error. + """ + from nemo_skills.mcp.adapters import apply_schema_overrides + + tool = { + "name": "test", + "description": "Test", + "input_schema": {"type": "object", "properties": {"code": {"type": "string"}}, "required": []}, + } + # Try to override 'script' which doesn't exist (tool only has 'code') + with pytest.raises(ValueError, match="Parameter 'script' not in schema"): + apply_schema_overrides(tool, {"parameters": {"script": {"name": "renamed"}}}) From 6d5db21a5b44b4cc6436d49cddf391698ed629c4 Mon Sep 17 00:00:00 2001 From: George Armstrong Date: Tue, 16 Dec 2025 18:02:07 -0800 Subject: [PATCH 56/88] FIX tool call error handling and search tool errors (#1120) Signed-off-by: George Armstrong Signed-off-by: Cheng-Ping Hsieh --- nemo_skills/inference/model/tool_call.py | 5 +- nemo_skills/mcp/clients.py | 58 ++++++++++++++----- nemo_skills/mcp/servers/tavily_search_tool.py | 56 +++++++++++++++--- nemo_skills/mcp/tool_manager.py | 10 ++++ tests/test_mcp_clients.py | 8 +-- 5 files changed, 110 insertions(+), 27 deletions(-) diff --git a/nemo_skills/inference/model/tool_call.py b/nemo_skills/inference/model/tool_call.py index 2891389bd8..8d25cbf762 100644 --- a/nemo_skills/inference/model/tool_call.py +++ b/nemo_skills/inference/model/tool_call.py @@ -27,7 +27,7 @@ load_schema_overrides, remap_tool_call, ) -from nemo_skills.mcp.tool_manager import ToolManager +from nemo_skills.mcp.tool_manager import FatalToolError, ToolManager from nemo_skills.utils import get_logger_name from .base import BaseModel, EndpointType @@ -87,6 +87,9 @@ async def _execute_tool_call(self, tool_call, request_id: str, endpoint_type: En result = await self.tool_manager.execute_tool( original_tool_name, tool_args, extra_args={"request_id": request_id} ) + except FatalToolError: + # Fatal errors should propagate up and stop the process + raise except Exception as e: LOG.exception(e) return {"error": "Tool execution failed."} diff --git a/nemo_skills/mcp/clients.py b/nemo_skills/mcp/clients.py index 0b89f9e6dd..33b8cede46 100644 --- a/nemo_skills/mcp/clients.py +++ b/nemo_skills/mcp/clients.py @@ -14,6 +14,7 @@ import copy import functools import json +import logging import os from abc import abstractmethod from typing import Any, Callable, Dict, List @@ -22,6 +23,10 @@ from mcp.client.stdio import stdio_client from mcp.client.streamable_http import streamablehttp_client +from nemo_skills.utils import get_logger_name + +LOG = logging.getLogger(get_logger_name(__file__)) + def _process_hide_args(result, hide_args): if hide_args: @@ -101,6 +106,42 @@ def _sanitize_input_args_for_tool(args_dict, tool_name, hide_args): return {k: v for k, v in args_dict.items() if k not in hidden_keys} +def _extract_tool_result(result) -> Any: + """Extract a JSON-serializable result from an MCP CallToolResult. + + Handles various response formats: + - structuredContent: Returns directly if present + - content[].text: Parses as JSON or returns as string + - Fallback: Returns error dict to avoid returning raw CallToolResult objects + + This ensures the return value is always JSON-serializable. + """ + # Check if tool explicitly returned an error - return generic message to avoid leaking details + is_error = getattr(result, "isError", False) + if is_error: + return {"error": "Tool execution failed"} + + struct = getattr(result, "structuredContent", None) + if struct is not None: + return struct + # Fallback: try to parse first content item as JSON, else return text + content = getattr(result, "content", None) + if content: + first = content[0] + text = getattr(first, "text", None) + if isinstance(text, str): + try: + return json.loads(text) + except Exception: + return text + LOG.error("Unsupported content type in tool result: %s", content) + return {"error": "Unsupported content type returned from tool"} + # No content at all + # This could happen due to a tool failure (like hitting uncaught API limits) + LOG.error("No content in tool result. Full result: %s", result) + return {"error": "No content returned from tool"} + + def _wrap_call_tool_output_formatter(method): async def wrapped(self, *args, **kwargs): # Normalize to keyword-style and sanitize before delegating. @@ -355,7 +396,7 @@ async def call_tool(self, tool: str, args: dict) -> Any: async with ClientSession(read_stream, write_stream) as session: await session.initialize() result = await session.call_tool(tool, arguments=args) - return struct if (struct := result.structuredContent) is not None else result + return _extract_tool_result(result) class MCPStdioClient(MCPClient): @@ -415,17 +456,4 @@ async def call_tool(self, tool: str, args: dict) -> Any: async with ClientSession(read_stream, write_stream) as session: await session.initialize() result = await session.call_tool(tool, arguments=args) - struct = getattr(result, "structuredContent", None) - if struct is not None: - return struct - # Fallback: try to parse first content item as JSON, else return text - content = getattr(result, "content", None) - if content: - first = content[0] - text = getattr(first, "text", None) - if isinstance(text, str): - try: - return json.loads(text) - except Exception: - return text - return result + return _extract_tool_result(result) diff --git a/nemo_skills/mcp/servers/tavily_search_tool.py b/nemo_skills/mcp/servers/tavily_search_tool.py index f72f6761dd..445d5d53fd 100644 --- a/nemo_skills/mcp/servers/tavily_search_tool.py +++ b/nemo_skills/mcp/servers/tavily_search_tool.py @@ -23,6 +23,7 @@ from mcp.server.fastmcp import FastMCP from pydantic import Field +from nemo_skills.mcp.tool_manager import FatalToolError from nemo_skills.mcp.tool_providers import MCPClientTool logger = logging.getLogger(__name__) @@ -42,6 +43,17 @@ class ExecutionResult: EXCLUDE_DOMAINS: list[str] | None = None MAX_NUM_RESULTS: int = 20 +STATUS_CODE_ERRORS = { + 429: "Search rate limit exceeded", + 500: "Search request failed due to server error", + 502: "Search request failed due to bad gateway", + 503: "Search request failed due to service unavailable", + 504: "Search request failed due to gateway timeout", +} + +# These errors should stop the process - no point continuing with bad credentials +FATAL_STATUS_CODES = {401, 403} + ## See docs https://docs.tavily.com/documentation/api-reference/endpoint/search ## There is also a hosted MCP that can be used instead of this tool: https://github.com/tavily-ai/tavily-mcp?tab=readme-ov-file#remote-mcp-server @@ -60,8 +72,12 @@ async def answer( """Search the web for a query""" api_url = "https://api.tavily.com/search" - assert answer_type in ["answer", "results"], "Invalid answer type. Choose 'answer' or 'results'." - assert num_results <= MAX_NUM_RESULTS, f"Number of results must be less than or equal to {MAX_NUM_RESULTS}." + + # Validate inputs + if answer_type not in ["answer", "results"]: + return {"error": "Invalid answer type. Choose 'answer' or 'results'."} + if num_results > MAX_NUM_RESULTS: + return {"error": f"Number of results must be less than or equal to {MAX_NUM_RESULTS}."} headers = { "Authorization": f"Bearer {TAVILY_API_KEY}", @@ -78,12 +94,33 @@ async def answer( "exclude_domains": exclude_domains, } - async with httpx.AsyncClient() as client: - response = await client.post(api_url, headers=headers, json=payload) - if response.status_code != 200: - return {"error": response.json()["error"]} + try: + async with httpx.AsyncClient() as client: + response = await client.post(api_url, headers=headers, json=payload) + except httpx.TimeoutException: + return {"error": "Search request timed out"} + except httpx.RequestError: + return {"error": "Search request failed due to network error"} + + # Handle non-200 responses + if response.status_code in FATAL_STATUS_CODES: + return {"error": "Search authentication failed", "fatal": True} + if response.status_code != 200: + error_msg = STATUS_CODE_ERRORS.get( + response.status_code, f"Search request failed with status {response.status_code}" + ) + return {"error": error_msg} - result = response.json()[answer_type] + # Parse response + try: + data = response.json() + except json.JSONDecodeError: + return {"error": "Search returned invalid response"} + + # Extract result + result = data.get(answer_type) + if result is None: + return {"error": "Search response is missing required field"} return result @@ -135,6 +172,11 @@ async def execute(self, tool_name: str, arguments: dict[str, Any], extra_args: d if key in self._config: merged_extra[key] = self._config[key] result = await self._client.call_tool(tool=tool_name, args=arguments, extra_args=merged_extra) + + # Check for fatal errors that should stop the process + if isinstance(result, dict) and result.get("fatal"): + raise FatalToolError(result.get("error", "Fatal tool error")) + return result diff --git a/nemo_skills/mcp/tool_manager.py b/nemo_skills/mcp/tool_manager.py index 2d98cdd772..8e28619aae 100644 --- a/nemo_skills/mcp/tool_manager.py +++ b/nemo_skills/mcp/tool_manager.py @@ -31,6 +31,16 @@ from nemo_skills.mcp.utils import locate +class FatalToolError(Exception): + """Exception for fatal tool errors that should stop the entire process. + + Use this for unrecoverable errors like authentication failures where + continuing would be pointless (e.g., invalid API keys). + """ + + pass + + class Tool(ABC): """Abstract base for module-based tools. diff --git a/tests/test_mcp_clients.py b/tests/test_mcp_clients.py index a26c6df277..56b89976da 100644 --- a/tests/test_mcp_clients.py +++ b/tests/test_mcp_clients.py @@ -482,8 +482,8 @@ async def list_tools(self): async def call_tool(self, tool, arguments): if tool == "t1": return ResultObj({"ok": True}) - # No structured content -> client should return raw object - return types.SimpleNamespace(structuredContent=None, raw=True, tool=tool, arguments=arguments) + # No structured content and no text content -> client should return error dict + return types.SimpleNamespace(structuredContent=None, content=None) class FakeHttpCtx: async def __aenter__(self): @@ -505,9 +505,9 @@ async def __aexit__(self, exc_type, exc, tb): out1 = await client.call_tool("t1", {}) assert out1 == {"ok": True} - # structured content absent -> return raw + # structured content absent and no text content -> return error dict (not raw object) out2 = await client.call_tool("t2", {"x": 1}) - assert getattr(out2, "raw", False) is True and getattr(out2, "tool", "") == "t2" + assert out2 == {"error": "No content returned from tool"} @pytest.mark.asyncio From 07c23baac97c417d011af766e851c13f1c843020 Mon Sep 17 00:00:00 2001 From: George Armstrong Date: Tue, 16 Dec 2025 18:17:43 -0800 Subject: [PATCH 57/88] Use run.Script for generate pipeline (#1052) Signed-off-by: George Armstrong Signed-off-by: Cheng-Ping Hsieh --- .github/workflows/gpu_tests.yml | 10 +- nemo_skills/pipeline/generate.py | 452 +++++++++++------- nemo_skills/pipeline/nemo_evaluator.py | 217 ++++----- nemo_skills/pipeline/utils/__init__.py | 2 + nemo_skills/pipeline/utils/declarative.py | 551 ++++++++++++---------- nemo_skills/pipeline/utils/generation.py | 107 ++++- nemo_skills/pipeline/utils/scripts.py | 419 ++++++++++++++++ tests/gpu-tests/test_eval.py | 1 + tests/test_declarative_pipeline.py | 274 +++++------ tests/test_generation.py | 40 +- tests/test_nemo_evaluator_pipeline.py | 41 +- 11 files changed, 1396 insertions(+), 718 deletions(-) create mode 100644 nemo_skills/pipeline/utils/scripts.py diff --git a/.github/workflows/gpu_tests.yml b/.github/workflows/gpu_tests.yml index 16f77633a8..a500fc59b2 100644 --- a/.github/workflows/gpu_tests.yml +++ b/.github/workflows/gpu_tests.yml @@ -52,7 +52,15 @@ jobs: cd ${{ github.run_id }} nvidia-smi set -o pipefail # this will make sure next line returns non-0 exit code if tests fail - ./tests/gpu-tests/run_qwen.sh + # Run heartbeat in background, capture its PID, and ensure cleanup + (while true; do sleep 60; echo "[HEARTBEAT] $(date '+%Y-%m-%d %H:%M:%S') - still running..."; done) & + HEARTBEAT_PID=$! + # Run tests and capture exit code + EXIT_CODE=0 + ./tests/gpu-tests/run_qwen.sh || EXIT_CODE=$? + # Kill heartbeat and exit with test result + kill $HEARTBEAT_PID 2>/dev/null || true + exit $EXIT_CODE - name: Cleanup if: always() run: | diff --git a/nemo_skills/pipeline/generate.py b/nemo_skills/pipeline/generate.py index f33796d05c..90ec987bca 100644 --- a/nemo_skills/pipeline/generate.py +++ b/nemo_skills/pipeline/generate.py @@ -14,7 +14,7 @@ import importlib import logging import os -from typing import Callable, Dict, List, Optional +from typing import Dict, List, Optional import typer @@ -23,14 +23,17 @@ from nemo_skills.inference import GENERATION_MODULE_MAP, GenerationType from nemo_skills.pipeline.app import app, typer_unpacker from nemo_skills.pipeline.utils.cluster import parse_kwargs -from nemo_skills.pipeline.utils.commands import sandbox_command from nemo_skills.pipeline.utils.declarative import ( Command, CommandGroup, HardwareConfig, Pipeline, ) -from nemo_skills.pipeline.utils.server import get_free_port +from nemo_skills.pipeline.utils.scripts import ( + GenerationClientScript, + SandboxScript, + ServerScript, +) from nemo_skills.utils import ( compute_chunk_ids, get_logger_name, @@ -44,118 +47,160 @@ # TODO: add num_jobs here for consistency with eval? -def _create_commandgroup_from_config( - generation_cmd: str, - server_config: Optional[Dict], - with_sandbox: bool, - sandbox_port: Optional[int], +def _create_job_unified( + models: List[str], + server_configs: List[Optional[Dict]], + generation_params: Dict, cluster_config: Dict, installation_command: Optional[str], - get_server_command_fn: Callable, + with_sandbox: bool, partition: Optional[str], keep_mounts_for_sandbox: bool, task_name: str, log_dir: str, sbatch_kwargs: Optional[Dict] = None, sandbox_env_overrides: Optional[List[str]] = None, -) -> CommandGroup: - """Create a CommandGroup from server_config. - - Component ordering: - 1. Server (if server_config provided) - 2. Client command - 3. Sandbox (if with_sandbox=True) +) -> List[CommandGroup]: """ + Create CommandGroups for n models (unified for n=1 and n>1). + + Structure: + - Group 0: Model 0 server + client + (optional sandbox) + - Group 1: Model 1 server (if n>1) + - Group N: Model N server (if n>1) + + For n=1, returns a single-element list. The Pipeline automatically + optimizes single-group lists to efficient single-group jobs. + + Args: + models: List of model paths + server_configs: List of server configurations (one per model, None if not hosting) + generation_params: Dict of parameters for generation (output_dir, etc.) + cluster_config: Cluster configuration + installation_command: Installation command to run before client + with_sandbox: Whether to include sandbox + partition: Slurm partition + keep_mounts_for_sandbox: Whether to keep mounts for sandbox + task_name: Name for the task + log_dir: Directory for logs + sbatch_kwargs: Additional sbatch kwargs + + Returns: + List of CommandGroup objects (one per het group) + """ + num_models = len(models) + groups = [] + server_scripts = [] # Track server Script objects for cross-component references + + for model_idx, (model_path, server_config) in enumerate(zip(models, server_configs)): + components = [] + server_script = None + + # Track GPU/node requirements for this group (from server config) + group_gpus = 0 + group_nodes = 1 + + # 1. Add server if needed + if server_config is not None and int(server_config.get("num_gpus", 0)) > 0: + server_type = server_config["server_type"] + server_container = server_config.get("container") or cluster_config["containers"][server_type] - components = [] + # Create ServerScript + server_script = ServerScript( + server_type=server_type, + model_path=server_config["model_path"], + cluster_config=cluster_config, + num_gpus=server_config["num_gpus"], + num_nodes=server_config["num_nodes"], + server_args=server_config.get("server_args", ""), + server_entrypoint=server_config.get("server_entrypoint"), + port=server_config.get("server_port"), + allocate_port=(server_config.get("server_port") is None), + ) - # 1. Add server if server_config is provided - if server_config is not None and int(server_config["num_gpus"]) > 0: - server_type = server_config["server_type"] - # Get container from server_config if provided, otherwise fall back to cluster config - if "container" in server_config: - server_container = server_config.pop("container") + # Set group GPU/node requirements from server config + group_gpus = server_config["num_gpus"] + group_nodes = server_config["num_nodes"] + + server_cmd = Command( + script=server_script, + container=server_container, + name=f"{task_name}_model_{model_idx}_server" if num_models > 1 else f"{task_name}_server", + ) + components.append(server_cmd) + server_scripts.append(server_script) else: - server_container = cluster_config["containers"][server_type] + # No server for this model (pre-hosted) + server_scripts.append(None) + + # 2. Group 0 gets the client and sandbox + if model_idx == 0: + # Create sandbox script (if with_sandbox) + sandbox_script = None + if with_sandbox: + sandbox_script = SandboxScript( + cluster_config=cluster_config, + keep_mounts=keep_mounts_for_sandbox, + allocate_port=True, # Always allocate port for sandbox + env_overrides=sandbox_env_overrides, + ) + + sandbox_cmd = Command( + script=sandbox_script, + container=cluster_config["containers"]["sandbox"], + name=f"{task_name}_sandbox", + ) + components.append(sandbox_cmd) + + # Create client script with cross-component references to all servers + client_script = GenerationClientScript( + output_dir=generation_params["output_dir"], + input_file=generation_params.get("input_file"), + input_dir=generation_params.get("input_dir"), + extra_arguments=generation_params.get("extra_arguments", ""), + random_seed=generation_params.get("random_seed"), + chunk_id=generation_params.get("chunk_id"), + num_chunks=generation_params.get("num_chunks"), + preprocess_cmd=generation_params.get("preprocess_cmd"), + postprocess_cmd=generation_params.get("postprocess_cmd"), + wandb_parameters=generation_params.get("wandb_parameters"), + with_sandbox=with_sandbox, + script=generation_params.get("script", "nemo_skills.inference.generate"), + # Multi-server support (works for single and multi-model) + servers=server_scripts if server_scripts else None, + server_addresses_prehosted=generation_params.get("server_addresses_prehosted"), + model_names=generation_params.get("model_names"), + server_types=generation_params.get("server_types"), + sandbox=sandbox_script, + installation_command=installation_command, + ) - # Call server command builder directly with cluster_config - cmd, num_tasks = get_server_command_fn(**server_config, cluster_config=cluster_config) + client_cmd = Command( + script=client_script, + container=cluster_config["containers"]["nemo-skills"], + name=f"{task_name}", + ) + components.append(client_cmd) - # Create metadata dict - metadata = { - "num_tasks": num_tasks, - "gpus": server_config["num_gpus"], - "nodes": server_config["num_nodes"], - "log_prefix": "server", - } + # Only create group if it has components (skip empty groups for pre-hosted models) + if components: + group_tasks = server_script.num_tasks if (server_config and server_script) else 1 - server_cmd = Command( - command=cmd, - container=server_container, - gpus=server_config["num_gpus"], - nodes=server_config["num_nodes"], - name=task_name, - metadata=metadata, - ) - components.append(server_cmd) - - # 2. Add main generation command - # Note: General cluster config env vars are automatically added by get_env_variables() in get_executor() - client_env = {} - if with_sandbox and sandbox_port is not None: - client_env["NEMO_SKILLS_SANDBOX_PORT"] = str(sandbox_port) - - client_cmd = Command( - command=generation_cmd, - container=cluster_config["containers"]["nemo-skills"], - name=task_name, - installation_command=installation_command, - metadata={ - "log_prefix": "main", - "environment": client_env, - }, - ) - components.append(client_cmd) - - # 3. Add sandbox if requested - if with_sandbox: - # Call sandbox command builder directly with cluster_config - cmd, metadata = sandbox_command(cluster_config=cluster_config, port=sandbox_port) - metadata["log_prefix"] = "sandbox" - - # Apply user-specified environment overrides for the sandbox - if sandbox_env_overrides: - sandbox_env = metadata.get("environment", {}) - for override in sandbox_env_overrides: - key, value = override.split("=", 1) - sandbox_env[key] = value - metadata["environment"] = sandbox_env - - sandbox_cmd = Command( - command=cmd, - container=cluster_config["containers"]["sandbox"], - name=task_name, - metadata=metadata, - ) + group = CommandGroup( + commands=components, + hardware=HardwareConfig( + partition=partition, + num_gpus=group_gpus, + num_nodes=group_nodes, + num_tasks=group_tasks, + sbatch_kwargs=sbatch_kwargs, + ), + name=f"{task_name}_model_{model_idx}_group" if num_models > 1 else task_name, + log_dir=log_dir, + ) + groups.append(group) - components.append(sandbox_cmd) - - # Find maximum GPUs/nodes needed by any component for the HardwareConfig - # The job-level resource request must be the maximum across all components - max_gpus = max((comp.gpus or 0) for comp in components) - max_nodes = max((comp.nodes or 1) for comp in components) - - return CommandGroup( - commands=components, - hardware=HardwareConfig( - partition=partition, - num_gpus=max_gpus, - num_nodes=max_nodes, - sbatch_kwargs=sbatch_kwargs, - ), - name=task_name, - log_dir=log_dir, - ) + return groups @app.command(context_settings={"allow_extra_args": True, "ignore_unknown_options": True}) @@ -186,21 +231,45 @@ def generate( "If not specified, will use the registered generation module for the " "generation type (which is required in this case).", ), - model: str = typer.Option(None, help="Path to the model or model name in API"), - server_address: str = typer.Option( - None, help="Use ip:port for self-hosted models or the API url if using model providers" + model: List[str] = typer.Option( + None, + help="Path to the model(s). CLI: space-separated. Python API: string or list. " + "Single value broadcasts to all models for multi-model generation.", + ), + server_address: List[str] = typer.Option( + None, + help="Server address(es). CLI: space-separated. Python API: string or list. " + "Single value broadcasts to all models.", + ), + server_type: List[pipeline_utils.SupportedServers] = typer.Option( + ..., + help="Server type(s). CLI: space-separated. Python API: string or list. " + "Single value broadcasts to all models.", ), - server_type: pipeline_utils.SupportedServers = typer.Option(..., help="Type of server to use"), - server_gpus: int = typer.Option(None, help="Number of GPUs to use if hosting the model"), - server_nodes: int = typer.Option(1, help="Number of nodes required for hosting LLM server"), - server_args: str = typer.Option("", help="Any extra arguments to pass to the server"), - server_entrypoint: str = typer.Option( + server_gpus: List[int] = typer.Option( None, - help="Path to the entrypoint of the server. " - "If not specified, will use the default entrypoint for the server type.", + help="Number of GPUs per model. CLI: space-separated ints. Python API: int or list. " + "Single value broadcasts to all models.", ), - server_container: str = typer.Option( - None, help="Override container image for the hosted server (if server_gpus is set)" + server_nodes: List[int] = typer.Option( + [1], + help="Number of nodes per model. CLI: space-separated ints. Python API: int or list. " + "Single value broadcasts to all models.", + ), + server_args: List[str] = typer.Option( + [""], + help="Server arguments per model. CLI: space-separated. Python API: string or list. " + "Single value broadcasts to all models.", + ), + server_entrypoint: List[str] = typer.Option( + None, + help="Server entrypoint(s). CLI: space-separated. Python API: string or list. " + "Single value broadcasts to all models.", + ), + server_container: List[str] = typer.Option( + None, + help="Container image(s). CLI: space-separated. Python API: string or list. " + "Single value broadcasts to all models.", ), dependent_jobs: int = typer.Option(0, help="Specify this to launch that number of dependent jobs"), mount_paths: str = typer.Option(None, help="Comma separated list of paths to mount on the remote machine"), @@ -296,7 +365,18 @@ def generate( None, help="Internal option to specify task dependencies.", hidden=True ), ): - """Generate LLM completions for a given input file. + """Generate LLM completions for single or multiple models. + + Supports both single-model and multi-model generation through a unified interface. + + Parameter Types: + Multi-model parameters (model, server_*, etc.) use List[T] type hints for Typer CLI + compatibility, but accept both scalars and lists when called from Python: + - CLI: --model m1 m2 (space-separated) → Typer converts to ["m1", "m2"] + - Python API: model="m1" or model=["m1", "m2"] → Both work (normalized internally) + - Single values broadcast to all models: server_gpus=8 → [8, 8, 8] for 3 models + + Multi-model usage requires either --generation-type or --generation-module. Run `python -m nemo_skills.inference.generate --help` for other supported arguments (need to be prefixed with ++, since we use Hydra for that script). @@ -306,10 +386,42 @@ def generate( LOG.info("Starting generation job") LOG.info("Extra arguments that will be passed to the underlying script: %s", extra_arguments) - try: - server_type = server_type.value - except AttributeError: - pass + # Normalize model configuration to list + models_list = pipeline_utils.normalize_models_config(model) + num_models = len(models_list) + + LOG.info(f"Number of models: {num_models}") + for model_idx, model_name in enumerate(models_list): + LOG.info(f" Model {model_idx}: {model_name}") + + # Convert server_type enum values to strings + def convert_server_type_to_string(server_type): + return server_type.value if hasattr(server_type, "value") else server_type + + if isinstance(server_type, list): + server_type = [convert_server_type_to_string(st) for st in server_type] + else: + server_type = convert_server_type_to_string(server_type) + + # Normalize all server parameters to per-model lists + server_types_list = pipeline_utils.normalize_parameter(server_type, num_models, "server_type") + server_gpus_list = pipeline_utils.normalize_parameter(server_gpus, num_models, "server_gpus") + server_nodes_list = pipeline_utils.normalize_parameter(server_nodes, num_models, "server_nodes") + server_args_list = pipeline_utils.normalize_parameter(server_args, num_models, "server_args") + server_entrypoints_list = pipeline_utils.normalize_parameter(server_entrypoint, num_models, "server_entrypoint") + server_containers_list = pipeline_utils.normalize_parameter(server_container, num_models, "server_container") + + if server_address is not None: + server_addresses_list = pipeline_utils.normalize_parameter(server_address, num_models, "server_address") + else: + server_addresses_list = [None] * num_models + + # Validate multi-model requirements + if num_models > 1: + if generation_type is None and generation_module is None: + raise ValueError( + "Multi-model generation requires either --generation-type or --generation-module to be specified" + ) if log_samples: wandb_parameters = { @@ -325,8 +437,6 @@ def generate( else: wandb_parameters = None - get_random_port = pipeline_utils.should_get_random_port(server_gpus, exclusive) - if random_seeds and num_random_seeds: raise ValueError("Cannot specify both random_seeds and num_random_seeds") if num_random_seeds: @@ -355,8 +465,6 @@ def generate( check_mounted_paths=check_mounted_paths, ) - original_server_address = server_address - if generation_module is not None and generation_type is not None: raise ValueError("Cannot specify both generation_module and generation_type. ") if generation_module is None: @@ -407,36 +515,36 @@ def generate( chunk_id=None, ) for chunk_id in chunk_ids: - # Configure client (same as before) - server_config, server_address, extra_arguments = pipeline_utils.configure_client( - model=model, - server_type=server_type, - server_address=original_server_address, - server_gpus=server_gpus, - server_nodes=server_nodes, - server_args=server_args, - server_entrypoint=server_entrypoint, - server_container=server_container, - extra_arguments=extra_arguments_original, - get_random_port=get_random_port, - ) + # Configure clients for each model + server_configs = [] + server_addresses_resolved = [] + # For single model: configure_client returns extra_args with server config appended + # For multi-model: use original extra_args (server config added as lists in get_generation_cmd) + extra_arguments = extra_arguments_original + + for model_idx in range(num_models): + get_random_port_for_server = pipeline_utils.should_get_random_port( + server_gpus_list[model_idx], exclusive + ) - # Build generation command (same as before) - cmd = pipeline_utils.get_generation_cmd( - input_file=input_file, - input_dir=input_dir, - random_seed=seed, - output_dir=output_dir, - extra_arguments=extra_arguments, - chunk_id=chunk_id, - num_chunks=num_chunks, - preprocess_cmd=preprocess_cmd, - postprocess_cmd=postprocess_cmd, - wandb_parameters=wandb_parameters if seed_idx == 0 else None, - script=generation_module, - with_sandbox=with_sandbox, - ) - cmd = pipeline_utils.wrap_python_path(cmd=cmd) + srv_config, srv_address, srv_extra_args = pipeline_utils.configure_client( + model=models_list[model_idx], + server_type=server_types_list[model_idx], + server_address=server_addresses_list[model_idx], + server_gpus=server_gpus_list[model_idx], + server_nodes=server_nodes_list[model_idx], + server_args=server_args_list[model_idx], + server_entrypoint=server_entrypoints_list[model_idx], + server_container=server_containers_list[model_idx], + extra_arguments=extra_arguments_original if model_idx == 0 else "", + get_random_port=get_random_port_for_server, + ) + server_configs.append(srv_config) + server_addresses_resolved.append(srv_address) + + # For single model, capture the extra_args with server config from configure_client + if model_idx == 0 and num_models == 1: + extra_arguments = srv_extra_args # Base task name (shared across all dependent jobs in the chain) task_name = f"{expname}-rs{seed}" if seed is not None else expname @@ -448,22 +556,35 @@ def generate( prev_job = None for dep_idx in range(dependent_jobs + 1): - # Allocate sandbox port if needed - # This must be done BEFORE creating CommandGroup so client knows the port - if with_sandbox: - current_sandbox_port = get_free_port(strategy="random") if get_random_port else 6000 - else: - current_sandbox_port = None + # Build generation parameters dict for Script + generation_params = { + "output_dir": output_dir, + "input_file": input_file, + "input_dir": input_dir, + "extra_arguments": extra_arguments, + "random_seed": seed, + "chunk_id": chunk_id, + "num_chunks": num_chunks, + "preprocess_cmd": preprocess_cmd, + "postprocess_cmd": postprocess_cmd, + "wandb_parameters": wandb_parameters if seed_idx == 0 else None, + "script": generation_module, + # Multi-model specific fields + "server_addresses_prehosted": server_addresses_resolved, + "model_names": models_list, + "server_types": server_types_list, + } - # Create CommandGroup for this task - cmd_group = _create_commandgroup_from_config( - generation_cmd=cmd, - server_config=server_config.copy() if server_config else None, - with_sandbox=with_sandbox, - sandbox_port=current_sandbox_port, + # Create CommandGroup(s) using Script objects + # For multi-model, this creates multiple CommandGroups (one per model + one for client) + # For single-model, this creates a single CommandGroup + job_groups = _create_job_unified( + models=models_list, + server_configs=[cfg.copy() if cfg else None for cfg in server_configs], + generation_params=generation_params, cluster_config=cluster_config, installation_command=installation_command, - get_server_command_fn=generation_task.get_server_command_fn(), + with_sandbox=with_sandbox, partition=partition, keep_mounts_for_sandbox=keep_mounts_for_sandbox, task_name=task_name, @@ -487,11 +608,16 @@ def generate( # Subsequent jobs in chain depend on previous job (use job object, not string) job_deps = [prev_job] + # For multi-group jobs, use "groups" key; for single-group, use "group" key job_spec = { "name": internal_job_name, - "group": cmd_group, "dependencies": job_deps, } + if len(job_groups) > 1: + job_spec["groups"] = job_groups + else: + job_spec["group"] = job_groups[0] + jobs.append(job_spec) prev_job = job_spec # Track for next iteration diff --git a/nemo_skills/pipeline/nemo_evaluator.py b/nemo_skills/pipeline/nemo_evaluator.py index 020162692a..39838737ed 100644 --- a/nemo_skills/pipeline/nemo_evaluator.py +++ b/nemo_skills/pipeline/nemo_evaluator.py @@ -89,7 +89,7 @@ import copy import logging -from dataclasses import dataclass +from dataclasses import dataclass, field from pathlib import Path from typing import Dict, List, Optional @@ -97,12 +97,12 @@ from nemo_evaluator_launcher.api import RunConfig from nemo_evaluator_launcher.common.helpers import get_eval_factory_command from nemo_evaluator_launcher.common.mapping import get_task_from_mapping, load_tasks_mapping -from omegaconf import DictConfig, OmegaConf +from omegaconf import OmegaConf import nemo_skills.pipeline.utils as pipeline_utils from nemo_skills.pipeline.app import app, typer_unpacker -from nemo_skills.pipeline.utils.commands import vllm_server_command from nemo_skills.pipeline.utils.declarative import Command, CommandGroup, HardwareConfig, Pipeline +from nemo_skills.pipeline.utils.scripts import BaseJobScript, ServerScript from nemo_skills.utils import get_logger_name, setup_logging LOG = logging.getLogger(get_logger_name(__file__)) @@ -289,8 +289,8 @@ def nemo_evaluator( expname=expname, idx=idx, task_name=task.name, - launcher_run_cfg=launcher_run_cfg, - task_cfg=task, + launcher_run_cfg=OmegaConf.to_container(launcher_run_cfg, resolve=True), + task_cfg=OmegaConf.to_container(task, resolve=True), task_definition=task_definition, base_output_root=base_output_root, eval_image=eval_image, @@ -443,10 +443,8 @@ def _create_serving_command_obj( idx: int, task_name: str, ) -> Command: - """Create a Command object for a hosted serving component (main or judge server). + """Create a `Command` backed by a `ServerScript` for a hosted serving component. - This function wraps vllm_server_command and standardizes container selection, - logging prefixes, and metadata for both main and judge servers. Args: cluster_config: Cluster configuration dictionary @@ -464,54 +462,53 @@ def _create_serving_command_obj( task_name: Task name for naming Returns: - Command object configured for the serving component + Command: A Command object whose `script` is a configured `ServerScript`. """ stype = (server_type or "vllm").lower() - sargs = args or "" if stype != "vllm": LOG.warning("Only vllm server_type is supported currently; got %s", stype) - cmd_str, meta = vllm_server_command( + server_script = ServerScript( + server_type=stype, + model_path=model or "", cluster_config=cluster_config, - model=model, # type: ignore[arg-type] + num_gpus=gpus, + num_nodes=nodes or 1, + server_args=args or "", + server_entrypoint=entrypoint, port=port, - server_type=stype, - gpus=gpus, - nodes=nodes, - args=sargs, - entrypoint=entrypoint, + allocate_port=port is None, ) - # Resolve container fallback when not explicitly provided + # Judge servers get a distinct log prefix for clarity + if is_judge: + server_script.log_prefix = "judge-server" + if not container: container = cluster_config["containers"][stype] - log_prefix = "judge-server" if is_judge else "server" name_role = "judge-server" if is_judge else "server" return Command( - command=cmd_str, + script=server_script, container=container, - gpus=gpus, - nodes=nodes or 1, name=f"{expname}-{name_role}-{idx}-{task_name}", - metadata={ - **meta, - "gpus": gpus, - "log_prefix": log_prefix, - }, ) @dataclass class _TaskCreationContext: - """Local helper to pass around the information about the task and easier logic sharing.""" + """Local helper to pass around the information about the task and easier logic sharing. + + Note: launcher_run_cfg and task_cfg are stored as plain dicts (not OmegaConf) to allow + serialization by nemo_run/fiddle. Convert back to DictConfig if OmegaConf operations are needed. + """ expname: str idx: int task_name: str - launcher_run_cfg: RunConfig - task_cfg: DictConfig + launcher_run_cfg: dict # Stored as plain dict for serialization compatibility + task_cfg: dict # Stored as plain dict for serialization compatibility task_definition: dict base_output_root: Optional[str] eval_image: str @@ -630,12 +627,7 @@ def _build_judge_server_if_needed(ctx: _TaskCreationContext) -> Optional[Command def _build_client_command( ctx: _TaskCreationContext, main_server_cmd: Optional[Command], judge_server_cmd: Optional[Command] ) -> Command: - """Build Command for evaluator client. - - The client command behavior depends on server hosting: - - If servers are co-hosted: Uses lambda factory to resolve runtime URLs via hostname_ref/meta_ref - - If using external servers: Uses static URLs from server_base_url/judge_server_base_url - - If no servers: Uses URLs from evaluator config or defaults + """Create the evaluator client `Command` using `EvaluatorClientScript`. Args: ctx: Task creation context with all configuration @@ -643,100 +635,26 @@ def _build_client_command( judge_server_cmd: Judge server Command if self-hosted, None otherwise Returns: - Command object for evaluator client + Command: A Command whose script builds the evaluator CLI at runtime """ - if ctx.hosting_server or ctx.hosting_judge: - # Co-hosted servers: Use lambda factory to resolve runtime URLs - # The lambda is evaluated at execution time when het_group_index is assigned - def _client_cmd_factory(): - waits: List[str] = [] - target_url: Optional[str] = None - judge_url: Optional[str] = None - # Build main server URL from runtime references - if ctx.hosting_server and main_server_cmd is not None: - server_host = main_server_cmd.hostname_ref() - server_port_val = main_server_cmd.meta_ref("port") - base_url = f"http://{server_host}:{server_port_val}" - waits.append(pipeline_utils.get_server_wait_cmd(f"{base_url}{ctx.server_health_path}")) - target_url = f"{base_url}{ctx.server_api_path}" - - # Build judge server URL from runtime references - if ctx.hosting_judge and judge_server_cmd is not None: - jhost = judge_server_cmd.hostname_ref() - jport = judge_server_cmd.meta_ref("port") - jbase = f"http://{jhost}:{jport}" - waits.append(pipeline_utils.get_server_wait_cmd(f"{jbase}{ctx.judge_server_health_path}")) - judge_url = f"{jbase}{ctx.judge_server_api_path}" - - # Wait for servers to be ready, then run evaluator - wait_cmd = " && ".join(waits) if waits else "true" - cmd = _build_task_cmd( - task_name=ctx.task_name, - launcher_run_cfg=ctx.launcher_run_cfg, - task_cfg=ctx.task_cfg, - task_definition=ctx.task_definition, - expname=ctx.expname, - base_output_root=ctx.base_output_root, - url_override=target_url, - model_id=ctx.server_model, - judge_url_override=judge_url, - judge_model_id=ctx.judge_server_model, - ) - return f"{wait_cmd} && {cmd}" - - return Command( - command=_client_cmd_factory, - container=ctx.eval_image, - gpus=ctx.job_gpus or None, - nodes=ctx.job_nodes or 1, - name=f"{ctx.expname}-client-{ctx.idx}-{ctx.task_name}", - metadata={ - "log_prefix": "main", - "environment": ctx.env_vars, - "gpus": ctx.job_gpus or None, - }, - ) - - # No hosted servers: Use external URLs or config defaults - server_url = None - if ctx.with_external_server and ctx.server_base_url: - server_url = ctx.server_base_url.rstrip("/") + ctx.server_api_path - judge_url = None - if ctx.with_external_judge and ctx.judge_server_base_url: - judge_url = ctx.judge_server_base_url.rstrip("/") + ctx.judge_server_api_path - - eval_cmd = _build_task_cmd( - task_name=ctx.task_name, - launcher_run_cfg=ctx.launcher_run_cfg, - task_cfg=ctx.task_cfg, - task_definition=ctx.task_definition, - expname=ctx.expname, - base_output_root=ctx.base_output_root, - url_override=server_url, - model_id=ctx.server_model, - judge_url_override=judge_url, - judge_model_id=ctx.judge_server_model, + client_script = EvaluatorClientScript( + ctx=ctx, + main_server_script=main_server_cmd.script if main_server_cmd else None, + judge_server_script=judge_server_cmd.script if judge_server_cmd else None, ) return Command( - command=eval_cmd, + script=client_script, container=ctx.eval_image, - gpus=None, - nodes=ctx.job_nodes or 1, - name=f"{ctx.expname}-{ctx.idx}-{ctx.task_name}", - metadata={ - "log_prefix": "main", - "environment": ctx.env_vars, - "gpus": ctx.job_gpus or None, - }, + name=f"{ctx.expname}-client-{ctx.idx}-{ctx.task_name}", ) def _build_task_cmd( task_name: str, - launcher_run_cfg: DictConfig, - task_cfg: DictConfig, + launcher_run_cfg: dict, + task_cfg: dict, task_definition: dict, expname: str, base_output_root: Optional[str], @@ -752,8 +670,8 @@ def _build_task_cmd( Args: task_name: Task identifier (e.g., "ifeval", "gpqa_diamond") - launcher_run_cfg: Global evaluator configuration from RunConfig - task_cfg: Task-specific configuration (may include task-level overrides) + launcher_run_cfg: Global evaluator configuration (as plain dict) + task_cfg: Task-specific configuration (as plain dict, may include task-level overrides) task_definition: Task definition from mapping (container, harness info) expname: Experiment name for output directory structure base_output_root: Base directory for task outputs @@ -771,7 +689,9 @@ def _build_task_cmd( - Judge: config.params.extra.judge.url Output directory is set to: {base_output_root}/{expname}/nemo-evaluator-results/{task_name} """ - task_cfg_copy = copy.deepcopy(task_cfg) + # Convert back to DictConfig for OmegaConf operations + launcher_run_cfg = OmegaConf.create(launcher_run_cfg) + task_cfg_copy = OmegaConf.create(copy.deepcopy(task_cfg)) if url_override: OmegaConf.update(task_cfg_copy, "overrides", {"target.api_endpoint.url": url_override}, force_add=True) @@ -806,3 +726,56 @@ def _build_task_cmd( cmd_struct = get_eval_factory_command(launcher_run_cfg, task_cfg_copy, task_definition) return cmd_struct.cmd + + +@dataclass(kw_only=True) +class EvaluatorClientScript(BaseJobScript): + """run.Script implementation for nemo-evaluator client with runtime server resolution.""" + + ctx: _TaskCreationContext + main_server_script: Optional[ServerScript] = None + judge_server_script: Optional[ServerScript] = None + log_prefix: str = field(default="main", init=False) + + def __post_init__(self): + def build_command(): + waits: List[str] = [] + target_url: Optional[str] = None + judge_url: Optional[str] = None + + if self.ctx.hosting_server and self.main_server_script is not None: + server_host = self.main_server_script.hostname_ref() + base_url = f"http://{server_host}:{self.main_server_script.port}" + waits.append(pipeline_utils.get_server_wait_cmd(f"{base_url}{self.ctx.server_health_path}")) + target_url = f"{base_url}{self.ctx.server_api_path}" + elif self.ctx.with_external_server and self.ctx.server_base_url: + target_url = self.ctx.server_base_url.rstrip("/") + self.ctx.server_api_path + + if self.ctx.hosting_judge and self.judge_server_script is not None: + judge_host = self.judge_server_script.hostname_ref() + judge_base = f"http://{judge_host}:{self.judge_server_script.port}" + waits.append(pipeline_utils.get_server_wait_cmd(f"{judge_base}{self.ctx.judge_server_health_path}")) + judge_url = f"{judge_base}{self.ctx.judge_server_api_path}" + elif self.ctx.with_external_judge and self.ctx.judge_server_base_url: + judge_url = self.ctx.judge_server_base_url.rstrip("/") + self.ctx.judge_server_api_path + + cmd = _build_task_cmd( + task_name=self.ctx.task_name, + launcher_run_cfg=self.ctx.launcher_run_cfg, + task_cfg=self.ctx.task_cfg, + task_definition=self.ctx.task_definition, + expname=self.ctx.expname, + base_output_root=self.ctx.base_output_root, + url_override=target_url, + model_id=self.ctx.server_model, + judge_url_override=judge_url, + judge_model_id=self.ctx.judge_server_model, + ) + + wait_cmd = " && ".join(waits) if waits else None + final_cmd = f"{wait_cmd} && {cmd}" if wait_cmd else cmd + env_vars = copy.deepcopy(self.ctx.env_vars) + return final_cmd, {"environment": env_vars} + + self.set_inline(build_command) + super().__post_init__() diff --git a/nemo_skills/pipeline/utils/__init__.py b/nemo_skills/pipeline/utils/__init__.py index 1e470f3539..3e738a530f 100644 --- a/nemo_skills/pipeline/utils/__init__.py +++ b/nemo_skills/pipeline/utils/__init__.py @@ -49,6 +49,8 @@ get_chunked_rs_filename, get_generation_cmd, get_remaining_jobs, + normalize_models_config, + normalize_parameter, wrap_cmd, ) from nemo_skills.pipeline.utils.mounts import ( diff --git a/nemo_skills/pipeline/utils/declarative.py b/nemo_skills/pipeline/utils/declarative.py index e294a3ed82..51d0746c63 100644 --- a/nemo_skills/pipeline/utils/declarative.py +++ b/nemo_skills/pipeline/utils/declarative.py @@ -12,39 +12,71 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import annotations + +import logging +from contextlib import nullcontext +from dataclasses import dataclass +from typing import Dict, List, Optional, Tuple, Union + +import nemo_run as run + +from nemo_skills.pipeline.utils import ( + get_env_variables, + get_executor, + get_exp, + get_exp_handles, + get_registered_external_repo, + get_tunnel, + run_exp, + temporary_env_update, +) +from nemo_skills.pipeline.utils.exp import ( + REUSE_CODE_EXP, + get_packaging_job_key, + tunnel_hash, +) +from nemo_skills.pipeline.utils.mounts import is_mounted_filepath +from nemo_skills.pipeline.utils.server import wrap_python_path +from nemo_skills.utils import get_logger_name + """ -Simplified declarative pipeline system using only Command for all task types. +Simplified declarative pipeline system using Command with run.Script objects. Basic Example (Single job with multiple commands): - from nemo_skills.pipeline.utils.commands import vllm_server_command, sandbox_command + from nemo_skills.pipeline.utils.scripts import ServerScript, SandboxScript, GenerationClientScript from nemo_skills.pipeline.utils.declarative import Command, CommandGroup, HardwareConfig, Pipeline - from nemo_skills.pipeline.utils.server import get_free_port - - # Allocate ports for server and sandbox - server_port = get_free_port(strategy="random") - sandbox_port = get_free_port(strategy="random") - - # Commands that run together in one SLURM job - # Note: Lambdas are needed for cross-component references (hostname_ref, meta_ref) - # which aren't resolved until het_group_index is assigned at pipeline execution time. - server_cmd, server_meta = vllm_server_command(cluster_cfg, model="Qwen/Qwen3-8B", port=server_port) - server = Command(command=server_cmd, gpus=8, name="server", metadata=server_meta) - - sandbox_cmd, sandbox_meta = sandbox_command(cluster_cfg, port=sandbox_port) - sandbox = Command(command=sandbox_cmd, name="sandbox", metadata=sandbox_meta) - - # This lambda is ESSENTIAL - server.hostname_ref() and meta_ref() aren't available until runtime - # Client needs NEMO_SKILLS_SANDBOX_PORT to connect to sandbox - client = Command( - command=lambda: f"curl {server.hostname_ref()}:{server.meta_ref('port')}/health", - name="client", - metadata={"environment": {"NEMO_SKILLS_SANDBOX_PORT": str(sandbox_port)}} + + # Create Script objects for server and sandbox + # Scripts handle port allocation, cross-component references, and command building + server_script = ServerScript( + server_type="vllm", + model_path="Qwen/Qwen2.5-Math-7B-Instruct", + server_args="--tensor-parallel-size 1" ) + sandbox_script = SandboxScript() + + # Create generation client that references server and sandbox + # Cross-component references (hostname_ref, port) are resolved at runtime + client_script = GenerationClientScript( + output_dir="/results/inference", + extra_arguments="++prompt_config=math ++split=test", + servers=[server_script], # References server for hostname/port + model_names=["Qwen/Qwen2.5-Math-7B-Instruct"], + server_types=["vllm"], + sandbox=sandbox_script, # References sandbox for port + with_sandbox=True, + ) + + # Wrap Scripts in Commands with container and resource info + server = Command(script=server_script, container="vllm", name="server") + sandbox = Command(script=sandbox_script, container="nemo-skills", name="sandbox") + client = Command(script=client_script, container="nemo-skills", name="client") - # Group them together + # Group them together (they run in one SLURM job) inference_group = CommandGroup( commands=[server, sandbox, client], - hardware=HardwareConfig(partition="batch"), + hardware=HardwareConfig(partition="batch", num_gpus=1), name="inference" ) @@ -57,13 +89,27 @@ pipeline.run() Advanced Example (Multiple jobs with dependencies and heterogeneous components): + from nemo_skills.pipeline.utils.scripts import ServerScript, SandboxScript, GenerationClientScript + from nemo_run import Script + log_dir = "/experiments/full_pipeline/logs" - # Job 1: Preprocessing - preprocess = Command( - command="python preprocess.py --input data.jsonl --output processed.jsonl", - gpus=0, - name="preprocess" + + # Job 1: Preprocessing with custom Script + @dataclass(kw_only=True) + class PreprocessScript(Script): + input_file: str + output_file: str + + def __post_init__(self): + cmd = f"python preprocess.py --input {self.input_file} --output {self.output_file}" + self.inline = cmd + object.__setattr__(self, 'entrypoint', 'bash') + + preprocess_script = PreprocessScript( + input_file="data.jsonl", + output_file="processed.jsonl" ) + preprocess = Command(script=preprocess_script, name="preprocess") prep_group = CommandGroup( commands=[preprocess], hardware=HardwareConfig(partition="cpu"), @@ -72,39 +118,76 @@ ) prep_job = {"name": "prep", "group": prep_group} - # Job 2: Two different model servers (HETEROGENEOUS SLURM job with 2 het components) - # Allocate ports for each server/sandbox pair - from nemo_skills.pipeline.utils.server import get_free_port - server_8b_port = get_free_port(strategy="random") - sandbox_8b_port = get_free_port(strategy="random") - server_32b_port = get_free_port(strategy="random") - sandbox_32b_port = get_free_port(strategy="random") - - # Build commands with cluster_config - server_8b_cmd, server_8b_meta = vllm_server_command(cluster_config, model="Qwen/Qwen3-8B", port=server_8b_port) - sandbox_8b_cmd, sandbox_8b_meta = sandbox_command(cluster_config, port=sandbox_8b_port) - server_32b_cmd, server_32b_meta = vllm_server_command(cluster_config, model="Qwen/Qwen3-32B", port=server_32b_port) - sandbox_32b_cmd, sandbox_32b_meta = sandbox_command(cluster_config, port=sandbox_32b_port) + # Job 2: Two different model servers (HETEROGENEOUS SLURM job with 2 het groups) + # 8B model group + server_8b = ServerScript( + server_type="vllm", + model_path="Qwen/Qwen2.5-Math-7B-Instruct", + server_args="--tensor-parallel-size 1" + ) + sandbox_8b = SandboxScript() + client_8b = GenerationClientScript( + output_dir="/results/eval_8b", + extra_arguments="++prompt_config=math", + servers=[server_8b], + model_names=["Qwen/Qwen2.5-Math-7B-Instruct"], + server_types=["vllm"], + sandbox=sandbox_8b, + with_sandbox=True, + ) - server_8b = Command(command=server_8b_cmd, gpus=8, name="server_8b", metadata=server_8b_meta) - sandbox_8b = Command(command=sandbox_8b_cmd, name="sandbox_8b", metadata=sandbox_8b_meta) - eval_8b = Command(command="python eval.py --model 8b", gpus=1, name="eval_8b") + group_8b = CommandGroup( + commands=[ + Command(script=server_8b, container="vllm", name="server_8b"), + Command(script=sandbox_8b, container="nemo-skills", name="sandbox_8b"), + Command(script=client_8b, container="nemo-skills", name="eval_8b"), + ], + hardware=HardwareConfig(partition="batch", num_gpus=1), + name="eval_8b", + log_dir=log_dir + ) - server_32b = Command(command=server_32b_cmd, gpus=8, name="server_32b", metadata=server_32b_meta) - sandbox_32b = Command(command=sandbox_32b_cmd, name="sandbox_32b", metadata=sandbox_32b_meta) - eval_32b = Command(command="python eval.py --model 32b", gpus=1, name="eval_32b") + # 32B model group + server_32b = ServerScript( + server_type="vllm", + model_path="Qwen/Qwen2.5-Math-32B-Instruct", + server_args="--tensor-parallel-size 4" + ) + sandbox_32b = SandboxScript() + client_32b = GenerationClientScript( + output_dir="/results/eval_32b", + extra_arguments="++prompt_config=math", + servers=[server_32b], + model_names=["Qwen/Qwen2.5-Math-32B-Instruct"], + server_types=["vllm"], + sandbox=sandbox_32b, + with_sandbox=True, + ) - group_8b = CommandGroup(commands=[server_8b, sandbox_8b, eval_8b], name="eval_8b", log_dir=log_dir) - group_32b = CommandGroup(commands=[server_32b, sandbox_32b, eval_32b], name="eval_32b", log_dir=log_dir) + group_32b = CommandGroup( + commands=[ + Command(script=server_32b, container="vllm", name="server_32b"), + Command(script=sandbox_32b, container="nemo-skills", name="sandbox_32b"), + Command(script=client_32b, container="nemo-skills", name="eval_32b"), + ], + hardware=HardwareConfig(partition="batch", num_gpus=4), + name="eval_32b", + log_dir=log_dir + ) evals_job = {"name": "evals", "groups": [group_8b, group_32b], "dependencies": [prep_job]} # Job 3: Report generation (depends on both evaluations) - report = Command( - command="python generate_report.py --output report.txt", - gpus=0, - name="report" - ) + @dataclass(kw_only=True) + class ReportScript(Script): + output_file: str + + def __post_init__(self): + self.inline = f"python generate_report.py --output {self.output_file}" + object.__setattr__(self, 'entrypoint', 'bash') + + report_script = ReportScript(output_file="report.txt") + report = Command(script=report_script, name="report") report_group = CommandGroup(commands=[report], name="report", log_dir=log_dir) # Create pipeline with dependency graph @@ -121,130 +204,56 @@ pipeline.run() """ -import logging -import shlex -from contextlib import nullcontext -from dataclasses import dataclass, field -from typing import Callable, Dict, List, Optional, Tuple, Union - -import nemo_run as run - -from nemo_skills.pipeline.utils import ( - get_env_variables, - get_executor, - get_exp, - get_exp_handles, - get_tunnel, - run_exp, - temporary_env_update, -) -from nemo_skills.pipeline.utils.commands import wrap_command -from nemo_skills.pipeline.utils.exp import ( - REUSE_CODE_EXP, - get_packaging_job_key, - install_packages_wrap, - tunnel_hash, -) -from nemo_skills.pipeline.utils.mounts import is_mounted_filepath -from nemo_skills.pipeline.utils.packager import get_registered_external_repo -from nemo_skills.utils import get_logger_name - LOG = logging.getLogger(get_logger_name(__file__)) @dataclass class Command: - """Declarative command for running tasks in containers. - - The command can be either: - - A string: evaluated immediately - - A callable (lambda): evaluated lazily when the task is prepared + """Declarative command for running tasks in containers using run.Script objects. - Lambdas are ONLY needed for cross-component references (hostname_ref, meta_ref). - The het_group_index isn't assigned until pipeline execution, so these must be lazy: - # Lambda is ESSENTIAL here - server.hostname_ref() and meta_ref() don't exist yet - client = Command(command=lambda: f"curl {server.hostname_ref()}:{server.meta_ref('port')}") + Example: + server = ServerScript(server_type="vllm", model_path="/models/llama", ...) + Command(script=server, container="vllm", name="my_server") """ - # Command can be a string or callable (lambda). - # Lambdas are primarily used for cross-component references (hostname_ref, meta_ref). - command: Union[str, Callable] + script: run.Script container: str = "nemo-skills" - gpus: Optional[int] = None - nodes: int = 1 name: str = "command" - working_dir: str = "/nemo_run/code" - env_vars: Dict[str, str] = field(default_factory=dict) - installation_command: Optional[str] = None - port: Optional[int] = None # Can be set from metadata - metadata: Dict[str, any] = field(default_factory=dict) # Stores metadata from command builders - het_group_index: Optional[int] = None # Set per-job by Pipeline (not global) - - def __post_init__(self): - # Wrap plain strings with environment setup - if isinstance(self.command, str) and (self.env_vars or self.working_dir): - self.command = wrap_command(self.command, self.working_dir, self.env_vars) - - def hostname_ref(self) -> str: - """Get hostname reference for hetjob cross-component communication.""" - if self.het_group_index is None: - return "127.0.0.1" # Local fallback - # For heterogeneous SLURM jobs, resolve nodelist to actual hostname - return f"$(scontrol show hostnames $SLURM_JOB_NODELIST_HET_GROUP_{self.het_group_index} | head -n1)" - - def meta_ref(self, key: str) -> str: - """Get metadata value (like port). Fails if key not found.""" - if key not in self.metadata: - raise KeyError( - f"Metadata key '{key}' not found in Command '{self.name}'. " - f"Available keys: {list(self.metadata.keys())}" - ) - return str(self.metadata[key]) - def prepare_for_execution(self, cluster_config: Dict) -> Tuple[str, Dict]: - """Prepare command for execution. + def prepare_for_execution(self, cluster_config: Dict) -> Tuple[run.Script, Dict]: + """Prepare script for execution. This method: - 1. Evaluates callables (resolves cross-component references) - 2. Wraps with installation_command if provided + 1. Evaluates lazy commands (if script.inline is callable) + 2. Builds execution config from Script fields Returns: - Tuple of (final_command, execution_config) + Tuple of (Script_object, execution_config) """ - # 1. Evaluate if callable (for cross-component references like hostname_ref) - if callable(self.command): - result = self.command() + runtime_metadata = {} + + # If script.inline is callable (lazy command building), evaluate it now + if callable(self.script.inline): + result = self.script.inline() if isinstance(result, tuple): - final_command, runtime_metadata = result - # Deep merge metadata, especially environment dict - for key, value in runtime_metadata.items(): - if key == "environment" and key in self.metadata: - # Merge environment dicts instead of replacing - self.metadata[key].update(value) - else: - self.metadata[key] = value + evaluated_command, runtime_metadata = result else: - final_command = result - else: - final_command = self.command + evaluated_command = result - # 2. Wrap with installation_command if provided - if self.installation_command: - final_command = install_packages_wrap(final_command, self.installation_command) + # Update script.inline with evaluated command + self.script.set_inline(evaluated_command) - # 3. Build execution config from metadata + # Build execution config from Script fields execution_config = { - "num_tasks": self.metadata.get("num_tasks", 1), - "num_gpus": self.metadata.get("gpus", self.gpus or 0), - "num_nodes": self.metadata.get("nodes", self.nodes), - "environment": self.metadata.get("environment", {}), - "log_prefix": self.metadata.get("log_prefix", "main"), - "mounts": self.metadata.get("mounts"), - "container": self.metadata.get("container", self.container), # Use container from metadata if available + "log_prefix": getattr(self.script, "log_prefix", "main"), + "environment": runtime_metadata.get("environment", {}), + "mounts": None, # Mounts not currently exposed by Scripts + "container": self.container, } - return final_command, execution_config + # Return the Script object itself + return self.script, execution_config def get_name(self) -> str: return self.name @@ -257,6 +266,7 @@ class HardwareConfig: partition: Optional[str] = None num_gpus: Optional[int] = None num_nodes: Optional[int] = None + num_tasks: Optional[int] = 1 sbatch_kwargs: Optional[dict] = None @@ -482,16 +492,49 @@ def run(self, dry_run: bool = False, log_dir: Optional[str] = None, _reuse_exp=N return exp - def _prepare_command(self, command, cluster_config: Dict) -> Tuple[str, Dict]: - """Prepare command and handle mpirun wrapping.""" - final_cmd, exec_config = command.prepare_for_execution(cluster_config) - - # Handle mpirun wrapping for non-SLURM executors - num_tasks = exec_config["num_tasks"] - if cluster_config["executor"] != "slurm" and num_tasks > 1: - final_cmd = f"mpirun --allow-run-as-root -np {num_tasks} bash -c {shlex.quote(final_cmd)}" + def _prepare_command(self, command, cluster_config: Dict) -> Tuple[run.Script, Dict]: + """Prepare command for execution. - return final_cmd, exec_config + Returns: + Tuple of (Script_object, exec_config) + """ + script, exec_config = command.prepare_for_execution(cluster_config) + # Only rewrite paths for "none" executor (native execution without containers) + # For "local" executor (Docker), paths should stay as /nemo_run/code/... since + # that's where the code is mounted inside the container + if cluster_config.get("executor") == "none": + script = self._rewrite_local_paths(script) + # Note: mpirun wrapping for multi-task scripts is handled by the executor + return script, exec_config + + def _rewrite_local_paths(self, script: run.Script) -> run.Script: + """For executor='none', replace /nemo_run/code paths with local repo paths.""" + nemo_repo = get_registered_external_repo("nemo_skills") + if nemo_repo is None: + return script + + pkg_path = str(nemo_repo.path) + repo_root = str(nemo_repo.path.parent) + + def _replace(cmd: str) -> str: + return cmd.replace("/nemo_run/code/nemo_skills", pkg_path).replace("/nemo_run/code", repo_root) + + inline_cmd = script.inline + if isinstance(inline_cmd, str): + script.set_inline(_replace(inline_cmd)) + elif callable(inline_cmd): + original_inline = inline_cmd + + def wrapped_inline(): + result = original_inline() + if isinstance(result, tuple): + cmd, metadata = result + return _replace(cmd), metadata + return _replace(result) + + script.set_inline(wrapped_inline) + + return script def _resolve_container(self, exec_config: Dict, command, cluster_config: Dict) -> str: """Resolve container name to image path.""" @@ -513,6 +556,7 @@ def _create_executor( total_het_groups: int, overlap: bool, dependencies: Optional[List] = None, + job_name_override: Optional[str] = None, ): """Create executor with optional environment update.""" env_context = ( @@ -525,10 +569,10 @@ def _create_executor( return get_executor( cluster_config=cluster_config, container=container_image, - num_nodes=exec_config["num_nodes"], - tasks_per_node=exec_config["num_tasks"], - gpus_per_node=exec_config["num_gpus"], - job_name=command.name, + num_nodes=hardware.num_nodes if hardware and hardware.num_nodes is not None else 1, + tasks_per_node=hardware.num_tasks if hardware and hardware.num_tasks is not None else 1, + gpus_per_node=hardware.num_gpus if hardware and hardware.num_gpus is not None else 0, + job_name=job_name_override if job_name_override else command.name, log_dir=log_dir, log_prefix=exec_config["log_prefix"], partition=hardware.partition if hardware else None, @@ -567,81 +611,105 @@ def _plan_and_add_job( if log_dir is None: raise ValueError(f"CommandGroup '{groups[0].name}' must have log_dir set, or provide it to pipeline.run()") - commands: List[str] = [] + scripts: List[run.Script] = [] executors: List = [] het_group_indices: List[int] = [] - # In heterogeneous jobs, collect environment from all commands for cross-component refs - shared_env_vars: Dict[str, str] = {} - if heterogeneous: - for het_idx, group in enumerate(groups): - for command in group.commands: - _, exec_config_probe = command.prepare_for_execution(cluster_config) - shared_env_vars.update(exec_config_probe.get("environment", {})) + # Assign het_group_index values before evaluating any commands so cross-references + # (e.g., hostname_ref) see the correct indices regardless of processing order. + for het_idx, group in enumerate(groups): + for command in group.commands: + command.script.het_group_index = het_idx if heterogeneous else None - # Share packager across executors for efficiency (single-group only) - shared_packager = None + # Prepare commands once and collect runtime data for a second pass where we + # construct executors. This ensures all scripts have resolved cross-references. + prepared_commands: List[Dict] = [] + shared_env_vars: Dict[str, str] = {} - # Build commands and executors for het_idx, group in enumerate(groups): has_multiple_components = len(group.commands) > 1 total_het_groups = ( len(groups) if heterogeneous else (len(group.commands) if has_multiple_components else 1) ) - # For single-group jobs with multiple components, allow job-level GPU override for sbatch allocation - job_level_gpus = ( - group.hardware.num_gpus if (not heterogeneous and has_multiple_components and group.hardware) else None - ) - for comp_idx, command in enumerate(group.commands): - # Assign het_group_index ONLY for heterogeneous jobs (per-job, not global) - # Non-heterogeneous jobs use localhost, so het_group_index should remain None - if heterogeneous: - command.het_group_index = het_idx - else: - command.het_group_index = None - - final_cmd, exec_config = self._prepare_command(command, cluster_config) - commands.append(final_cmd) - - # Adjust GPU allocation (first component gets job-level GPUs for sbatch) for single-group jobs - exec_config["num_gpus"] = exec_config["num_gpus"] or 0 - if (not heterogeneous) and (comp_idx == 0) and (job_level_gpus is not None): - exec_config["num_gpus"] = job_level_gpus - - # Merge shared environment for heterogeneous jobs - if heterogeneous and shared_env_vars: - exec_config["environment"].update(shared_env_vars) - - # Resolve container and create executor - container_image = self._resolve_container(exec_config, command, cluster_config) - # Pass external dependencies only to the first executor (SLURM doesn't support per-component dependencies in hetjobs) - exec_dependencies = external_deps if (het_idx == 0 and comp_idx == 0) else None - executor = self._create_executor( - command, - exec_config, - container_image, - cluster_config, - log_dir, - group.hardware, - heterogeneous, - het_idx if heterogeneous else comp_idx, - total_het_groups, - (len(group.commands) > 1), - dependencies=exec_dependencies, + script, exec_config = self._prepare_command(command, cluster_config) + + if isinstance(script.inline, str): + if cluster_config.get("executor") not in ("none", "local"): + script.set_inline(wrap_python_path(script.inline)) + + prepared_commands.append( + { + "het_idx": het_idx, + "comp_idx": comp_idx, + "group": group, + "command": command, + "script": script, + "exec_config": exec_config, + "total_het_groups": total_het_groups, + "overlap": len(group.commands) > 1, + } ) - # Share packager across executors for single-group jobs - if not heterogeneous: - if comp_idx == 0 and het_idx == 0: - shared_packager = executor.packager - else: - executor.packager = shared_packager - - executors.append(executor) if heterogeneous: - het_group_indices.append(het_idx) + shared_env_vars.update(exec_config.get("environment", {})) + + # Share packager across executors for efficiency (single-group only) + shared_packager = None + + # Build commands and executors using prepared data + for entry in prepared_commands: + het_idx = entry["het_idx"] + comp_idx = entry["comp_idx"] + group = entry["group"] + command = entry["command"] + script = entry["script"] + exec_config = entry["exec_config"] + total_het_groups = entry["total_het_groups"] + overlap = entry["overlap"] + + scripts.append(script) + + # Merge shared environment for heterogeneous jobs + if heterogeneous and shared_env_vars: + exec_config["environment"].update(shared_env_vars) + + # Resolve container and create executor + container_image = self._resolve_container(exec_config, command, cluster_config) + # Pass external dependencies only to the first executor (SLURM doesn't support per-component dependencies in hetjobs) + exec_dependencies = external_deps if (het_idx == 0 and comp_idx == 0) else None + + # Always use group.name for SLURM job name (consistent across all components) + # The group name is set to task_name in generate.py, without component suffixes + # Component names (like {task_name}_server, {task_name}_sandbox) are only used for log_prefix + job_name_for_slurm = group.name + + executor = self._create_executor( + command, + exec_config, + container_image, + cluster_config, + log_dir, + group.hardware, + heterogeneous, + het_idx if heterogeneous else comp_idx, + total_het_groups, + overlap, + dependencies=exec_dependencies, + job_name_override=job_name_for_slurm, + ) + + # Share packager across executors for single-group jobs + if not heterogeneous: + if comp_idx == 0 and het_idx == 0: + shared_packager = executor.packager + else: + executor.packager = shared_packager + + executors.append(executor) + if heterogeneous: + het_group_indices.append(het_idx) # For heterogeneous jobs, set het_group_indices on the first executor if heterogeneous and executors: @@ -676,13 +744,7 @@ def _plan_and_add_job( # If reuse_code=False, clear cache REUSE_CODE_EXP.pop(tunnel_hash(tunnel), None) - # Handle executor="none" path replacements (single-group only) - if (not heterogeneous) and cluster_config["executor"] == "none": - for idx in range(len(commands)): - commands[idx] = commands[idx].replace( - "/nemo_run/code/nemo_skills", str(get_registered_external_repo("nemo_skills").path) - ) - commands[idx] = commands[idx].replace("/nemo_run/code", "./") + # Note: Path replacements for executor="none" are no longer needed with Script interface # Ray metadata handling if self.with_ray and cluster_config["executor"] == "slurm": @@ -693,19 +755,24 @@ def _plan_and_add_job( # Add to experiment and return task ID # Note: Internal dependencies (task handles from same experiment) go to exp.add() # External dependencies (SLURM job IDs from other experiments) go to executor - if (not heterogeneous) and len(commands) == 1: + if (not heterogeneous) and len(scripts) == 1: + # Single script - pass directly to exp.add() + if metadata: + scripts[0].metadata = metadata task_id = exp.add( - run.Script(inline=commands[0], metadata=metadata), + scripts[0], executor=executors[0], name="nemo-run", dependencies=internal_deps, ) else: + # Multiple scripts or heterogeneous job + # Apply metadata to first script only + if metadata: + scripts[0].metadata = metadata + task_id = exp.add( - [ - run.Script(inline=cmd, metadata=(metadata if idx == 0 else None)) - for idx, cmd in enumerate(commands) - ], + scripts, executor=executors, name="nemo-run", dependencies=internal_deps, diff --git a/nemo_skills/pipeline/utils/generation.py b/nemo_skills/pipeline/utils/generation.py index cd576053c1..8ae4e96bb5 100644 --- a/nemo_skills/pipeline/utils/generation.py +++ b/nemo_skills/pipeline/utils/generation.py @@ -17,6 +17,7 @@ import shlex import subprocess from collections import defaultdict +from typing import Any, List, Optional, Union from nemo_skills.pipeline.utils.cluster import get_tunnel from nemo_skills.pipeline.utils.mounts import get_unmounted_path @@ -26,6 +27,81 @@ LOG = logging.getLogger(get_logger_name(__file__)) +def normalize_models_config( + model: Optional[Union[str, List[str]]], +) -> List[str]: + """ + Normalize model specification to list. + + Handles both scalar and list inputs: + - CLI (Typer): Converts single values to single-element lists automatically + - Python API: Accepts both strings and lists + + Args: + model: Model path(s) - string or list from Python API, list from CLI + + Returns: + List of model paths + + Raises: + ValueError: If model is None or empty + """ + if model is None: + raise ValueError("Must specify --model") + + # Handle string (Python API with single model) + if isinstance(model, str): + return [model] + + # Handle list + if len(model) == 0: + raise ValueError("Must specify --model") + return list(model) + + +def normalize_parameter( + param_value: Any, + num_models: int, + param_name: str, +) -> List[Any]: + """ + Normalize a parameter to a per-model list. + + Handles both scalar and list inputs for flexible usage: + - CLI (Typer): Converts single values to single-element lists automatically + - Python API: Accepts both scalars and lists directly + + Broadcast logic: + - Scalar value: Broadcast to all models [value] * num_models + - Single-element list: Broadcast to all models + - Multi-element list: Must match num_models exactly + + Args: + param_value: Parameter value (scalar or list) + num_models: Number of models + param_name: Name of parameter (for error messages) + + Returns: + List of parameter values (one per model) + + Raises: + ValueError: If list length doesn't match num_models + """ + if not isinstance(param_value, list): + return [param_value] * num_models + + if len(param_value) == num_models: + return list(param_value) + + if len(param_value) == 1: + return param_value * num_models + + raise ValueError( + f"Parameter {param_name} has {len(param_value)} values but {num_models} models specified. " + f"Must be 1 value (broadcast) or {num_models} values (per-model)." + ) + + def get_chunked_rs_filename( output_dir: str, random_seed: int = None, @@ -294,8 +370,20 @@ def get_generation_cmd( wandb_parameters=None, with_sandbox: bool = False, script: str = "nemo_skills.inference.generate", + # Optional: for multi-model generation + server_addresses: Optional[List[str]] = None, + model_names: Optional[List[str]] = None, + server_types: Optional[List[str]] = None, ): - """Construct the generation command for language model inference.""" + """Construct the generation command for language model inference. + + Supports both single-model and multi-model generation. For multi-model: + - server_addresses: List of server addresses (one per model) + - model_names: List of model names (one per model) + - server_types: List of server types (one per model) + + For single-model, server config is passed via extra_arguments. + """ if input_file is None and input_dir is None: raise ValueError("Either input_file or input_dir must be provided.") if input_file is not None and input_dir is not None: @@ -313,6 +401,7 @@ def get_generation_cmd( output_dir=output_dir, random_seed=random_seed, ) + # Preamble for generation commands: added at executor/declarative level cmd = "export HYDRA_FULL_ERROR=1 && " # Separate Hydra config args (--config-*) from override args (++) @@ -327,6 +416,22 @@ def get_generation_cmd( else: # It's a module name, use -m flag cmd += f"python -m {script} {hydra_config_args} {common_args} " + + # Add multi-model configuration if provided + if server_addresses is not None and model_names is not None: + num_models = len(model_names) + if num_models > 1: + # Multi-model: pass server configuration as lists + model_names_arg = ",".join(model_names) + cmd += f"++server.model=[{model_names_arg}] " + + server_types_arg = ",".join(server_types) + cmd += f"++server.server_type=[{server_types_arg}] " + + server_addresses_arg = ",".join(server_addresses) + cmd += f"++server.base_url=[{server_addresses_arg}] " + # For n=1: server config is already in extra_arguments from configure_client + job_end_cmd = "" if random_seed is not None and input_dir is None: # if input_dir is not None, we default to greedy generations diff --git a/nemo_skills/pipeline/utils/scripts.py b/nemo_skills/pipeline/utils/scripts.py new file mode 100644 index 0000000000..4e37a6b594 --- /dev/null +++ b/nemo_skills/pipeline/utils/scripts.py @@ -0,0 +1,419 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Script classes for NeMo-Skills pipeline components. + +These classes wrap NeMo-Run's run.Script interface to provide typed, reusable +job components (servers, clients, sandboxes) with explicit fields and +cross-component reference support for heterogeneous jobs. + +Example: + # Create a server script with automatic port allocation + server = ServerScript( + server_type="vllm", + model_path="/models/llama-8b", + cluster_config=cluster_config, + num_gpus=8, + ) + + # Create a client that references the server + client = GenerationClientScript( + output_dir="/results", + input_file="/data/input.jsonl", + server=server, # Cross-component reference + ) + + # Use in Command objects + Command(script=server, container="vllm", ...) + Command(script=client, container="nemo-skills", ...) +""" + +import logging +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Tuple, Union + +import nemo_run as run + +from nemo_skills.pipeline.utils.commands import sandbox_command +from nemo_skills.pipeline.utils.exp import install_packages_wrap +from nemo_skills.pipeline.utils.generation import get_generation_cmd +from nemo_skills.pipeline.utils.server import get_free_port, get_server_command +from nemo_skills.utils import get_logger_name + +if TYPE_CHECKING: + # Avoid circular imports for type hints + pass + +LOG = logging.getLogger(get_logger_name(__file__)) + + +@dataclass +class BaseJobScript(run.Script): + """Base class for job component scripts with heterogeneous job support. + + This class provides: + - het_group_index tracking for cross-component references in heterogeneous SLURM jobs + - hostname_ref() method for getting hostnames in het jobs + - Common pattern for Script initialization + + Attributes: + het_group_index: Index in heterogeneous job group (set by Pipeline at runtime) + """ + + het_group_index: Optional[int] = field(default=None, init=False, repr=False) + installation_command: Optional[str] = None + entrypoint: str = field(default="bash", init=False) + + def __post_init__(self): + """Wrap inline command with installation_command if provided.""" + if not self.installation_command: + return + + if callable(self.inline): + original_inline = self.inline + + def wrapped_inline(): + result = original_inline() + if isinstance(result, tuple): + command, metadata = result + return install_packages_wrap(command, self.installation_command), metadata + return install_packages_wrap(result, self.installation_command) + + self.set_inline(wrapped_inline) + elif isinstance(self.inline, str): + self.set_inline(install_packages_wrap(self.inline, self.installation_command)) + + def set_inline(self, command: Union[str, Callable, run.Script]) -> None: + """Set the inline command safely on frozen dataclass.""" + object.__setattr__(self, "inline", command) + + def hostname_ref(self) -> str: + """Get hostname reference for hetjob cross-component communication. + + Returns a shell variable reference that resolves to the master node hostname + for this het group. Uses environment variables automatically exported by nemo-run: + SLURM_MASTER_NODE_HET_GROUP_0, SLURM_MASTER_NODE_HET_GROUP_1, etc. + + These are set via: + export SLURM_MASTER_NODE_HET_GROUP_N=$(scontrol show hostnames $SLURM_JOB_NODELIST_HET_GROUP_N | head -n1) + """ + if self.het_group_index is None: + return "127.0.0.1" # Local fallback for non-heterogeneous jobs + + # Use the environment variable exported by nemo-run + return f"${{SLURM_MASTER_NODE_HET_GROUP_{self.het_group_index}:-localhost}}" + + +@dataclass(kw_only=True) +class ServerScript(BaseJobScript): + """Script for model inference servers (vLLM, TRT-LLM, SGLang, etc.). + + This script wraps server command builders and provides: + - Automatic port allocation if not specified + - Type-safe server configuration + - Cross-component address sharing (get_address()) + - Resource requirement tracking (num_gpus, num_nodes, num_tasks) + + Attributes: + server_type: Type of server (vllm, trtllm, sglang, megatron, openai, etc.) + model_path: Path to model weights or model name for API services + cluster_config: Cluster configuration dictionary + num_gpus: Number of GPUs required (default: 8) + num_nodes: Number of nodes required (default: 1) + server_args: Additional server-specific arguments + server_entrypoint: Custom server entrypoint script (optional) + port: Server port (allocated automatically if None) + allocate_port: Whether to allocate port automatically (default: True) + num_tasks: Number of MPI tasks (computed in __post_init__) + log_prefix: Prefix for log files (default: "server") + + Example: + # Basic usage + server = ServerScript( + server_type="vllm", + model_path="/models/llama-3-8b", + cluster_config=cluster_config, + num_gpus=8, + ) + + # Access allocated port + print(f"Server will run on port {server.port}") + + # Get full address for client connection + address = server.get_address() # Returns "hostname:port" + """ + + server_type: str + model_path: str + cluster_config: Dict + num_gpus: int = 8 + num_nodes: int = 1 + server_args: str = "" + server_entrypoint: Optional[str] = None # Custom server entrypoint script + port: Optional[int] = None + allocate_port: bool = True + + # Computed fields (set in __post_init__) + num_tasks: int = field(init=False, repr=False) + log_prefix: str = field(default="server", init=False) + + def __post_init__(self): + """Initialize server script. + + - Allocates port if not provided + - Builds server command using get_server_command() + - Sets self.inline to the command string + - Computes num_tasks from server command builder + """ + # Allocate port if not provided + if self.port is None and self.allocate_port: + self.port = get_free_port(strategy="random") + LOG.debug(f"Allocated port {self.port} for {self.server_type} server") + + # Build server command + cmd, self.num_tasks = get_server_command( + server_type=self.server_type, + num_gpus=self.num_gpus, + num_nodes=self.num_nodes, + model_path=self.model_path, + cluster_config=self.cluster_config, + server_port=self.port, + server_args=self.server_args, + server_entrypoint=self.server_entrypoint, + ) + + self.set_inline(cmd) + super().__post_init__() + + def get_address(self) -> str: + """Get server address for client connections. + + Returns hostname:port string that clients can use to connect. + In heterogeneous jobs, hostname_ref() returns a bash expression + that resolves at runtime. + + Returns: + Server address in format "hostname:port" + + Example: + # Use in client command + client_cmd = f"python client.py --server-url http://{server.get_address()}" + """ + return f"{self.hostname_ref()}:{self.port}" + + +@dataclass(kw_only=True) +class SandboxScript(BaseJobScript): + """Script for code execution sandbox container. + + The sandbox provides a secure environment for executing LLM-generated code. + This script wraps sandbox command builders and provides: + - Automatic port allocation + - Mount configuration (can optionally keep mounts, though risky) + - Type-safe sandbox configuration + + Attributes: + cluster_config: Cluster configuration dictionary + port: Sandbox port (allocated automatically if None) + keep_mounts: Whether to keep filesystem mounts (default: False, risky if True). + Note: This is stored for documentation but actually handled at + the executor level, not in the sandbox command itself. + allocate_port: Whether to allocate port automatically (default: True) + log_prefix: Prefix for log files (default: "sandbox") + + Example: + sandbox = SandboxScript( + cluster_config=cluster_config, + keep_mounts=False, # Safer: sandbox has no access to mounted paths + ) + + # Client can reference sandbox port + client = GenerationClientScript(..., sandbox=sandbox) + """ + + cluster_config: Dict + port: Optional[int] = None + keep_mounts: bool = False + allocate_port: bool = True + env_overrides: Optional[List[str]] = None # Extra env vars in KEY=VALUE form + log_prefix: str = field(default="sandbox", init=False) + + def __post_init__(self): + """Initialize sandbox script. + + - Allocates port if not provided + - Builds sandbox command using sandbox_command() + - Sets self.inline to a callable that returns command and environment vars + """ + # Allocate port if not provided + if self.port is None and self.allocate_port: + self.port = get_free_port(strategy="random") + LOG.debug(f"Allocated port {self.port} for sandbox") + + # Build sandbox command and metadata (including environment vars) + # Note: keep_mounts is handled at the executor level, not in the command itself + cmd, metadata = sandbox_command( + cluster_config=self.cluster_config, + port=self.port, + ) + + # Use a callable to return both command and environment variables + # This ensures the sandbox's LISTEN_PORT and NGINX_PORT are properly set + def build_cmd() -> Tuple[str, Dict]: + env = dict(metadata.get("environment", {})) + # Apply user-specified environment overrides + if self.env_overrides: + for override in self.env_overrides: + key, value = override.split("=", 1) + env[key] = value + return cmd, {"environment": env} + + self.set_inline(build_cmd) + super().__post_init__() + + +@dataclass(kw_only=True) +class GenerationClientScript(BaseJobScript): + """Script for LLM generation/inference client. + + This script wraps generation command builders and provides: + - Cross-component references to multiple servers and sandbox + - Lazy command building for runtime hostname resolution + - Type-safe generation configuration + - Environment variable handling for sandbox/server communication + + Attributes: + output_dir: Directory for output files + input_file: Input JSONL file (mutually exclusive with input_dir) + input_dir: Input directory (mutually exclusive with input_file) + extra_arguments: Additional arguments for generation script + random_seed: Random seed for sampling (optional) + chunk_id: Chunk ID for parallel processing (optional) + num_chunks: Total number of chunks (required if chunk_id set) + preprocess_cmd: Command to run before generation (optional) + postprocess_cmd: Command to run after generation (optional) + wandb_parameters: WandB logging configuration (optional) + with_sandbox: Whether sandbox is enabled + script: Module or file path for generation script (default: nemo_skills.inference.generate) + servers: List of ServerScript references (None for pre-hosted servers) + server_addresses_prehosted: Addresses for pre-hosted servers (parallel to servers list) + model_names: Model names for multi-model generation (optional) + server_types: Server types for multi-model generation (optional) + sandbox: Reference to SandboxScript for cross-component communication (optional) + log_prefix: Prefix for log files (default: "main") + + Examples: + # Single server + client = GenerationClientScript( + output_dir="/results", + input_file="/data/input.jsonl", + servers=[server_script], + model_names=["llama-8b"], + server_types=["vllm"], + ) + + # Multi-model with self-hosted and pre-hosted servers + client = GenerationClientScript( + output_dir="/results", + input_file="/data/input.jsonl", + servers=[server1, server2, None], # None = pre-hosted + server_addresses_prehosted=["", "", "https://api.openai.com"], + model_names=["llama-8b", "llama-70b", "gpt-4"], + server_types=["vllm", "vllm", "openai"], + sandbox=sandbox_script, + with_sandbox=True, + ) + """ + + output_dir: str + input_file: Optional[str] = None + input_dir: Optional[str] = None + extra_arguments: str = "" + random_seed: Optional[int] = None + chunk_id: Optional[int] = None + num_chunks: Optional[int] = None + preprocess_cmd: Optional[str] = None + postprocess_cmd: Optional[str] = None + wandb_parameters: Optional[Dict] = None + with_sandbox: bool = False + script: str = "nemo_skills.inference.generate" + + # Cross-component references for single/multi-model + servers: Optional[List[Optional["ServerScript"]]] = None + server_addresses_prehosted: Optional[List[str]] = None + model_names: Optional[List[str]] = None + server_types: Optional[List[str]] = None + sandbox: Optional["SandboxScript"] = None + + log_prefix: str = field(default="main", init=False) + + def __post_init__(self): + """Initialize generation client script with lazy command building. + + Builds command lazily via a callable that is evaluated when het_group_index + is assigned, allowing hostname_ref() to resolve correctly for heterogeneous jobs. + + This works for both cases: + - With cross-refs: Resolves server hostnames and sandbox ports at runtime + - Without cross-refs: Just builds the command string (no runtime resolution needed) + """ + + def build_cmd() -> Tuple[str, Dict]: + """Build command at runtime when cross-refs are resolved.""" + env_vars = {} + + # Add sandbox port to environment if sandbox is referenced + if self.sandbox: + env_vars["NEMO_SKILLS_SANDBOX_PORT"] = str(self.sandbox.port) + + # Build server addresses if servers are provided + server_addresses = None + if self.servers is not None: + server_addresses = [] + for server_idx, server_script in enumerate(self.servers): + if server_script is not None: + # Self-hosted: construct address from hostname and port refs + addr = f"{server_script.hostname_ref()}:{server_script.port}" + else: + # Pre-hosted: use the address from server_addresses_prehosted + addr = self.server_addresses_prehosted[server_idx] + server_addresses.append(addr) + + # Build generation command + cmd = get_generation_cmd( + output_dir=self.output_dir, + input_file=self.input_file, + input_dir=self.input_dir, + extra_arguments=self.extra_arguments, + random_seed=self.random_seed, + chunk_id=self.chunk_id, + num_chunks=self.num_chunks, + preprocess_cmd=self.preprocess_cmd, + postprocess_cmd=self.postprocess_cmd, + wandb_parameters=self.wandb_parameters, + with_sandbox=self.with_sandbox, + script=self.script, + # Multi-model parameters (None for single-model) + server_addresses=server_addresses, + model_names=self.model_names, + server_types=self.server_types, + ) + + # Return command and runtime metadata (environment vars) + return cmd, {"environment": env_vars} + + # Always use lazy command building + self.set_inline(build_cmd) + super().__post_init__() diff --git a/tests/gpu-tests/test_eval.py b/tests/gpu-tests/test_eval.py index 31c8f2cccf..91d148877e 100644 --- a/tests/gpu-tests/test_eval.py +++ b/tests/gpu-tests/test_eval.py @@ -45,6 +45,7 @@ "mmau-pro", "asr-leaderboard", "aalcr", # Has tokenization mismatch issues + "mrcr", "audiobench", "librispeech-pc", } diff --git a/tests/test_declarative_pipeline.py b/tests/test_declarative_pipeline.py index 92117e403d..9d76fd4721 100644 --- a/tests/test_declarative_pipeline.py +++ b/tests/test_declarative_pipeline.py @@ -16,6 +16,7 @@ import json import os +from typing import Callable, Optional from unittest.mock import MagicMock, patch import pytest @@ -26,127 +27,83 @@ from nemo_skills.pipeline.utils.declarative import Command, CommandGroup, HardwareConfig, Pipeline -class TestCommand: - """Test Command class functionality.""" +class DummyScript: + """Minimal run.Script stand-in for unit tests.""" - def test_command_basic_string(self): - """Test creating a Command with a simple string.""" - cmd = Command(command="echo hello", name="test") - assert cmd.name == "test" - assert cmd.container == "nemo-skills" - assert cmd.gpus is None - assert cmd.nodes == 1 + def __init__(self, inline: str | Callable | None = "echo test"): + self.inline = inline + self.log_prefix = "main" + self.metadata = {} + self.het_group_index: Optional[int] = None - def test_command_with_metadata(self): - """Test Command with metadata passed separately.""" - cmd = Command(command="echo hello", name="server", metadata={"port": 8080, "log_prefix": "server"}) - assert cmd.metadata["port"] == 8080 - assert cmd.metadata["log_prefix"] == "server" - # Command gets wrapped with working_dir by default - assert "echo hello" in cmd.command + def set_inline(self, inline): + self.inline = inline - def test_command_with_callable(self): - """Test Command with callable that returns tuple.""" + def hostname_ref(self) -> str: + if self.het_group_index is None: + return "127.0.0.1" + return f"${{SLURM_MASTER_NODE_HET_GROUP_{self.het_group_index}:-localhost}}" - def make_cmd(): - return ("echo world", {"port": 5000}) - cmd = Command(command=make_cmd, name="dynamic") - assert callable(cmd.command) - assert cmd.name == "dynamic" +def make_command(*, inline: str | Callable | None = "echo test", name: str = "cmd", script: DummyScript | None = None): + """Helper to build Command objects with DummyScript instances.""" + script_obj = script or DummyScript(inline=inline) + return Command(script=script_obj, name=name) + + +class TestCommand: + """Tests for the new Script-based Command wrapper.""" + + def test_command_basic_script(self): + cmd = make_command(inline="echo hello", name="test") + assert cmd.name == "test" + assert cmd.container == "nemo-skills" + assert cmd.script.inline == "echo hello" def test_command_prepare_for_execution_string(self): - """Test prepare_for_execution with string command.""" - cmd = Command(command="python script.py", gpus=2, name="test") + cmd = make_command(inline="python script.py", name="test") cluster_config = {"executor": "local", "containers": {}} - final_cmd, exec_config = cmd.prepare_for_execution(cluster_config) + script_obj, exec_config = cmd.prepare_for_execution(cluster_config) - assert "python script.py" in final_cmd - assert exec_config["num_gpus"] == 2 - assert exec_config["num_nodes"] == 1 - assert exec_config["num_tasks"] == 1 + assert script_obj.inline == "python script.py" + assert exec_config["log_prefix"] == "main" + assert exec_config["environment"] == {} def test_command_prepare_for_execution_callable(self): - """Test prepare_for_execution with callable command.""" - - def make_cmd(): - return "echo test" - - cmd = Command(command=make_cmd, name="test") + script = DummyScript(inline=lambda: "echo test") + cmd = make_command(name="test", script=script) cluster_config = {"executor": "local", "containers": {}} - final_cmd, exec_config = cmd.prepare_for_execution(cluster_config) - - assert final_cmd == "echo test" + script_obj, _ = cmd.prepare_for_execution(cluster_config) + assert script_obj.inline == "echo test" def test_command_prepare_for_execution_callable_with_metadata(self): - """Test prepare_for_execution with callable returning tuple.""" - def make_cmd(): - return ("echo metadata", {"num_tasks": 4, "environment": {"VAR": "value"}}) + return ("echo metadata", {"environment": {"VAR": "value"}}) - cmd = Command(command=make_cmd, name="test") + script = DummyScript(inline=make_cmd) + cmd = make_command(name="test", script=script) cluster_config = {"executor": "local", "containers": {}} - final_cmd, exec_config = cmd.prepare_for_execution(cluster_config) + _, exec_config = cmd.prepare_for_execution(cluster_config) - assert final_cmd == "echo metadata" - assert exec_config["num_tasks"] == 4 assert exec_config["environment"]["VAR"] == "value" - def test_command_meta_ref(self): - """Test meta_ref for accessing metadata.""" - cmd = Command(command="echo test", name="server", metadata={"port": 8080, "host": "localhost"}) - - assert cmd.meta_ref("port") == "8080" - assert cmd.meta_ref("host") == "localhost" - - def test_command_meta_ref_missing_key(self): - """Test meta_ref with missing key raises KeyError.""" - cmd = Command(command="echo test", name="test") - - with pytest.raises(KeyError, match="Metadata key 'port' not found"): - cmd.meta_ref("port") - def test_command_hostname_ref_none(self): - """Test hostname_ref returns localhost when het_group_index is None.""" - cmd = Command(command="echo test", name="test") - assert cmd.het_group_index is None - assert cmd.hostname_ref() == "127.0.0.1" - - def test_command_hostname_ref_heterogeneous(self): - """Test hostname_ref returns SLURM variable when het_group_index is set.""" - cmd = Command(command="echo test", name="test") - cmd.het_group_index = 2 - - hostname = cmd.hostname_ref() - assert "$SLURM_JOB_NODELIST_HET_GROUP_2" in hostname - assert "scontrol" in hostname - - def test_command_with_installation_command(self): - """Test Command with installation_command.""" - cmd = Command(command="python script.py", installation_command="pip install package", name="test") - cluster_config = {"executor": "local", "containers": {}} - - final_cmd, _ = cmd.prepare_for_execution(cluster_config) + script = DummyScript() + cmd = make_command(name="test", script=script) - # Installation command should be wrapped around the main command - assert "pip install package" in final_cmd - assert "python script.py" in final_cmd + assert script.hostname_ref() == "127.0.0.1" + assert cmd.get_name() == "test" - def test_command_env_vars_wrapping(self): - """Test that env_vars and working_dir are applied to string commands.""" - cmd = Command( - command="python script.py", - env_vars={"MY_VAR": "value"}, - working_dir="/custom/path", - name="test", - ) + def test_command_hostname_ref_heterogeneous(self): + script = DummyScript() + script.het_group_index = 2 + make_command(name="test", script=script) - # The command should be wrapped with env setup - assert "export MY_VAR=value" in cmd.command - assert "cd /custom/path" in cmd.command + hostname = script.hostname_ref() + assert "${SLURM_MASTER_NODE_HET_GROUP_2" in hostname class TestCommandGroup: @@ -154,8 +111,8 @@ class TestCommandGroup: def test_commandgroup_basic(self): """Test creating a basic CommandGroup.""" - cmd1 = Command(command="echo 1", name="cmd1") - cmd2 = Command(command="echo 2", name="cmd2") + cmd1 = make_command(inline="echo 1", name="cmd1") + cmd2 = make_command(inline="echo 2", name="cmd2") group = CommandGroup(commands=[cmd1, cmd2], name="test_group") @@ -165,7 +122,7 @@ def test_commandgroup_basic(self): def test_commandgroup_with_hardware(self): """Test CommandGroup with HardwareConfig.""" - cmd = Command(command="echo test", name="cmd") + cmd = make_command(inline="echo test", name="cmd") hardware = HardwareConfig(partition="batch", sbatch_kwargs={"time_min": "01:00:00"}, num_gpus=8) group = CommandGroup(commands=[cmd], hardware=hardware, name="gpu_group") @@ -176,7 +133,7 @@ def test_commandgroup_with_hardware(self): def test_commandgroup_with_log_dir(self): """Test CommandGroup with log_dir.""" - cmd = Command(command="echo test", name="cmd") + cmd = make_command(inline="echo test", name="cmd") group = CommandGroup(commands=[cmd], log_dir="/logs/test", name="group") assert group.log_dir == "/logs/test" @@ -187,7 +144,7 @@ class TestPipeline: def test_pipeline_with_single_job(self): """Test Pipeline with single job.""" - cmd = Command(command="echo test", name="cmd") + cmd = make_command(inline="echo test", name="cmd") group = CommandGroup(commands=[cmd], name="group") cluster_config = {"executor": "local", "containers": {}} @@ -204,10 +161,10 @@ def test_pipeline_with_single_job(self): def test_pipeline_with_jobs(self): """Test Pipeline with jobs parameter (full format with dependencies).""" - cmd1 = Command(command="echo 1", name="cmd1") + cmd1 = make_command(inline="echo 1", name="cmd1") group1 = CommandGroup(commands=[cmd1], name="group1", log_dir="/logs") - cmd2 = Command(command="echo 2", name="cmd2") + cmd2 = make_command(inline="echo 2", name="cmd2") group2 = CommandGroup(commands=[cmd2], name="group2", log_dir="/logs") job1 = {"name": "job1", "group": group1} @@ -232,7 +189,7 @@ def test_pipeline_requires_jobs(self): def test_pipeline_with_run_after(self): """Test Pipeline with run_after parameter.""" - cmd = Command(command="echo test", name="cmd") + cmd = make_command(inline="echo test", name="cmd") group = CommandGroup(commands=[cmd], name="group") cluster_config = {"executor": "local", "containers": {}} @@ -248,7 +205,7 @@ def test_pipeline_with_run_after(self): def test_pipeline_with_run_after_list(self): """Test Pipeline with run_after as list.""" - cmd = Command(command="echo test", name="cmd") + cmd = make_command(inline="echo test", name="cmd") group = CommandGroup(commands=[cmd], name="group") cluster_config = {"executor": "local", "containers": {}} @@ -264,7 +221,7 @@ def test_pipeline_with_run_after_list(self): def test_pipeline_cluster_config_passed_directly(self): """Test that cluster_config is passed directly (no more string resolution).""" - cmd = Command(command="echo test", name="cmd") + cmd = make_command(inline="echo test", name="cmd") group = CommandGroup(commands=[cmd], name="group") cluster_config = {"executor": "local", "containers": {}} @@ -299,7 +256,7 @@ def test_pipeline_run_basic(self, mock_run_exp, mock_env_vars, mock_get_exp): mock_get_exp.return_value.__enter__.return_value = mock_exp # Create pipeline - cmd = Command(command="echo test", name="cmd") + cmd = make_command(inline="echo test", name="cmd") group = CommandGroup(commands=[cmd], name="group", log_dir="/logs") pipeline = Pipeline( name="test", cluster_config=mock_config, jobs=[{"name": "job1", "group": group}], skip_hf_home_check=True @@ -329,10 +286,10 @@ def test_pipeline_run_with_dependencies(self, mock_run_exp, mock_env_vars, mock_ mock_get_exp.return_value.__enter__.return_value = mock_exp # Create pipeline with internal dependencies - cmd1 = Command(command="echo 1", name="cmd1") + cmd1 = make_command(inline="echo 1", name="cmd1") group1 = CommandGroup(commands=[cmd1], name="group1", log_dir="/logs") - cmd2 = Command(command="echo 2", name="cmd2") + cmd2 = make_command(inline="echo 2", name="cmd2") group2 = CommandGroup(commands=[cmd2], name="group2", log_dir="/logs") job1 = {"name": "job1", "group": group1, "dependencies": []} @@ -375,7 +332,7 @@ def test_pipeline_hf_home_validation(self, mock_get_executor, mock_is_mounted, m mock_exp.add.return_value = "handle" mock_get_exp.return_value.__enter__.return_value = mock_exp - cmd = Command(command="echo test", name="cmd") + cmd = make_command(inline="echo test", name="cmd") group = CommandGroup(commands=[cmd], name="group", log_dir="/logs") pipeline = Pipeline(name="test", cluster_config=mock_config, jobs=[{"name": "job1", "group": group}]) @@ -391,7 +348,7 @@ def test_pipeline_hf_home_missing(self, mock_env_vars): mock_config = {"executor": "slurm", "containers": {}} mock_env_vars.return_value = {} # No HF_HOME - cmd = Command(command="echo test", name="cmd") + cmd = make_command(inline="echo test", name="cmd") group = CommandGroup(commands=[cmd], name="group", log_dir="/logs") # Should raise in __init__ now, not run() @@ -406,7 +363,7 @@ def test_pipeline_hf_home_not_mounted(self, mock_is_mounted, mock_env_vars): mock_env_vars.return_value = {"HF_HOME": "/hf"} mock_is_mounted.return_value = False - cmd = Command(command="echo test", name="cmd") + cmd = make_command(inline="echo test", name="cmd") group = CommandGroup(commands=[cmd], name="group", log_dir="/logs") # Should raise in __init__ now, not run() @@ -432,8 +389,8 @@ def test_het_group_index_non_heterogeneous(self, mock_env_vars, mock_get_exp): mock_get_exp.return_value.__enter__.return_value = mock_exp # Create single-group job with multiple components - cmd1 = Command(command="echo 1", name="cmd1") - cmd2 = Command(command="echo 2", name="cmd2") + cmd1 = make_command(inline="echo 1", name="cmd1") + cmd2 = make_command(inline="echo 2", name="cmd2") group = CommandGroup(commands=[cmd1, cmd2], name="group", log_dir="/logs") pipeline = Pipeline( @@ -442,10 +399,10 @@ def test_het_group_index_non_heterogeneous(self, mock_env_vars, mock_get_exp): pipeline.run(dry_run=True) # Both commands should have None het_group_index (localhost communication) - assert cmd1.het_group_index is None - assert cmd2.het_group_index is None - assert cmd1.hostname_ref() == "127.0.0.1" - assert cmd2.hostname_ref() == "127.0.0.1" + assert cmd1.script.het_group_index is None + assert cmd2.script.het_group_index is None + assert cmd1.script.hostname_ref() == "127.0.0.1" + assert cmd2.script.hostname_ref() == "127.0.0.1" @patch("nemo_skills.pipeline.utils.declarative.get_exp") @patch("nemo_skills.pipeline.utils.declarative.get_env_variables") @@ -462,10 +419,10 @@ def test_het_group_index_heterogeneous(self, mock_env_vars, mock_get_exp): mock_get_exp.return_value.__enter__.return_value = mock_exp # Create multi-group heterogeneous job - cmd1 = Command(command="echo 1", name="cmd1") + cmd1 = make_command(inline="echo 1", name="cmd1") group1 = CommandGroup(commands=[cmd1], name="group1", log_dir="/logs") - cmd2 = Command(command="echo 2", name="cmd2") + cmd2 = make_command(inline="echo 2", name="cmd2") group2 = CommandGroup(commands=[cmd2], name="group2", log_dir="/logs") jobs = [{"name": "hetjob", "groups": [group1, group2]}] @@ -473,10 +430,10 @@ def test_het_group_index_heterogeneous(self, mock_env_vars, mock_get_exp): pipeline.run(dry_run=True) # Commands should have het_group_index 0 and 1 - assert cmd1.het_group_index == 0 - assert cmd2.het_group_index == 1 - assert "$SLURM_JOB_NODELIST_HET_GROUP_0" in cmd1.hostname_ref() - assert "$SLURM_JOB_NODELIST_HET_GROUP_1" in cmd2.hostname_ref() + assert cmd1.script.het_group_index == 0 + assert cmd2.script.het_group_index == 1 + assert "SLURM_MASTER_NODE_HET_GROUP_0" in cmd1.script.hostname_ref() + assert "SLURM_MASTER_NODE_HET_GROUP_1" in cmd2.script.hostname_ref() @patch("nemo_skills.pipeline.utils.declarative.get_exp") @patch("nemo_skills.pipeline.utils.declarative.get_env_variables") @@ -493,16 +450,16 @@ def test_het_group_index_per_job_not_global(self, mock_env_vars, mock_get_exp): mock_get_exp.return_value.__enter__.return_value = mock_exp # Create two separate heterogeneous jobs - cmd1 = Command(command="echo 1", name="cmd1") + cmd1 = make_command(inline="echo 1", name="cmd1") group1 = CommandGroup(commands=[cmd1], name="group1", log_dir="/logs") - cmd2 = Command(command="echo 2", name="cmd2") + cmd2 = make_command(inline="echo 2", name="cmd2") group2 = CommandGroup(commands=[cmd2], name="group2", log_dir="/logs") - cmd3 = Command(command="echo 3", name="cmd3") + cmd3 = make_command(inline="echo 3", name="cmd3") group3 = CommandGroup(commands=[cmd3], name="group3", log_dir="/logs") - cmd4 = Command(command="echo 4", name="cmd4") + cmd4 = make_command(inline="echo 4", name="cmd4") group4 = CommandGroup(commands=[cmd4], name="group4", log_dir="/logs") jobs = [ @@ -513,10 +470,10 @@ def test_het_group_index_per_job_not_global(self, mock_env_vars, mock_get_exp): pipeline.run(dry_run=True) # Both jobs should have het_group_index starting from 0 - assert cmd1.het_group_index == 0 - assert cmd2.het_group_index == 1 - assert cmd3.het_group_index == 0 # Starts from 0 again! - assert cmd4.het_group_index == 1 + assert cmd1.script.het_group_index == 0 + assert cmd2.script.het_group_index == 1 + assert cmd3.script.het_group_index == 0 # Starts from 0 again! + assert cmd4.script.het_group_index == 1 class TestDependencyResolution: @@ -536,7 +493,7 @@ def test_dependency_none_handling(self, mock_env_vars, mock_get_exp): mock_exp.add.return_value = "handle" mock_get_exp.return_value.__enter__.return_value = mock_exp - cmd = Command(command="echo test", name="cmd") + cmd = make_command(inline="echo test", name="cmd") group = CommandGroup(commands=[cmd], name="group", log_dir="/logs") jobs = [{"name": "job", "group": group, "dependencies": None}] @@ -559,7 +516,7 @@ def test_pipeline_run_after_applies_to_jobs(self, mock_env_vars, mock_get_exp): mock_exp.add.return_value = "handle" mock_get_exp.return_value.__enter__.return_value = mock_exp - cmd = Command(command="echo test", name="cmd") + cmd = make_command(inline="echo test", name="cmd") group = CommandGroup(commands=[cmd], name="group", log_dir="/logs") pipeline = Pipeline( @@ -589,7 +546,7 @@ def test_pipeline_job_missing_group_or_groups(self): def test_commandgroup_missing_log_dir(self): """Test that CommandGroup without log_dir raises error during execution.""" mock_config = {"executor": "none", "containers": {}} - cmd = Command(command="echo test", name="cmd") + cmd = make_command(inline="echo test", name="cmd") group = CommandGroup(commands=[cmd], name="group") # No log_dir pipeline = Pipeline(name="test", cluster_config=mock_config, jobs=[{"name": "job1", "group": group}]) @@ -626,14 +583,14 @@ def test_multiple_internal_dependencies(self): } # Job 1 and Job 2: independent - cmd1 = Command(command="echo job1", name="job1") + cmd1 = make_command(inline="echo job1", name="job1") group1 = CommandGroup(commands=[cmd1], name="group1", log_dir="/tmp/logs") - cmd2 = Command(command="echo job2", name="job2") + cmd2 = make_command(inline="echo job2", name="job2") group2 = CommandGroup(commands=[cmd2], name="group2", log_dir="/tmp/logs") # Job 3: depends on both job1 and job2 - cmd3 = Command(command="echo job3", name="job3") + cmd3 = make_command(inline="echo job3", name="job3") group3 = CommandGroup(commands=[cmd3], name="group3", log_dir="/tmp/logs") job1_spec = {"name": "job1", "group": group1} @@ -715,11 +672,11 @@ def mock_get_executor(**kwargs): } # Job 1: depends on external experiment - cmd1 = Command(command="echo job1", name="job1") + cmd1 = make_command(inline="echo job1", name="job1") group1 = CommandGroup(commands=[cmd1], name="group1", log_dir="/tmp/logs") # Job 2: depends on job1 (internal) AND external experiment - cmd2 = Command(command="echo job2", name="job2") + cmd2 = make_command(inline="echo job2", name="job2") group2 = CommandGroup(commands=[cmd2], name="group2", log_dir="/tmp/logs") job1_spec = { @@ -931,35 +888,38 @@ def capture_env_update(cluster_config, updates): # Debug: print what we captured print(f"Captured env updates: {env_updates_captured}") - # Find the client and sandbox environment updates - client_env = None - sandbox_env = None + # Verify both sandbox and client environment variables are captured + assert len(env_updates_captured) >= 2, ( + f"Expected at least 2 environment updates (sandbox + client), got {len(env_updates_captured)}: {env_updates_captured}" + ) + # Find the sandbox and client environment updates + sandbox_env = None + client_env = None for env_update in env_updates_captured: + if "LISTEN_PORT" in env_update and "NGINX_PORT" in env_update: + sandbox_env = env_update if "NEMO_SKILLS_SANDBOX_PORT" in env_update: client_env = env_update - elif "LISTEN_PORT" in env_update and "NGINX_PORT" in env_update: - sandbox_env = env_update - # Verify client got NEMO_SKILLS_SANDBOX_PORT (old behavior: exp.py line 493) - # This is the key fix - ensuring sandbox port is passed to client - assert client_env is not None, ( - f"Client environment update not found. Captured updates: {env_updates_captured}\n" - f"This means NEMO_SKILLS_SANDBOX_PORT was not set for the client command, " - f"so the Sandbox class cannot connect to the sandbox server." + # Verify sandbox got LISTEN_PORT and NGINX_PORT + assert sandbox_env is not None, ( + f"LISTEN_PORT/NGINX_PORT not set for sandbox command: {env_updates_captured}" ) - assert "NEMO_SKILLS_SANDBOX_PORT" in client_env, ( - "NEMO_SKILLS_SANDBOX_PORT not set for client command" + assert sandbox_env["LISTEN_PORT"] == sandbox_env["NGINX_PORT"], ( + f"LISTEN_PORT and NGINX_PORT should match: {sandbox_env}" ) - # Verify sandbox got its environment vars (old behavior: exp.py lines 525-538) - assert sandbox_env is not None, ( - f"Sandbox environment update not found. Captured: {env_updates_captured}" + # Verify client got NEMO_SKILLS_SANDBOX_PORT + assert client_env is not None, ( + f"NEMO_SKILLS_SANDBOX_PORT not set for client command: {env_updates_captured}" ) - assert "LISTEN_PORT" in sandbox_env, "LISTEN_PORT not set for sandbox" - assert "NGINX_PORT" in sandbox_env, "NGINX_PORT not set for sandbox" - # This test verifies the fix works end-to-end through the actual generate() function + # Verify the ports match between sandbox and client + assert client_env["NEMO_SKILLS_SANDBOX_PORT"] == sandbox_env["LISTEN_PORT"], ( + f"Sandbox port mismatch: client has {client_env['NEMO_SKILLS_SANDBOX_PORT']}, " + f"sandbox has {sandbox_env['LISTEN_PORT']}" + ) if __name__ == "__main__": diff --git a/tests/test_generation.py b/tests/test_generation.py index b69b526a0e..2693d62241 100644 --- a/tests/test_generation.py +++ b/tests/test_generation.py @@ -16,12 +16,12 @@ # running most things through subprocess since that's how it's usually used import subprocess -from unittest.mock import MagicMock import pytest from nemo_skills.evaluation.metrics import ComputeMetrics -from nemo_skills.pipeline.generate import _create_commandgroup_from_config +from nemo_skills.pipeline.generate import _create_job_unified +from nemo_skills.pipeline.utils.scripts import ServerScript def test_eval_gsm8k_api(tmp_path): @@ -153,36 +153,42 @@ def test_generate_openai_format(tmp_path, format): assert len(data[1]["generation"]) > 0 -def test_server_metadata_from_num_tasks(): +def test_server_metadata_from_num_tasks(tmp_path): """Test that metadata dict is properly created from server command returning (cmd, num_tasks).""" - mock_server_fn = MagicMock(return_value=("python server.py", 4)) cluster_config = { - "containers": {"vllm": "nvcr.io/nvidia/nemo:vllm", "nemo-skills": "nvcr.io/nvidia/nemo:skills"}, - "executor": "slurm", + "containers": { + "vllm": "apitest/vllm", + "nemo-skills": "apitest/nemo-skills", + "sandbox": "apitest/sandbox", + }, + "executor": "none", } server_config = { "server_type": "vllm", "num_gpus": 8, "num_nodes": 1, - "model_path": "/models/test", + "model_path": str(tmp_path / "model"), "server_port": 5000, + "server_args": "", } + generation_params = {"output_dir": "/tmp/out"} - cmd_group = _create_commandgroup_from_config( - generation_cmd="python generate.py", - server_config=server_config, - with_sandbox=False, - sandbox_port=None, + groups = _create_job_unified( + models=[server_config["model_path"]], + server_configs=[server_config], + generation_params=generation_params, cluster_config=cluster_config, installation_command=None, - get_server_command_fn=mock_server_fn, + with_sandbox=False, partition=None, keep_mounts_for_sandbox=False, task_name="test-task", log_dir="/tmp/logs", ) - server_cmd = cmd_group.commands[0] - assert isinstance(server_cmd.metadata, dict) - assert server_cmd.metadata["num_tasks"] == 4 - assert server_cmd.metadata["gpus"] == 8 + server_cmd = groups[0].commands[0] + assert isinstance(server_cmd.script, ServerScript) + assert server_cmd.script.num_tasks >= 1 + assert server_cmd.script.num_gpus == server_config["num_gpus"] + assert groups[0].hardware.num_gpus == server_config["num_gpus"] + assert groups[0].hardware.num_tasks == server_cmd.script.num_tasks diff --git a/tests/test_nemo_evaluator_pipeline.py b/tests/test_nemo_evaluator_pipeline.py index 22ac250882..0f333ab748 100644 --- a/tests/test_nemo_evaluator_pipeline.py +++ b/tests/test_nemo_evaluator_pipeline.py @@ -17,8 +17,14 @@ import pytest -from nemo_skills.pipeline.nemo_evaluator import nemo_evaluator as nemo_evaluator_fn +from nemo_skills.pipeline.nemo_evaluator import ( + EvaluatorClientScript, +) +from nemo_skills.pipeline.nemo_evaluator import ( + nemo_evaluator as nemo_evaluator_fn, +) from nemo_skills.pipeline.utils.declarative import Command, CommandGroup +from nemo_skills.pipeline.utils.scripts import ServerScript @pytest.fixture @@ -131,9 +137,8 @@ def test_no_servers_external_urls( # Verify client command client_cmd = group.commands[0] assert isinstance(client_cmd, Command) - assert "evaluator-test-0" in client_cmd.name - assert client_cmd.gpus is None # No GPUs when no hosted servers - assert client_cmd.nodes == 1 + assert client_cmd.name.startswith("evaluator-test-client-0") + assert isinstance(client_cmd.script, EvaluatorClientScript) # Verify hardware config assert group.hardware is not None @@ -181,16 +186,17 @@ def test_main_server_hosted( server_cmd = group.commands[0] assert isinstance(server_cmd, Command) assert "server" in server_cmd.name - assert server_cmd.gpus == 8 - assert server_cmd.nodes == 1 - assert "port" in server_cmd.metadata - assert server_cmd.metadata["log_prefix"] == "server" + assert isinstance(server_cmd.script, ServerScript) + assert server_cmd.script.num_gpus == 8 + assert server_cmd.script.log_prefix == "server" + assert server_cmd.script.port is not None # Verify client command client_cmd = group.commands[1] assert isinstance(client_cmd, Command) assert "client" in client_cmd.name - assert callable(client_cmd.command) # Should be lambda for cross-component refs + assert isinstance(client_cmd.script, EvaluatorClientScript) + assert callable(client_cmd.script.inline) # Should be lambda for cross-component refs # Verify hardware config (should use server GPUs) assert group.hardware.num_gpus == 8 @@ -235,14 +241,16 @@ def test_judge_server_hosted( judge_cmd = group.commands[0] assert isinstance(judge_cmd, Command) assert "judge-server" in judge_cmd.name - assert judge_cmd.gpus == 32 - assert judge_cmd.metadata["log_prefix"] == "judge-server" + assert isinstance(judge_cmd.script, ServerScript) + assert judge_cmd.script.num_gpus == 32 + assert judge_cmd.script.log_prefix == "judge-server" # Verify client command client_cmd = group.commands[1] assert isinstance(client_cmd, Command) assert "client" in client_cmd.name - assert callable(client_cmd.command) # Should be lambda for cross-component refs + assert isinstance(client_cmd.script, EvaluatorClientScript) + assert callable(client_cmd.script.inline) # Should be lambda for cross-component refs # Verify hardware config (should use judge server GPUs) assert group.hardware.num_gpus == 32 @@ -300,19 +308,22 @@ def test_both_servers_hosted_separate_groups( server_cmd = server_group.commands[0] assert isinstance(server_cmd, Command) assert "server" in server_cmd.name - assert server_cmd.gpus == 8 + assert isinstance(server_cmd.script, ServerScript) + assert server_cmd.script.num_gpus == 8 # Verify client command in first group client_cmd = server_group.commands[1] assert isinstance(client_cmd, Command) assert "client" in client_cmd.name - assert callable(client_cmd.command) # Lambda for cross-component refs + assert isinstance(client_cmd.script, EvaluatorClientScript) + assert callable(client_cmd.script.inline) # Lambda for cross-component refs # Verify judge server command in second group judge_cmd = judge_group.commands[0] assert isinstance(judge_cmd, Command) assert "judge-server" in judge_cmd.name - assert judge_cmd.gpus == 32 + assert isinstance(judge_cmd.script, ServerScript) + assert judge_cmd.script.num_gpus == 32 @patch("nemo_skills.pipeline.nemo_evaluator.Pipeline") From 5d4cb8e6bea3418148534967a6b528bfefcdb9bb Mon Sep 17 00:00:00 2001 From: Sean Naren Date: Wed, 17 Dec 2025 11:00:24 +0000 Subject: [PATCH 58/88] Port ICPC changes to IOI (#1046) Signed-off-by: SeanNaren Co-authored-by: Mehrzad Samadi Signed-off-by: Cheng-Ping Hsieh --- .github/workflows/tests.yml | 5 +- docs/evaluation/code.md | 18 +- .../dataset/{ioi24 => ioi}/__init__.py | 0 nemo_skills/dataset/{ioi24 => ioi}/prepare.py | 5 +- nemo_skills/dataset/ioi25/__init__.py | 31 --- nemo_skills/evaluation/evaluator/ioi.py | 259 ++++++++++++------ nemo_skills/evaluation/metrics/ioi_metrics.py | 156 ++++++++--- 7 files changed, 306 insertions(+), 168 deletions(-) rename nemo_skills/dataset/{ioi24 => ioi}/__init__.py (100%) rename nemo_skills/dataset/{ioi24 => ioi}/prepare.py (93%) delete mode 100644 nemo_skills/dataset/ioi25/__init__.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0908e9d58e..3c6cd2e0c7 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -85,7 +85,10 @@ jobs: NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} HF_TOKEN: ${{ secrets.HF_TOKEN }} run: | - docker run --rm --network=host nemo-skills-sandbox-image & + # Default shared runtime directory + sudo mkdir -p /nemo_run + sudo chmod 777 /nemo_run + docker run --rm --network=host -v /nemo_run:/nemo_run nemo-skills-sandbox-image & sleep 10 set -o pipefail # this will make sure next line returns non-0 exit code if tests fail ns prepare_data gsm8k math-500 diff --git a/docs/evaluation/code.md b/docs/evaluation/code.md index 5a8d634bcc..1b4a60a14e 100644 --- a/docs/evaluation/code.md +++ b/docs/evaluation/code.md @@ -185,10 +185,10 @@ We currently support IOI24 and are working to support IOI25 for evaluation. The #### Data Preparation -First, prepare the dataset by running the `ns prepare_data` command. The arguments below will generate `test.jsonl` and `test_metadata.json`. +First, prepare the dataset by running the `ns prepare_data` command. The arguments below will generate `ioi24.jsonl` and `ioi24_metadata.json`. ``` -ns prepare_data ioi24 +ns prepare_data ioi ``` #### Running the Evaluation @@ -209,10 +209,11 @@ ns eval \ --server_gpus=8 \ --benchmarks=ioi24:50 \ --with_sandbox \ - --split=test \ + --split=ioi24 \ --data_dir= \ --output_dir= \ - --extra_eval_args="++eval_config.test_file=" \ + --eval_subfolder=eval-results/ioi24/ \ # set the folder if you want to differentiate subsets. + --extra_eval_args="++eval_config.test_file=/ioi24_metadata.json" \ ++inference.temperature=0.6 \ ++inference.top_p=0.95 \ ++inference.tokens_to_generate=65536 @@ -220,13 +221,12 @@ ns eval \ ##### Verifying Results -After all jobs are complete, you can check the results in `/eval-results/ioi24/metrics.json`. You can also take a look at `/eval-results/ioi24/summarized-results/main_*`. They should look something like this: +After all jobs are complete, you can check the results in `/eval-results/ioi24/ioi/metrics.json`. You can also take a look at `/eval-results/ioi24/ioi/summarized-results/main_*`. They should look something like this: ``` ------------------------------------------------------- ioi24 ------------------------------------------------------ -evaluation_mode | num_entries | avg_tokens | gen_seconds | correct | total_score | round_robin_score -pass@1[avg-of-50] | 39 | 40387 | 7410 | 0.51% ± 1.04% | 303.47 | 261.01 -pass@50 | 39 | 40387 | 7410 | 2.56% | 303.47 | 261.01 +------------------------------------ ioi24 ------------------------------------- +evaluation_mode | num_entries | avg_tokens | gen_seconds | correct | total_score +pass@50 | 39 | 52225 | 99630 | 23.08% | 500 ``` ### livecodebench diff --git a/nemo_skills/dataset/ioi24/__init__.py b/nemo_skills/dataset/ioi/__init__.py similarity index 100% rename from nemo_skills/dataset/ioi24/__init__.py rename to nemo_skills/dataset/ioi/__init__.py diff --git a/nemo_skills/dataset/ioi24/prepare.py b/nemo_skills/dataset/ioi/prepare.py similarity index 93% rename from nemo_skills/dataset/ioi24/prepare.py rename to nemo_skills/dataset/ioi/prepare.py index 656e480b60..3849607b0f 100644 --- a/nemo_skills/dataset/ioi24/prepare.py +++ b/nemo_skills/dataset/ioi/prepare.py @@ -27,6 +27,7 @@ if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("--split", type=str, default="test") + parser.add_argument("--suffix", type=str, default="24") args = parser.parse_args() data_dir = Path(__file__).absolute().parent @@ -50,7 +51,7 @@ } ) - with open(os.path.join(data_dir, f"{args.split}.jsonl"), "w") as f: + with open(os.path.join(data_dir, f"ioi{args.suffix}.jsonl"), "w") as f: f.write("\n".join(json.dumps(x) for x in entries)) tests_dataset = load_dataset("open-r1/ioi-test-cases", name="2024", split="train") @@ -82,5 +83,5 @@ "grader_files": entry["grader_files"], } - with open(os.path.join(data_dir, f"{args.split}_metadata.json"), "w") as f: + with open(os.path.join(data_dir, f"ioi{args.suffix}_metadata.json"), "w") as f: json.dump(final_structure, f) diff --git a/nemo_skills/dataset/ioi25/__init__.py b/nemo_skills/dataset/ioi25/__init__.py deleted file mode 100644 index 3032b16653..0000000000 --- a/nemo_skills/dataset/ioi25/__init__.py +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -todo: We are working on providing the data files that are necessary to run IOI25 evaluation. -""" - -# settings that define how evaluation should be done by default (all can be changed from cmdline) -GENERATION_ARGS = "++prompt_config=generic/default ++eval_type=ioi" -DATASET_GROUP = "code" -METRICS_TYPE = "ioi" - -# environment variables required by this benchmark -SANDBOX_ENV_VARS = [ - "UWSGI_PROCESSES=1024", - "UWSGI_CPU_AFFINITY=8", - "UWSGI_CHEAPER=1023", - "NUM_WORKERS=1", - "STATEFUL_SANDBOX=0", -] diff --git a/nemo_skills/evaluation/evaluator/ioi.py b/nemo_skills/evaluation/evaluator/ioi.py index 239a23db6c..9d1738518b 100644 --- a/nemo_skills/evaluation/evaluator/ioi.py +++ b/nemo_skills/evaluation/evaluator/ioi.py @@ -12,23 +12,25 @@ # See the License for the specific language governing permissions and # limitations under the License. import asyncio +import hashlib import json import multiprocessing import os import re +import shutil import threading import time -from typing import Dict from nemo_skills.code_execution.sandbox import LocalSandbox from nemo_skills.evaluation.evaluator.base import BaseEvaluator, BaseEvaluatorConfig from nemo_skills.file_utils import jdump -from nemo_skills.utils import nested_dataclass +from nemo_skills.utils import nested_dataclass, unroll_files @nested_dataclass(kw_only=True) class IOIEvaluatorConfig(BaseEvaluatorConfig): test_file: str = "test_metadata.json" + input_file: str | None = None num_workers: int = 16 # number of test workers test_batch_size: int = 16 # number of tests to run concurrently overwrite: bool = False @@ -40,6 +42,10 @@ class IOIEvaluatorConfig(BaseEvaluatorConfig): asyncio.set_event_loop(worker_loop) +def sha256_hex(text: str) -> str: + return hashlib.sha256(text.encode("utf-8", errors="replace")).hexdigest() + + def _sandbox_exec_sync(sandbox: LocalSandbox, cmd: str, *, language: str = "shell", timeout: int = 120): """Run sandbox.execute_code synchronously with a persistent event loop. @@ -88,29 +94,31 @@ def _precompile_grader( wait_for_sandbox(sandbox) sandbox._owner_tid = threading.get_ident() - pre_dir = f"/tmp/ioi_pre_{problem_name}_{os.getpid()}" - # Build shell script to create files and invoke compile.sh. - creation_cmds = [ - f"mkdir -p {pre_dir}/graders", - ] - # Dump grader related files + pre_dir = f"/nemo_run/ioi_pre_{problem_name}_{os.getpid()}" + # Create directories and files locally; sandbox shares the same filesystem + os.makedirs(os.path.join(pre_dir, "graders"), exist_ok=True) + + # Dump grader related files locally for filepath, content in grader_files: - dir_name = os.path.dirname(filepath) - if dir_name: - creation_cmds.append(f"mkdir -p {pre_dir}/{dir_name}") - creation_cmds.append(f"cat <<'_EOT_' > {pre_dir}/{filepath}\n{content}\n_EOT_\n") - - # Write compile.sh and run.sh as provided (needed later in workers) - creation_cmds.append( - f"cat <<'_EOT_' > {pre_dir}/compile.sh\n{compile_code}\n_EOT_\nchmod +x {pre_dir}/compile.sh\n" - ) - creation_cmds.append(f"cat <<'_EOT_' > {pre_dir}/run.sh\n{run_code}\n_EOT_\nchmod +x {pre_dir}/run.sh\n") - - setup_script = "\n".join(creation_cmds) - # 1. create files - _sandbox_exec_sync(sandbox, setup_script, language="shell", timeout=120) - - # 2. run compile.sh but ignore final failure when problem cpp missing + target_path = os.path.join(pre_dir, filepath) + target_dir = os.path.dirname(target_path) + if target_dir: + os.makedirs(target_dir, exist_ok=True) + with open(target_path, "w", encoding="utf-8") as f: + f.write(content) + + # Write compile.sh and run.sh locally and make them executable + compile_path = os.path.join(pre_dir, "compile.sh") + with open(compile_path, "w", encoding="utf-8") as f: + f.write(compile_code) + os.chmod(compile_path, 0o755) + + run_path = os.path.join(pre_dir, "run.sh") + with open(run_path, "w", encoding="utf-8") as f: + f.write(run_code) + os.chmod(run_path, 0o755) + + # Run compile.sh inside the sandbox (same filesystem) _sandbox_exec_sync(sandbox, f"cd {pre_dir} && ./compile.sh || true", language="shell", timeout=120) return pre_dir @@ -118,46 +126,28 @@ def _precompile_grader( def run_test_case(task_args: dict, worker_id: int) -> dict: # Use high-resolution timestamp to guarantee uniqueness across parallel calls. - unique_dir = f"/tmp/ioi_run_{worker_id}_{os.getpid()}_{time.time_ns()}" + unique_dir = f"/nemo_run/ioi_run_{worker_id}_{os.getpid()}_{time.time_ns()}" try: - # 1. Create all necessary files in one batch command + # 1. Create all necessary files locally (sandbox shares filesystem) precompiled_dir = task_args.get("precompiled_dir") - # Step 1: prepare the working directory and copy shared pre-compiled artifacts first - file_creation_commands = [ - # Create the unique run directory itself - f"mkdir -p {unique_dir}", - # Ensure `graders/` directory exists - f"mkdir -p {unique_dir}/graders", - f"cp -r {precompiled_dir}/* {unique_dir}/", - # Next write the contestant's generated solution into the graders folder so it is not overwritten - f"cat <<'_EOT_' > {unique_dir}/graders/{task_args['problem_id']}.cpp\n{task_args['generated_code']}\n_EOT_\n", - ] - + os.makedirs(unique_dir, exist_ok=True) + os.makedirs(os.path.join(unique_dir, "graders"), exist_ok=True) + # Copy precompiled assets into unique run directory + if precompiled_dir and os.path.isdir(precompiled_dir): + shutil.copytree(precompiled_dir, unique_dir, dirs_exist_ok=True) + # Write contestant solution + with open(os.path.join(unique_dir, "graders", f"{task_args['problem_id']}.cpp"), "w", encoding="utf-8") as f: + f.write(task_args["generated_code"]) # Prepare input and expected output files - file_creation_commands.append(f"cat <<'_EOT_' > {unique_dir}/input.txt\n{task_args['test_input']}\n_EOT_\n") - file_creation_commands.append( - f"cat <<'_EOT_' > {unique_dir}/correct_output.txt\n{task_args['test_output']}\n_EOT_\n" - ) - - setup_script = "\n".join(file_creation_commands) - sandbox = LocalSandbox() - setup_result, _ = worker_loop.run_until_complete( - sandbox.execute_code(setup_script, language="shell", timeout=120) - ) - if setup_result.get("stderr"): - raise Exception(f"File setup failed: {setup_result['stderr']}") + with open(os.path.join(unique_dir, "input.txt"), "w", encoding="utf-8") as f: + f.write(task_args["test_input"]) + with open(os.path.join(unique_dir, "correct_output.txt"), "w", encoding="utf-8") as f: + f.write(task_args["test_output"]) # 2. Compile only the problem solution (skip checker/grader recompilation) - # Compile the solution together with optional grader/stub sources without - # recompiling the checker/manager again. - compile_command = ( - f"cd {unique_dir} && " - f'SRC="graders/{task_args["problem_id"]}.cpp"; ' - f'[ -e graders/grader.cpp ] && SRC="$SRC graders/grader.cpp"; ' - f'[ -e graders/stub.cpp ] && SRC="$SRC graders/stub.cpp"; ' - f"g++ -DEVAL -std=gnu++17 -O2 -pipe -s -o graders/{task_args['problem_id']} $SRC" - ) + compile_command = f"cd {unique_dir} && ./compile.sh" + sandbox = LocalSandbox() compile_result, _ = worker_loop.run_until_complete( sandbox.execute_code(compile_command, language="shell", timeout=120) ) @@ -202,11 +192,80 @@ def run_test_case(task_args: dict, worker_id: int) -> dict: return {"score": 0.0, "output": "", "error": str(e)} finally: - # 4. Clean up the directory - # Fire and forget; ignore return values + # 4. Clean up the directory locally + try: + shutil.rmtree(unique_dir, ignore_errors=True) + except Exception: + pass + + +def run_input_case(task_args: dict, worker_id: int) -> dict: + # Use high-resolution timestamp to guarantee uniqueness across parallel calls. + unique_dir = f"/nemo_run/ioi_run_{worker_id}_{os.getpid()}_{time.time_ns()}" + + try: + # 1. Create all necessary files locally (sandbox shares filesystem) + os.makedirs(unique_dir, exist_ok=True) + for filepath, content in task_args.get("run_files", []): + target_path = os.path.join(unique_dir, os.path.basename(filepath)) + with open(target_path, "w", encoding="utf-8") as f: + f.write(content) + for fname in ("compile", "run"): + fpath = os.path.join(unique_dir, fname) + if os.path.exists(fpath): + os.chmod(fpath, 0o755) + # Write contestant solution into problem solution file + solution_path = os.path.join(unique_dir, f"{task_args['problem_id']}.cpp") + with open(solution_path, "w", encoding="utf-8") as f: + f.write(task_args["generated_code"]) + # Prepare only input file (no ground-truth for input-only runs) + with open(os.path.join(unique_dir, "input.txt"), "w", encoding="utf-8") as f: + f.write(task_args["test_input"]) + + # 2. Compile using run_files toolchain + compile_command = f"cd {unique_dir} && ./compile" + sandbox = LocalSandbox() + compile_result, _ = worker_loop.run_until_complete( + sandbox.execute_code(compile_command, language="shell", timeout=120) + ) + + result = { + "compile_success": not compile_result.get("stderr"), + "compile_stdout": compile_result.get("stdout", ""), + "compile_stderr": compile_result.get("stderr", ""), + "run_stdout": "", + "run_stderr": "", + "error": "", + } + + if not result["compile_success"]: + return result + + # 3. Run the code using run_files runner + run_command = f"cd {unique_dir} && ./run < input.txt" + run_result, _ = worker_loop.run_until_complete( + sandbox.execute_code(run_command, language="shell", timeout=120, max_output_characters=1000000) + ) + + run_stdout = sha256_hex(run_result.get("stdout", "")) + run_stderr = run_result.get("stderr", "") + + result.update( + { + "run_stdout": run_stdout, + "run_stderr": run_stderr, + } + ) + + return result + + except Exception as e: + return {"run_stdout": "", "run_stderr": "", "error": str(e)} + + finally: + # 4. Clean up the directory locally try: - sandbox = LocalSandbox() - worker_loop.run_until_complete(sandbox.execute_code(f"rm -rf {unique_dir}", language="shell", timeout=120)) + shutil.rmtree(unique_dir, ignore_errors=True) except Exception: pass @@ -250,10 +309,11 @@ def __init__(self, config: dict, num_parallel_requests: int = 10): self.eval_cfg = IOIEvaluatorConfig(_init_nested=True, **config) # Heavy runtime resources are lazily initialized within _evaluate_entry. - self.sandbox = None # type: ignore - self.metadata = None # type: ignore - self.precompiled_cache: Dict[str, str] = {} - self.pool = None # type: ignore + self.sandbox = None + self.metadata = None + self.inputdata = None + self.precompiled_cache = {} + self.pool = None async def _initialize_runtime(self): """Asynchronously create sandbox and related runtime state on first use.""" @@ -275,14 +335,23 @@ def _setup(): ) with open(self.eval_cfg.test_file, "r") as f: metadata_local = json.load(f) + input_local = None + if self.eval_cfg.input_file: + if not os.path.exists(self.eval_cfg.input_file): + raise FileNotFoundError( + f"Input file {self.eval_cfg.input_file} does not exist." + " Please provide a valid parameter for ++eval_config.input_file=x when running IOI Evaluation." + ) + with open(self.eval_cfg.input_file, "r") as f: + input_local = json.load(f) pool_local = multiprocessing.Pool( processes=self.eval_cfg.test_batch_size, initializer=init_worker, ) - return sbox, metadata_local, pool_local + return sbox, metadata_local, input_local, pool_local - self.sandbox, self.metadata, self.pool = await asyncio.to_thread(_setup) + self.sandbox, self.metadata, self.inputdata, self.pool = await asyncio.to_thread(_setup) # Internal helper async def _evaluate_entry(self, entry: dict) -> dict: @@ -298,9 +367,10 @@ async def _evaluate_entry(self, entry: dict) -> dict: compile_code = subtask_meta["compile"] run_code = subtask_meta["run"] grader_files = subtask_meta["grader_files"] + run_files = subtask_meta.get("run_files", []) if pid not in self.precompiled_cache: - self.precompiled_cache[pid] = await asyncio.to_thread( + grader_dir = await asyncio.to_thread( _precompile_grader, pid, grader_files, @@ -308,7 +378,8 @@ async def _evaluate_entry(self, entry: dict) -> dict: run_code, self.sandbox, ) - pre_dir = self.precompiled_cache[pid] + self.precompiled_cache[pid] = {"grader": grader_dir} + pre_dir = self.precompiled_cache[pid]["grader"] subtask_state = { st: { @@ -368,25 +439,53 @@ async def _evaluate_entry(self, entry: dict) -> dict: score = round(min(data["scores"]) * data["score"], data["precision"]) if data["scores"] else 0.0 test_case_results[st] = {"score": score, "outputs": data["outputs"]} + # Optionally run custom input cases + input_outputs = [] + if self.inputdata is not None: + problem_inputs = self.inputdata[str(entry["id"])] + for i in range(0, len(problem_inputs), batch_size): + batch = problem_inputs[i : i + batch_size] + tasks = [] + for test_data in batch: + tasks.append( + { + "generated_code": completion, + "problem_id": pid, + "run_files": run_files, + "test_input": test_data["content"], + } + ) + # map with unique worker id argument + results = await asyncio.to_thread( + self.pool.starmap, run_input_case, [(ta, idx) for idx, ta in enumerate(tasks)] + ) + for test_data, result in zip(batch, results): + test_name = test_data["file_name"] + test_type = "input" + result["test_name"] = test_name + result["test_type"] = test_type + input_outputs.append(result) + return { "name": entry["name"], "subtask": entry["subtask"], "test_case_results": test_case_results, + "input_case_results": input_outputs, } - async def eval_full(self): # type: ignore[override] - jsonl_file = self.eval_cfg.input_file - with open(jsonl_file, "r", encoding="utf-8") as f: - all_samples = [json.loads(line) for line in f] + async def eval_full(self, input_files): # type: ignore[override] + for jsonl_file in unroll_files(input_files): + with open(jsonl_file, "r", encoding="utf-8") as f: + all_samples = [json.loads(line) for line in f] - tasks = [self._evaluate_entry(s) for s in all_samples] - outputs = await asyncio.gather(*tasks) + tasks = [self._evaluate_entry(s) for s in all_samples] + outputs = await asyncio.gather(*tasks) - for s, o in zip(all_samples, outputs): - s["test_case_results"] = o["test_case_results"] - s["eval_status"] = o["eval_status"] + for s, o in zip(all_samples, outputs): + s["test_case_results"] = o["test_case_results"] + s["input_case_results"] = o["input_case_results"] - jdump(all_samples, jsonl_file, mode="wt") + jdump(all_samples, jsonl_file, mode="wt") if self.pool is not None: self.pool.close() diff --git a/nemo_skills/evaluation/metrics/ioi_metrics.py b/nemo_skills/evaluation/metrics/ioi_metrics.py index a2028f6a6d..4f4431a3bd 100644 --- a/nemo_skills/evaluation/metrics/ioi_metrics.py +++ b/nemo_skills/evaluation/metrics/ioi_metrics.py @@ -11,15 +11,26 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import json +import os +import re from collections import defaultdict from nemo_skills.evaluation.metrics.base import BaseMetrics +def extract_final_cpp_block(text): + pattern = r"```(?:cpp|Cpp)\s*\n(.*?)```" + matches = re.findall(pattern, text, re.DOTALL) + return matches[-1] if matches else "" + + class IOIMetrics(BaseMetrics): - def __init__(self): + def __init__(self, **kwargs): super().__init__() self.reset() + self.cluster_folder = kwargs.get("cluster_folder", None) + print(f"Cluster folder: {self.cluster_folder}") def update(self, predictions): super().update(predictions) @@ -30,6 +41,54 @@ def update(self, predictions): def _get_score_dict(self, p): return {"correct": all(r["score"] > 0 for r in p["test_case_results"].values())} + def extract_info(self, submission) -> dict: + # Aggregate IOI per-submission scores for convenience + subtask_scores = [v["score"] for _, v in submission["test_case_results"].items()] + return { + "grade": subtask_scores, + "tokens": submission["num_generated_tokens"], + "code": extract_final_cpp_block(submission["generation"]), + } + + def get_clusters(self, submissions) -> dict: + clusters = defaultdict(list) + id = 0 + + for submission in submissions: + input_results = submission.get("input_case_results", []) + run_outputs = [] + for output in input_results: + if "run_stdout" not in output: + continue + run_outputs.append(output["run_stdout"]) + output_key = tuple(run_outputs) + + extract_info = self.extract_info(submission) + if output_key not in clusters: + # Initialize per-subtask maxima and counts with this submission's scores + subtask_score_list = [res["score"] for _, res in submission["test_case_results"].items()] + clusters[output_key] = { + "codes": [], + "max_score": subtask_score_list[:], + "max_score_solutions": [1] * len(subtask_score_list), + } + else: + # Update maxima and counts element-wise from this submission + subtask_score_list = [res["score"] for _, res in submission["test_case_results"].items()] + max_scores = clusters[output_key]["max_score"] + max_counts = clusters[output_key]["max_score_solutions"] + for idx, score_val in enumerate(subtask_score_list): + if score_val > max_scores[idx]: + max_scores[idx] = score_val + max_counts[idx] = 1 + elif score_val == max_scores[idx]: + max_counts[idx] += 1 + clusters[output_key]["codes"].append(extract_info) + + id = submission.get("id", id) + + return clusters, id + def get_problem_score(self, submissions) -> float: """ For a given problem (list of submissions), compute the score as follows: @@ -37,7 +96,7 @@ def get_problem_score(self, submissions) -> float: - Sum these maximum scores to get the problem score. """ if not submissions: - return 0.0 + return 0.0, {} subtask_scores = {} for submission in submissions: @@ -45,63 +104,70 @@ def get_problem_score(self, submissions) -> float: subtask_scores[subtask] = max(subtask_scores.get(subtask, 0), result["score"]) return sum(subtask_scores.values()), subtask_scores - def simulate_round_robin_score(self, submissions) -> float: - """ - Computes a round robin score for a problem. - The procedure is as follows: - 1. For each submission, compute an aggregate score (sum of subtask scores). - 2. Sort submissions in descending order by the aggregate score. - 3. Select up to 50 submissions. - 4. For each subtask, take the maximum score among the selected submissions. - 5. Return the sum of these maximum subtask scores. - """ - if not submissions: - return 0.0 - - # compute an aggregate score per submission - for submission in submissions: - aggregate_score = sum(result["score"] for result in submission["test_case_results"].values()) - submission["_aggregate_score"] = aggregate_score - - # sort submissions in descending order by aggregate score - sorted_submissions = sorted(submissions, key=lambda s: s["_aggregate_score"], reverse=True) - # Select up to 50 submissions. - selected = sorted_submissions[:50] - - # for each subtask, take the maximum score among the selected submissions - subtask_scores = {} - for submission in selected: - for subtask, result in submission["test_case_results"].items(): - subtask_scores[subtask] = max(subtask_scores.get(subtask, 0), result["score"]) - return sum(subtask_scores.values()) - def get_metrics(self): - total_score = total_round_robin = 0.0 + total_score = 0.0 self.problem_scores = {} for name, submissions in self.predictions_by_problem.items(): + # Cluster the submissions if requested + if self.cluster_folder: + os.makedirs(self.cluster_folder, exist_ok=True) + submissions_by_id = defaultdict(list) + for sub in submissions: + submissions_by_id[sub["id"]].append(sub) + for sid, sid_submissions in submissions_by_id.items(): + clusters, _ = self.get_clusters(sid_submissions) + final_clusters = {} + for i, (output_key, cluster) in enumerate(clusters.items()): + final_clusters[f"cluster_{i + 1}"] = { + "output": output_key, + "codes": cluster["codes"], + "max_score": cluster["max_score"], + "max_score_solutions": cluster["max_score_solutions"], + } + output_file = os.path.join(self.cluster_folder, f"{sid}_cluster.jsonl") + with open(output_file, "w") as f: + json.dump(final_clusters, f, indent=4) + score, subtasks = self.get_problem_score(submissions) self.problem_scores[name] = (score, subtasks) total_score += score - total_round_robin += self.simulate_round_robin_score(submissions) - self.print_problem_scores() + + per_problem_subtask_scores = {} + for name, (achieved_total, achieved_subtasks) in self.problem_scores.items(): + submissions = self.predictions_by_problem[name] + max_subtasks = {} + for sub in submissions: + max_subtasks[sub["subtask"]] = sub["subtask_score"] + max_total = sum(max_subtasks.values()) + per_problem_subtask_scores[name] = { + "total": {"score": achieved_total, "max_score": max_total}, + "subtasks": { + subtask: {"score": achieved, "max_score": max_subtasks[subtask]} + for subtask, achieved in achieved_subtasks.items() + }, + } + metrics_dict = super().get_metrics() for m in metrics_dict.values(): - m["total_score"], m["round_robin_score"] = str(total_score), str(total_round_robin) + m["total_score"] = int(total_score) + m["per_problem_subtask_scores"] = per_problem_subtask_scores + self.per_problem_subtask_scores = per_problem_subtask_scores + self.print_problem_scores() return metrics_dict def reset(self): super().reset() self.predictions_by_problem = defaultdict(list) self.problem_scores = {} + self.per_problem_subtask_scores = {} + + def evaluations_to_print(self): + return [f"pass@{self.max_k}"] def print_problem_scores(self): print("---------------------------------Problem and subtask scores---------------------------------") - for name, (achieved_total, achieved_subtasks) in self.problem_scores.items(): - submissions = self.predictions_by_problem[name] - max_subtasks = {} - for sub in submissions: - max_subtasks[sub["subtask"]] = sub["subtask_score"] - max_total = sum(max_subtasks.values()) - print(f"# {name}: {achieved_total}/{max_total}") - for subtask, achieved in achieved_subtasks.items(): - print(f" {subtask}: {achieved}/{max_subtasks[subtask]}") + for name, info in self.per_problem_subtask_scores.items(): + total = info["total"] + print(f"# {name}: {int(total['score'])}/{int(total['max_score'])}") + for subtask, subinfo in info["subtasks"].items(): + print(f" {subtask}: {int(subinfo['score'])}/{int(subinfo['max_score'])}") From 832561110a7a5a69e74bac4feb717416658fb329 Mon Sep 17 00:00:00 2001 From: anowaczynski-nvidia Date: Wed, 17 Dec 2025 18:57:45 +0100 Subject: [PATCH 59/88] replace raise error with LOG.warning in AA LCR dataset prepare (#1119) Signed-off-by: Arkadiusz Nowaczynski Signed-off-by: George Armstrong Co-authored-by: George Armstrong Signed-off-by: Cheng-Ping Hsieh --- nemo_skills/dataset/aalcr/prepare.py | 2 +- tests/gpu-tests/test_eval.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/nemo_skills/dataset/aalcr/prepare.py b/nemo_skills/dataset/aalcr/prepare.py index ec9fa75256..81f2b724a1 100644 --- a/nemo_skills/dataset/aalcr/prepare.py +++ b/nemo_skills/dataset/aalcr/prepare.py @@ -187,7 +187,7 @@ def write_data_to_file(output_file, data, txt_file_folder, max_context_window, t continue if n_tokens != entry["input_tokens"]: # check if the n_tokens exactly match the input_tokens in the entry - raise ValueError(f"n_tokens: {n_tokens} != input_tokens: {entry['input_tokens']}") + LOG.warning(f"n_tokens: {n_tokens} != input_tokens: {entry['input_tokens']}") entry[f"n_tokens_{tokenizer_name}"] = n_tokens entry["question"] = question diff --git a/tests/gpu-tests/test_eval.py b/tests/gpu-tests/test_eval.py index 91d148877e..05dccf6b51 100644 --- a/tests/gpu-tests/test_eval.py +++ b/tests/gpu-tests/test_eval.py @@ -44,7 +44,6 @@ "mbpp", "mmau-pro", "asr-leaderboard", - "aalcr", # Has tokenization mismatch issues "mrcr", "audiobench", "librispeech-pc", From 83a0ab054c98ca725df7a43d3079c3d5b849e380 Mon Sep 17 00:00:00 2001 From: George Armstrong Date: Wed, 17 Dec 2025 11:59:13 -0800 Subject: [PATCH 60/88] FIX tavily search results return type (#1123) Signed-off-by: George Armstrong Signed-off-by: Cheng-Ping Hsieh --- nemo_skills/mcp/clients.py | 45 ++++++++++++++++++++++++-------------- tests/test_mcp_clients.py | 29 ++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 17 deletions(-) diff --git a/nemo_skills/mcp/clients.py b/nemo_skills/mcp/clients.py index 33b8cede46..4fc33a5f53 100644 --- a/nemo_skills/mcp/clients.py +++ b/nemo_skills/mcp/clients.py @@ -106,40 +106,51 @@ def _sanitize_input_args_for_tool(args_dict, tool_name, hide_args): return {k: v for k, v in args_dict.items() if k not in hidden_keys} +def _extract_item(item) -> Any: + """Extract a JSON-serializable value from a single content item. + + Returns the parsed JSON if text is valid JSON, otherwise the raw text. + Raises ValueError if the item doesn't have a text attribute. + """ + text = getattr(item, "text", None) + if not isinstance(text, str): + raise ValueError(f"Content item has no text attribute: {item}") + try: + return json.loads(text) + except json.JSONDecodeError: + return text + + def _extract_tool_result(result) -> Any: """Extract a JSON-serializable result from an MCP CallToolResult. Handles various response formats: - structuredContent: Returns directly if present - - content[].text: Parses as JSON or returns as string + - content[].text: Parses as JSON or returns as string (handles single or multiple items) - Fallback: Returns error dict to avoid returning raw CallToolResult objects This ensures the return value is always JSON-serializable. """ # Check if tool explicitly returned an error - return generic message to avoid leaking details - is_error = getattr(result, "isError", False) - if is_error: + if getattr(result, "isError", False): return {"error": "Tool execution failed"} struct = getattr(result, "structuredContent", None) if struct is not None: return struct - # Fallback: try to parse first content item as JSON, else return text + content = getattr(result, "content", None) - if content: - first = content[0] - text = getattr(first, "text", None) - if isinstance(text, str): - try: - return json.loads(text) - except Exception: - return text - LOG.error("Unsupported content type in tool result: %s", content) + if not content: + LOG.error("No content in tool result. Full result: %s", result) + return {"error": "No content returned from tool"} + + try: + if len(content) == 1: + return _extract_item(content[0]) + return [_extract_item(item) for item in content] + except ValueError as e: + LOG.error("Unsupported content type in tool result: %s", e) return {"error": "Unsupported content type returned from tool"} - # No content at all - # This could happen due to a tool failure (like hitting uncaught API limits) - LOG.error("No content in tool result. Full result: %s", result) - return {"error": "No content returned from tool"} def _wrap_call_tool_output_formatter(method): diff --git a/tests/test_mcp_clients.py b/tests/test_mcp_clients.py index 56b89976da..30d4cf8627 100644 --- a/tests/test_mcp_clients.py +++ b/tests/test_mcp_clients.py @@ -605,3 +605,32 @@ def test_schema_override_nonexistent_param_fails(): # Try to override 'script' which doesn't exist (tool only has 'code') with pytest.raises(ValueError, match="Parameter 'script' not in schema"): apply_schema_overrides(tool, {"parameters": {"script": {"name": "renamed"}}}) + + +@pytest.mark.asyncio +async def test_stdio_client_returns_list_for_multiple_content_items(tmp_path): + """Tool without return type hint that returns a list should produce multiple content items.""" + # FastMCP without return type hint - returns list as multiple TextContent items + server_code = """ +from mcp.server.fastmcp import FastMCP + +mcp = FastMCP(name="multi_result_tool") + +@mcp.tool() +async def get_items(count: int): + # No return type hint - FastMCP will serialize list items as separate TextContent + return [{"id": i} for i in range(1, count + 1)] + +if __name__ == "__main__": + mcp.run(transport="stdio") +""" + script_path = tmp_path / "multi_result_server.py" + script_path.write_text(server_code) + + client = MCPStdioClient(command="python", args=[str(script_path)]) + result = await client.call_tool("get_items", {"count": 3}) + + # Should return all items, not just the first one + assert isinstance(result, list), f"Expected list, got {type(result)}: {result}" + assert len(result) == 3 + assert result == [{"id": 1}, {"id": 2}, {"id": 3}] From f81e450ea1a11699a89a08caf07b184ea209b8e1 Mon Sep 17 00:00:00 2001 From: George Armstrong Date: Wed, 17 Dec 2025 16:03:58 -0800 Subject: [PATCH 61/88] Revert "Use run.Script for generate pipeline (#1052)" (#1125) Signed-off-by: George Armstrong Signed-off-by: Cheng-Ping Hsieh --- .github/workflows/gpu_tests.yml | 10 +- nemo_skills/pipeline/generate.py | 452 +++++++----------- nemo_skills/pipeline/nemo_evaluator.py | 217 +++++---- nemo_skills/pipeline/utils/__init__.py | 2 - nemo_skills/pipeline/utils/declarative.py | 551 ++++++++++------------ nemo_skills/pipeline/utils/generation.py | 107 +---- nemo_skills/pipeline/utils/scripts.py | 419 ---------------- tests/gpu-tests/test_eval.py | 2 +- tests/test_declarative_pipeline.py | 274 ++++++----- tests/test_generation.py | 40 +- tests/test_nemo_evaluator_pipeline.py | 41 +- 11 files changed, 719 insertions(+), 1396 deletions(-) delete mode 100644 nemo_skills/pipeline/utils/scripts.py diff --git a/.github/workflows/gpu_tests.yml b/.github/workflows/gpu_tests.yml index a500fc59b2..16f77633a8 100644 --- a/.github/workflows/gpu_tests.yml +++ b/.github/workflows/gpu_tests.yml @@ -52,15 +52,7 @@ jobs: cd ${{ github.run_id }} nvidia-smi set -o pipefail # this will make sure next line returns non-0 exit code if tests fail - # Run heartbeat in background, capture its PID, and ensure cleanup - (while true; do sleep 60; echo "[HEARTBEAT] $(date '+%Y-%m-%d %H:%M:%S') - still running..."; done) & - HEARTBEAT_PID=$! - # Run tests and capture exit code - EXIT_CODE=0 - ./tests/gpu-tests/run_qwen.sh || EXIT_CODE=$? - # Kill heartbeat and exit with test result - kill $HEARTBEAT_PID 2>/dev/null || true - exit $EXIT_CODE + ./tests/gpu-tests/run_qwen.sh - name: Cleanup if: always() run: | diff --git a/nemo_skills/pipeline/generate.py b/nemo_skills/pipeline/generate.py index 90ec987bca..f33796d05c 100644 --- a/nemo_skills/pipeline/generate.py +++ b/nemo_skills/pipeline/generate.py @@ -14,7 +14,7 @@ import importlib import logging import os -from typing import Dict, List, Optional +from typing import Callable, Dict, List, Optional import typer @@ -23,17 +23,14 @@ from nemo_skills.inference import GENERATION_MODULE_MAP, GenerationType from nemo_skills.pipeline.app import app, typer_unpacker from nemo_skills.pipeline.utils.cluster import parse_kwargs +from nemo_skills.pipeline.utils.commands import sandbox_command from nemo_skills.pipeline.utils.declarative import ( Command, CommandGroup, HardwareConfig, Pipeline, ) -from nemo_skills.pipeline.utils.scripts import ( - GenerationClientScript, - SandboxScript, - ServerScript, -) +from nemo_skills.pipeline.utils.server import get_free_port from nemo_skills.utils import ( compute_chunk_ids, get_logger_name, @@ -47,160 +44,118 @@ # TODO: add num_jobs here for consistency with eval? -def _create_job_unified( - models: List[str], - server_configs: List[Optional[Dict]], - generation_params: Dict, +def _create_commandgroup_from_config( + generation_cmd: str, + server_config: Optional[Dict], + with_sandbox: bool, + sandbox_port: Optional[int], cluster_config: Dict, installation_command: Optional[str], - with_sandbox: bool, + get_server_command_fn: Callable, partition: Optional[str], keep_mounts_for_sandbox: bool, task_name: str, log_dir: str, sbatch_kwargs: Optional[Dict] = None, sandbox_env_overrides: Optional[List[str]] = None, -) -> List[CommandGroup]: - """ - Create CommandGroups for n models (unified for n=1 and n>1). - - Structure: - - Group 0: Model 0 server + client + (optional sandbox) - - Group 1: Model 1 server (if n>1) - - Group N: Model N server (if n>1) - - For n=1, returns a single-element list. The Pipeline automatically - optimizes single-group lists to efficient single-group jobs. - - Args: - models: List of model paths - server_configs: List of server configurations (one per model, None if not hosting) - generation_params: Dict of parameters for generation (output_dir, etc.) - cluster_config: Cluster configuration - installation_command: Installation command to run before client - with_sandbox: Whether to include sandbox - partition: Slurm partition - keep_mounts_for_sandbox: Whether to keep mounts for sandbox - task_name: Name for the task - log_dir: Directory for logs - sbatch_kwargs: Additional sbatch kwargs - - Returns: - List of CommandGroup objects (one per het group) - """ - num_models = len(models) - groups = [] - server_scripts = [] # Track server Script objects for cross-component references - - for model_idx, (model_path, server_config) in enumerate(zip(models, server_configs)): - components = [] - server_script = None - - # Track GPU/node requirements for this group (from server config) - group_gpus = 0 - group_nodes = 1 - - # 1. Add server if needed - if server_config is not None and int(server_config.get("num_gpus", 0)) > 0: - server_type = server_config["server_type"] - server_container = server_config.get("container") or cluster_config["containers"][server_type] +) -> CommandGroup: + """Create a CommandGroup from server_config. - # Create ServerScript - server_script = ServerScript( - server_type=server_type, - model_path=server_config["model_path"], - cluster_config=cluster_config, - num_gpus=server_config["num_gpus"], - num_nodes=server_config["num_nodes"], - server_args=server_config.get("server_args", ""), - server_entrypoint=server_config.get("server_entrypoint"), - port=server_config.get("server_port"), - allocate_port=(server_config.get("server_port") is None), - ) + Component ordering: + 1. Server (if server_config provided) + 2. Client command + 3. Sandbox (if with_sandbox=True) + """ - # Set group GPU/node requirements from server config - group_gpus = server_config["num_gpus"] - group_nodes = server_config["num_nodes"] + components = [] - server_cmd = Command( - script=server_script, - container=server_container, - name=f"{task_name}_model_{model_idx}_server" if num_models > 1 else f"{task_name}_server", - ) - components.append(server_cmd) - server_scripts.append(server_script) + # 1. Add server if server_config is provided + if server_config is not None and int(server_config["num_gpus"]) > 0: + server_type = server_config["server_type"] + # Get container from server_config if provided, otherwise fall back to cluster config + if "container" in server_config: + server_container = server_config.pop("container") else: - # No server for this model (pre-hosted) - server_scripts.append(None) - - # 2. Group 0 gets the client and sandbox - if model_idx == 0: - # Create sandbox script (if with_sandbox) - sandbox_script = None - if with_sandbox: - sandbox_script = SandboxScript( - cluster_config=cluster_config, - keep_mounts=keep_mounts_for_sandbox, - allocate_port=True, # Always allocate port for sandbox - env_overrides=sandbox_env_overrides, - ) - - sandbox_cmd = Command( - script=sandbox_script, - container=cluster_config["containers"]["sandbox"], - name=f"{task_name}_sandbox", - ) - components.append(sandbox_cmd) - - # Create client script with cross-component references to all servers - client_script = GenerationClientScript( - output_dir=generation_params["output_dir"], - input_file=generation_params.get("input_file"), - input_dir=generation_params.get("input_dir"), - extra_arguments=generation_params.get("extra_arguments", ""), - random_seed=generation_params.get("random_seed"), - chunk_id=generation_params.get("chunk_id"), - num_chunks=generation_params.get("num_chunks"), - preprocess_cmd=generation_params.get("preprocess_cmd"), - postprocess_cmd=generation_params.get("postprocess_cmd"), - wandb_parameters=generation_params.get("wandb_parameters"), - with_sandbox=with_sandbox, - script=generation_params.get("script", "nemo_skills.inference.generate"), - # Multi-server support (works for single and multi-model) - servers=server_scripts if server_scripts else None, - server_addresses_prehosted=generation_params.get("server_addresses_prehosted"), - model_names=generation_params.get("model_names"), - server_types=generation_params.get("server_types"), - sandbox=sandbox_script, - installation_command=installation_command, - ) + server_container = cluster_config["containers"][server_type] - client_cmd = Command( - script=client_script, - container=cluster_config["containers"]["nemo-skills"], - name=f"{task_name}", - ) - components.append(client_cmd) + # Call server command builder directly with cluster_config + cmd, num_tasks = get_server_command_fn(**server_config, cluster_config=cluster_config) - # Only create group if it has components (skip empty groups for pre-hosted models) - if components: - group_tasks = server_script.num_tasks if (server_config and server_script) else 1 + # Create metadata dict + metadata = { + "num_tasks": num_tasks, + "gpus": server_config["num_gpus"], + "nodes": server_config["num_nodes"], + "log_prefix": "server", + } - group = CommandGroup( - commands=components, - hardware=HardwareConfig( - partition=partition, - num_gpus=group_gpus, - num_nodes=group_nodes, - num_tasks=group_tasks, - sbatch_kwargs=sbatch_kwargs, - ), - name=f"{task_name}_model_{model_idx}_group" if num_models > 1 else task_name, - log_dir=log_dir, - ) - groups.append(group) + server_cmd = Command( + command=cmd, + container=server_container, + gpus=server_config["num_gpus"], + nodes=server_config["num_nodes"], + name=task_name, + metadata=metadata, + ) + components.append(server_cmd) + + # 2. Add main generation command + # Note: General cluster config env vars are automatically added by get_env_variables() in get_executor() + client_env = {} + if with_sandbox and sandbox_port is not None: + client_env["NEMO_SKILLS_SANDBOX_PORT"] = str(sandbox_port) + + client_cmd = Command( + command=generation_cmd, + container=cluster_config["containers"]["nemo-skills"], + name=task_name, + installation_command=installation_command, + metadata={ + "log_prefix": "main", + "environment": client_env, + }, + ) + components.append(client_cmd) + + # 3. Add sandbox if requested + if with_sandbox: + # Call sandbox command builder directly with cluster_config + cmd, metadata = sandbox_command(cluster_config=cluster_config, port=sandbox_port) + metadata["log_prefix"] = "sandbox" + + # Apply user-specified environment overrides for the sandbox + if sandbox_env_overrides: + sandbox_env = metadata.get("environment", {}) + for override in sandbox_env_overrides: + key, value = override.split("=", 1) + sandbox_env[key] = value + metadata["environment"] = sandbox_env + + sandbox_cmd = Command( + command=cmd, + container=cluster_config["containers"]["sandbox"], + name=task_name, + metadata=metadata, + ) - return groups + components.append(sandbox_cmd) + + # Find maximum GPUs/nodes needed by any component for the HardwareConfig + # The job-level resource request must be the maximum across all components + max_gpus = max((comp.gpus or 0) for comp in components) + max_nodes = max((comp.nodes or 1) for comp in components) + + return CommandGroup( + commands=components, + hardware=HardwareConfig( + partition=partition, + num_gpus=max_gpus, + num_nodes=max_nodes, + sbatch_kwargs=sbatch_kwargs, + ), + name=task_name, + log_dir=log_dir, + ) @app.command(context_settings={"allow_extra_args": True, "ignore_unknown_options": True}) @@ -231,45 +186,21 @@ def generate( "If not specified, will use the registered generation module for the " "generation type (which is required in this case).", ), - model: List[str] = typer.Option( - None, - help="Path to the model(s). CLI: space-separated. Python API: string or list. " - "Single value broadcasts to all models for multi-model generation.", - ), - server_address: List[str] = typer.Option( - None, - help="Server address(es). CLI: space-separated. Python API: string or list. " - "Single value broadcasts to all models.", - ), - server_type: List[pipeline_utils.SupportedServers] = typer.Option( - ..., - help="Server type(s). CLI: space-separated. Python API: string or list. " - "Single value broadcasts to all models.", + model: str = typer.Option(None, help="Path to the model or model name in API"), + server_address: str = typer.Option( + None, help="Use ip:port for self-hosted models or the API url if using model providers" ), - server_gpus: List[int] = typer.Option( + server_type: pipeline_utils.SupportedServers = typer.Option(..., help="Type of server to use"), + server_gpus: int = typer.Option(None, help="Number of GPUs to use if hosting the model"), + server_nodes: int = typer.Option(1, help="Number of nodes required for hosting LLM server"), + server_args: str = typer.Option("", help="Any extra arguments to pass to the server"), + server_entrypoint: str = typer.Option( None, - help="Number of GPUs per model. CLI: space-separated ints. Python API: int or list. " - "Single value broadcasts to all models.", + help="Path to the entrypoint of the server. " + "If not specified, will use the default entrypoint for the server type.", ), - server_nodes: List[int] = typer.Option( - [1], - help="Number of nodes per model. CLI: space-separated ints. Python API: int or list. " - "Single value broadcasts to all models.", - ), - server_args: List[str] = typer.Option( - [""], - help="Server arguments per model. CLI: space-separated. Python API: string or list. " - "Single value broadcasts to all models.", - ), - server_entrypoint: List[str] = typer.Option( - None, - help="Server entrypoint(s). CLI: space-separated. Python API: string or list. " - "Single value broadcasts to all models.", - ), - server_container: List[str] = typer.Option( - None, - help="Container image(s). CLI: space-separated. Python API: string or list. " - "Single value broadcasts to all models.", + server_container: str = typer.Option( + None, help="Override container image for the hosted server (if server_gpus is set)" ), dependent_jobs: int = typer.Option(0, help="Specify this to launch that number of dependent jobs"), mount_paths: str = typer.Option(None, help="Comma separated list of paths to mount on the remote machine"), @@ -365,18 +296,7 @@ def generate( None, help="Internal option to specify task dependencies.", hidden=True ), ): - """Generate LLM completions for single or multiple models. - - Supports both single-model and multi-model generation through a unified interface. - - Parameter Types: - Multi-model parameters (model, server_*, etc.) use List[T] type hints for Typer CLI - compatibility, but accept both scalars and lists when called from Python: - - CLI: --model m1 m2 (space-separated) → Typer converts to ["m1", "m2"] - - Python API: model="m1" or model=["m1", "m2"] → Both work (normalized internally) - - Single values broadcast to all models: server_gpus=8 → [8, 8, 8] for 3 models - - Multi-model usage requires either --generation-type or --generation-module. + """Generate LLM completions for a given input file. Run `python -m nemo_skills.inference.generate --help` for other supported arguments (need to be prefixed with ++, since we use Hydra for that script). @@ -386,42 +306,10 @@ def generate( LOG.info("Starting generation job") LOG.info("Extra arguments that will be passed to the underlying script: %s", extra_arguments) - # Normalize model configuration to list - models_list = pipeline_utils.normalize_models_config(model) - num_models = len(models_list) - - LOG.info(f"Number of models: {num_models}") - for model_idx, model_name in enumerate(models_list): - LOG.info(f" Model {model_idx}: {model_name}") - - # Convert server_type enum values to strings - def convert_server_type_to_string(server_type): - return server_type.value if hasattr(server_type, "value") else server_type - - if isinstance(server_type, list): - server_type = [convert_server_type_to_string(st) for st in server_type] - else: - server_type = convert_server_type_to_string(server_type) - - # Normalize all server parameters to per-model lists - server_types_list = pipeline_utils.normalize_parameter(server_type, num_models, "server_type") - server_gpus_list = pipeline_utils.normalize_parameter(server_gpus, num_models, "server_gpus") - server_nodes_list = pipeline_utils.normalize_parameter(server_nodes, num_models, "server_nodes") - server_args_list = pipeline_utils.normalize_parameter(server_args, num_models, "server_args") - server_entrypoints_list = pipeline_utils.normalize_parameter(server_entrypoint, num_models, "server_entrypoint") - server_containers_list = pipeline_utils.normalize_parameter(server_container, num_models, "server_container") - - if server_address is not None: - server_addresses_list = pipeline_utils.normalize_parameter(server_address, num_models, "server_address") - else: - server_addresses_list = [None] * num_models - - # Validate multi-model requirements - if num_models > 1: - if generation_type is None and generation_module is None: - raise ValueError( - "Multi-model generation requires either --generation-type or --generation-module to be specified" - ) + try: + server_type = server_type.value + except AttributeError: + pass if log_samples: wandb_parameters = { @@ -437,6 +325,8 @@ def convert_server_type_to_string(server_type): else: wandb_parameters = None + get_random_port = pipeline_utils.should_get_random_port(server_gpus, exclusive) + if random_seeds and num_random_seeds: raise ValueError("Cannot specify both random_seeds and num_random_seeds") if num_random_seeds: @@ -465,6 +355,8 @@ def convert_server_type_to_string(server_type): check_mounted_paths=check_mounted_paths, ) + original_server_address = server_address + if generation_module is not None and generation_type is not None: raise ValueError("Cannot specify both generation_module and generation_type. ") if generation_module is None: @@ -515,36 +407,36 @@ def convert_server_type_to_string(server_type): chunk_id=None, ) for chunk_id in chunk_ids: - # Configure clients for each model - server_configs = [] - server_addresses_resolved = [] - # For single model: configure_client returns extra_args with server config appended - # For multi-model: use original extra_args (server config added as lists in get_generation_cmd) - extra_arguments = extra_arguments_original - - for model_idx in range(num_models): - get_random_port_for_server = pipeline_utils.should_get_random_port( - server_gpus_list[model_idx], exclusive - ) - - srv_config, srv_address, srv_extra_args = pipeline_utils.configure_client( - model=models_list[model_idx], - server_type=server_types_list[model_idx], - server_address=server_addresses_list[model_idx], - server_gpus=server_gpus_list[model_idx], - server_nodes=server_nodes_list[model_idx], - server_args=server_args_list[model_idx], - server_entrypoint=server_entrypoints_list[model_idx], - server_container=server_containers_list[model_idx], - extra_arguments=extra_arguments_original if model_idx == 0 else "", - get_random_port=get_random_port_for_server, - ) - server_configs.append(srv_config) - server_addresses_resolved.append(srv_address) + # Configure client (same as before) + server_config, server_address, extra_arguments = pipeline_utils.configure_client( + model=model, + server_type=server_type, + server_address=original_server_address, + server_gpus=server_gpus, + server_nodes=server_nodes, + server_args=server_args, + server_entrypoint=server_entrypoint, + server_container=server_container, + extra_arguments=extra_arguments_original, + get_random_port=get_random_port, + ) - # For single model, capture the extra_args with server config from configure_client - if model_idx == 0 and num_models == 1: - extra_arguments = srv_extra_args + # Build generation command (same as before) + cmd = pipeline_utils.get_generation_cmd( + input_file=input_file, + input_dir=input_dir, + random_seed=seed, + output_dir=output_dir, + extra_arguments=extra_arguments, + chunk_id=chunk_id, + num_chunks=num_chunks, + preprocess_cmd=preprocess_cmd, + postprocess_cmd=postprocess_cmd, + wandb_parameters=wandb_parameters if seed_idx == 0 else None, + script=generation_module, + with_sandbox=with_sandbox, + ) + cmd = pipeline_utils.wrap_python_path(cmd=cmd) # Base task name (shared across all dependent jobs in the chain) task_name = f"{expname}-rs{seed}" if seed is not None else expname @@ -556,35 +448,22 @@ def convert_server_type_to_string(server_type): prev_job = None for dep_idx in range(dependent_jobs + 1): - # Build generation parameters dict for Script - generation_params = { - "output_dir": output_dir, - "input_file": input_file, - "input_dir": input_dir, - "extra_arguments": extra_arguments, - "random_seed": seed, - "chunk_id": chunk_id, - "num_chunks": num_chunks, - "preprocess_cmd": preprocess_cmd, - "postprocess_cmd": postprocess_cmd, - "wandb_parameters": wandb_parameters if seed_idx == 0 else None, - "script": generation_module, - # Multi-model specific fields - "server_addresses_prehosted": server_addresses_resolved, - "model_names": models_list, - "server_types": server_types_list, - } + # Allocate sandbox port if needed + # This must be done BEFORE creating CommandGroup so client knows the port + if with_sandbox: + current_sandbox_port = get_free_port(strategy="random") if get_random_port else 6000 + else: + current_sandbox_port = None - # Create CommandGroup(s) using Script objects - # For multi-model, this creates multiple CommandGroups (one per model + one for client) - # For single-model, this creates a single CommandGroup - job_groups = _create_job_unified( - models=models_list, - server_configs=[cfg.copy() if cfg else None for cfg in server_configs], - generation_params=generation_params, + # Create CommandGroup for this task + cmd_group = _create_commandgroup_from_config( + generation_cmd=cmd, + server_config=server_config.copy() if server_config else None, + with_sandbox=with_sandbox, + sandbox_port=current_sandbox_port, cluster_config=cluster_config, installation_command=installation_command, - with_sandbox=with_sandbox, + get_server_command_fn=generation_task.get_server_command_fn(), partition=partition, keep_mounts_for_sandbox=keep_mounts_for_sandbox, task_name=task_name, @@ -608,16 +487,11 @@ def convert_server_type_to_string(server_type): # Subsequent jobs in chain depend on previous job (use job object, not string) job_deps = [prev_job] - # For multi-group jobs, use "groups" key; for single-group, use "group" key job_spec = { "name": internal_job_name, + "group": cmd_group, "dependencies": job_deps, } - if len(job_groups) > 1: - job_spec["groups"] = job_groups - else: - job_spec["group"] = job_groups[0] - jobs.append(job_spec) prev_job = job_spec # Track for next iteration diff --git a/nemo_skills/pipeline/nemo_evaluator.py b/nemo_skills/pipeline/nemo_evaluator.py index 39838737ed..020162692a 100644 --- a/nemo_skills/pipeline/nemo_evaluator.py +++ b/nemo_skills/pipeline/nemo_evaluator.py @@ -89,7 +89,7 @@ import copy import logging -from dataclasses import dataclass, field +from dataclasses import dataclass from pathlib import Path from typing import Dict, List, Optional @@ -97,12 +97,12 @@ from nemo_evaluator_launcher.api import RunConfig from nemo_evaluator_launcher.common.helpers import get_eval_factory_command from nemo_evaluator_launcher.common.mapping import get_task_from_mapping, load_tasks_mapping -from omegaconf import OmegaConf +from omegaconf import DictConfig, OmegaConf import nemo_skills.pipeline.utils as pipeline_utils from nemo_skills.pipeline.app import app, typer_unpacker +from nemo_skills.pipeline.utils.commands import vllm_server_command from nemo_skills.pipeline.utils.declarative import Command, CommandGroup, HardwareConfig, Pipeline -from nemo_skills.pipeline.utils.scripts import BaseJobScript, ServerScript from nemo_skills.utils import get_logger_name, setup_logging LOG = logging.getLogger(get_logger_name(__file__)) @@ -289,8 +289,8 @@ def nemo_evaluator( expname=expname, idx=idx, task_name=task.name, - launcher_run_cfg=OmegaConf.to_container(launcher_run_cfg, resolve=True), - task_cfg=OmegaConf.to_container(task, resolve=True), + launcher_run_cfg=launcher_run_cfg, + task_cfg=task, task_definition=task_definition, base_output_root=base_output_root, eval_image=eval_image, @@ -443,8 +443,10 @@ def _create_serving_command_obj( idx: int, task_name: str, ) -> Command: - """Create a `Command` backed by a `ServerScript` for a hosted serving component. + """Create a Command object for a hosted serving component (main or judge server). + This function wraps vllm_server_command and standardizes container selection, + logging prefixes, and metadata for both main and judge servers. Args: cluster_config: Cluster configuration dictionary @@ -462,53 +464,54 @@ def _create_serving_command_obj( task_name: Task name for naming Returns: - Command: A Command object whose `script` is a configured `ServerScript`. + Command object configured for the serving component """ stype = (server_type or "vllm").lower() + sargs = args or "" if stype != "vllm": LOG.warning("Only vllm server_type is supported currently; got %s", stype) - server_script = ServerScript( - server_type=stype, - model_path=model or "", + cmd_str, meta = vllm_server_command( cluster_config=cluster_config, - num_gpus=gpus, - num_nodes=nodes or 1, - server_args=args or "", - server_entrypoint=entrypoint, + model=model, # type: ignore[arg-type] port=port, - allocate_port=port is None, + server_type=stype, + gpus=gpus, + nodes=nodes, + args=sargs, + entrypoint=entrypoint, ) - # Judge servers get a distinct log prefix for clarity - if is_judge: - server_script.log_prefix = "judge-server" - + # Resolve container fallback when not explicitly provided if not container: container = cluster_config["containers"][stype] + log_prefix = "judge-server" if is_judge else "server" name_role = "judge-server" if is_judge else "server" return Command( - script=server_script, + command=cmd_str, container=container, + gpus=gpus, + nodes=nodes or 1, name=f"{expname}-{name_role}-{idx}-{task_name}", + metadata={ + **meta, + "gpus": gpus, + "log_prefix": log_prefix, + }, ) @dataclass class _TaskCreationContext: - """Local helper to pass around the information about the task and easier logic sharing. - - Note: launcher_run_cfg and task_cfg are stored as plain dicts (not OmegaConf) to allow - serialization by nemo_run/fiddle. Convert back to DictConfig if OmegaConf operations are needed. - """ + """Local helper to pass around the information about the task and easier logic sharing.""" expname: str idx: int task_name: str - launcher_run_cfg: dict # Stored as plain dict for serialization compatibility - task_cfg: dict # Stored as plain dict for serialization compatibility + launcher_run_cfg: RunConfig + task_cfg: DictConfig task_definition: dict base_output_root: Optional[str] eval_image: str @@ -627,7 +630,12 @@ def _build_judge_server_if_needed(ctx: _TaskCreationContext) -> Optional[Command def _build_client_command( ctx: _TaskCreationContext, main_server_cmd: Optional[Command], judge_server_cmd: Optional[Command] ) -> Command: - """Create the evaluator client `Command` using `EvaluatorClientScript`. + """Build Command for evaluator client. + + The client command behavior depends on server hosting: + - If servers are co-hosted: Uses lambda factory to resolve runtime URLs via hostname_ref/meta_ref + - If using external servers: Uses static URLs from server_base_url/judge_server_base_url + - If no servers: Uses URLs from evaluator config or defaults Args: ctx: Task creation context with all configuration @@ -635,26 +643,100 @@ def _build_client_command( judge_server_cmd: Judge server Command if self-hosted, None otherwise Returns: - Command: A Command whose script builds the evaluator CLI at runtime + Command object for evaluator client """ + if ctx.hosting_server or ctx.hosting_judge: + # Co-hosted servers: Use lambda factory to resolve runtime URLs + # The lambda is evaluated at execution time when het_group_index is assigned + def _client_cmd_factory(): + waits: List[str] = [] + target_url: Optional[str] = None + judge_url: Optional[str] = None - client_script = EvaluatorClientScript( - ctx=ctx, - main_server_script=main_server_cmd.script if main_server_cmd else None, - judge_server_script=judge_server_cmd.script if judge_server_cmd else None, + # Build main server URL from runtime references + if ctx.hosting_server and main_server_cmd is not None: + server_host = main_server_cmd.hostname_ref() + server_port_val = main_server_cmd.meta_ref("port") + base_url = f"http://{server_host}:{server_port_val}" + waits.append(pipeline_utils.get_server_wait_cmd(f"{base_url}{ctx.server_health_path}")) + target_url = f"{base_url}{ctx.server_api_path}" + + # Build judge server URL from runtime references + if ctx.hosting_judge and judge_server_cmd is not None: + jhost = judge_server_cmd.hostname_ref() + jport = judge_server_cmd.meta_ref("port") + jbase = f"http://{jhost}:{jport}" + waits.append(pipeline_utils.get_server_wait_cmd(f"{jbase}{ctx.judge_server_health_path}")) + judge_url = f"{jbase}{ctx.judge_server_api_path}" + + # Wait for servers to be ready, then run evaluator + wait_cmd = " && ".join(waits) if waits else "true" + cmd = _build_task_cmd( + task_name=ctx.task_name, + launcher_run_cfg=ctx.launcher_run_cfg, + task_cfg=ctx.task_cfg, + task_definition=ctx.task_definition, + expname=ctx.expname, + base_output_root=ctx.base_output_root, + url_override=target_url, + model_id=ctx.server_model, + judge_url_override=judge_url, + judge_model_id=ctx.judge_server_model, + ) + return f"{wait_cmd} && {cmd}" + + return Command( + command=_client_cmd_factory, + container=ctx.eval_image, + gpus=ctx.job_gpus or None, + nodes=ctx.job_nodes or 1, + name=f"{ctx.expname}-client-{ctx.idx}-{ctx.task_name}", + metadata={ + "log_prefix": "main", + "environment": ctx.env_vars, + "gpus": ctx.job_gpus or None, + }, + ) + + # No hosted servers: Use external URLs or config defaults + server_url = None + if ctx.with_external_server and ctx.server_base_url: + server_url = ctx.server_base_url.rstrip("/") + ctx.server_api_path + judge_url = None + if ctx.with_external_judge and ctx.judge_server_base_url: + judge_url = ctx.judge_server_base_url.rstrip("/") + ctx.judge_server_api_path + + eval_cmd = _build_task_cmd( + task_name=ctx.task_name, + launcher_run_cfg=ctx.launcher_run_cfg, + task_cfg=ctx.task_cfg, + task_definition=ctx.task_definition, + expname=ctx.expname, + base_output_root=ctx.base_output_root, + url_override=server_url, + model_id=ctx.server_model, + judge_url_override=judge_url, + judge_model_id=ctx.judge_server_model, ) return Command( - script=client_script, + command=eval_cmd, container=ctx.eval_image, - name=f"{ctx.expname}-client-{ctx.idx}-{ctx.task_name}", + gpus=None, + nodes=ctx.job_nodes or 1, + name=f"{ctx.expname}-{ctx.idx}-{ctx.task_name}", + metadata={ + "log_prefix": "main", + "environment": ctx.env_vars, + "gpus": ctx.job_gpus or None, + }, ) def _build_task_cmd( task_name: str, - launcher_run_cfg: dict, - task_cfg: dict, + launcher_run_cfg: DictConfig, + task_cfg: DictConfig, task_definition: dict, expname: str, base_output_root: Optional[str], @@ -670,8 +752,8 @@ def _build_task_cmd( Args: task_name: Task identifier (e.g., "ifeval", "gpqa_diamond") - launcher_run_cfg: Global evaluator configuration (as plain dict) - task_cfg: Task-specific configuration (as plain dict, may include task-level overrides) + launcher_run_cfg: Global evaluator configuration from RunConfig + task_cfg: Task-specific configuration (may include task-level overrides) task_definition: Task definition from mapping (container, harness info) expname: Experiment name for output directory structure base_output_root: Base directory for task outputs @@ -689,9 +771,7 @@ def _build_task_cmd( - Judge: config.params.extra.judge.url Output directory is set to: {base_output_root}/{expname}/nemo-evaluator-results/{task_name} """ - # Convert back to DictConfig for OmegaConf operations - launcher_run_cfg = OmegaConf.create(launcher_run_cfg) - task_cfg_copy = OmegaConf.create(copy.deepcopy(task_cfg)) + task_cfg_copy = copy.deepcopy(task_cfg) if url_override: OmegaConf.update(task_cfg_copy, "overrides", {"target.api_endpoint.url": url_override}, force_add=True) @@ -726,56 +806,3 @@ def _build_task_cmd( cmd_struct = get_eval_factory_command(launcher_run_cfg, task_cfg_copy, task_definition) return cmd_struct.cmd - - -@dataclass(kw_only=True) -class EvaluatorClientScript(BaseJobScript): - """run.Script implementation for nemo-evaluator client with runtime server resolution.""" - - ctx: _TaskCreationContext - main_server_script: Optional[ServerScript] = None - judge_server_script: Optional[ServerScript] = None - log_prefix: str = field(default="main", init=False) - - def __post_init__(self): - def build_command(): - waits: List[str] = [] - target_url: Optional[str] = None - judge_url: Optional[str] = None - - if self.ctx.hosting_server and self.main_server_script is not None: - server_host = self.main_server_script.hostname_ref() - base_url = f"http://{server_host}:{self.main_server_script.port}" - waits.append(pipeline_utils.get_server_wait_cmd(f"{base_url}{self.ctx.server_health_path}")) - target_url = f"{base_url}{self.ctx.server_api_path}" - elif self.ctx.with_external_server and self.ctx.server_base_url: - target_url = self.ctx.server_base_url.rstrip("/") + self.ctx.server_api_path - - if self.ctx.hosting_judge and self.judge_server_script is not None: - judge_host = self.judge_server_script.hostname_ref() - judge_base = f"http://{judge_host}:{self.judge_server_script.port}" - waits.append(pipeline_utils.get_server_wait_cmd(f"{judge_base}{self.ctx.judge_server_health_path}")) - judge_url = f"{judge_base}{self.ctx.judge_server_api_path}" - elif self.ctx.with_external_judge and self.ctx.judge_server_base_url: - judge_url = self.ctx.judge_server_base_url.rstrip("/") + self.ctx.judge_server_api_path - - cmd = _build_task_cmd( - task_name=self.ctx.task_name, - launcher_run_cfg=self.ctx.launcher_run_cfg, - task_cfg=self.ctx.task_cfg, - task_definition=self.ctx.task_definition, - expname=self.ctx.expname, - base_output_root=self.ctx.base_output_root, - url_override=target_url, - model_id=self.ctx.server_model, - judge_url_override=judge_url, - judge_model_id=self.ctx.judge_server_model, - ) - - wait_cmd = " && ".join(waits) if waits else None - final_cmd = f"{wait_cmd} && {cmd}" if wait_cmd else cmd - env_vars = copy.deepcopy(self.ctx.env_vars) - return final_cmd, {"environment": env_vars} - - self.set_inline(build_command) - super().__post_init__() diff --git a/nemo_skills/pipeline/utils/__init__.py b/nemo_skills/pipeline/utils/__init__.py index 3e738a530f..1e470f3539 100644 --- a/nemo_skills/pipeline/utils/__init__.py +++ b/nemo_skills/pipeline/utils/__init__.py @@ -49,8 +49,6 @@ get_chunked_rs_filename, get_generation_cmd, get_remaining_jobs, - normalize_models_config, - normalize_parameter, wrap_cmd, ) from nemo_skills.pipeline.utils.mounts import ( diff --git a/nemo_skills/pipeline/utils/declarative.py b/nemo_skills/pipeline/utils/declarative.py index 51d0746c63..e294a3ed82 100644 --- a/nemo_skills/pipeline/utils/declarative.py +++ b/nemo_skills/pipeline/utils/declarative.py @@ -12,71 +12,39 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import annotations - -import logging -from contextlib import nullcontext -from dataclasses import dataclass -from typing import Dict, List, Optional, Tuple, Union - -import nemo_run as run - -from nemo_skills.pipeline.utils import ( - get_env_variables, - get_executor, - get_exp, - get_exp_handles, - get_registered_external_repo, - get_tunnel, - run_exp, - temporary_env_update, -) -from nemo_skills.pipeline.utils.exp import ( - REUSE_CODE_EXP, - get_packaging_job_key, - tunnel_hash, -) -from nemo_skills.pipeline.utils.mounts import is_mounted_filepath -from nemo_skills.pipeline.utils.server import wrap_python_path -from nemo_skills.utils import get_logger_name - """ -Simplified declarative pipeline system using Command with run.Script objects. +Simplified declarative pipeline system using only Command for all task types. Basic Example (Single job with multiple commands): - from nemo_skills.pipeline.utils.scripts import ServerScript, SandboxScript, GenerationClientScript + from nemo_skills.pipeline.utils.commands import vllm_server_command, sandbox_command from nemo_skills.pipeline.utils.declarative import Command, CommandGroup, HardwareConfig, Pipeline - - # Create Script objects for server and sandbox - # Scripts handle port allocation, cross-component references, and command building - server_script = ServerScript( - server_type="vllm", - model_path="Qwen/Qwen2.5-Math-7B-Instruct", - server_args="--tensor-parallel-size 1" + from nemo_skills.pipeline.utils.server import get_free_port + + # Allocate ports for server and sandbox + server_port = get_free_port(strategy="random") + sandbox_port = get_free_port(strategy="random") + + # Commands that run together in one SLURM job + # Note: Lambdas are needed for cross-component references (hostname_ref, meta_ref) + # which aren't resolved until het_group_index is assigned at pipeline execution time. + server_cmd, server_meta = vllm_server_command(cluster_cfg, model="Qwen/Qwen3-8B", port=server_port) + server = Command(command=server_cmd, gpus=8, name="server", metadata=server_meta) + + sandbox_cmd, sandbox_meta = sandbox_command(cluster_cfg, port=sandbox_port) + sandbox = Command(command=sandbox_cmd, name="sandbox", metadata=sandbox_meta) + + # This lambda is ESSENTIAL - server.hostname_ref() and meta_ref() aren't available until runtime + # Client needs NEMO_SKILLS_SANDBOX_PORT to connect to sandbox + client = Command( + command=lambda: f"curl {server.hostname_ref()}:{server.meta_ref('port')}/health", + name="client", + metadata={"environment": {"NEMO_SKILLS_SANDBOX_PORT": str(sandbox_port)}} ) - sandbox_script = SandboxScript() - - # Create generation client that references server and sandbox - # Cross-component references (hostname_ref, port) are resolved at runtime - client_script = GenerationClientScript( - output_dir="/results/inference", - extra_arguments="++prompt_config=math ++split=test", - servers=[server_script], # References server for hostname/port - model_names=["Qwen/Qwen2.5-Math-7B-Instruct"], - server_types=["vllm"], - sandbox=sandbox_script, # References sandbox for port - with_sandbox=True, - ) - - # Wrap Scripts in Commands with container and resource info - server = Command(script=server_script, container="vllm", name="server") - sandbox = Command(script=sandbox_script, container="nemo-skills", name="sandbox") - client = Command(script=client_script, container="nemo-skills", name="client") - # Group them together (they run in one SLURM job) + # Group them together inference_group = CommandGroup( commands=[server, sandbox, client], - hardware=HardwareConfig(partition="batch", num_gpus=1), + hardware=HardwareConfig(partition="batch"), name="inference" ) @@ -89,27 +57,13 @@ pipeline.run() Advanced Example (Multiple jobs with dependencies and heterogeneous components): - from nemo_skills.pipeline.utils.scripts import ServerScript, SandboxScript, GenerationClientScript - from nemo_run import Script - log_dir = "/experiments/full_pipeline/logs" - - # Job 1: Preprocessing with custom Script - @dataclass(kw_only=True) - class PreprocessScript(Script): - input_file: str - output_file: str - - def __post_init__(self): - cmd = f"python preprocess.py --input {self.input_file} --output {self.output_file}" - self.inline = cmd - object.__setattr__(self, 'entrypoint', 'bash') - - preprocess_script = PreprocessScript( - input_file="data.jsonl", - output_file="processed.jsonl" + # Job 1: Preprocessing + preprocess = Command( + command="python preprocess.py --input data.jsonl --output processed.jsonl", + gpus=0, + name="preprocess" ) - preprocess = Command(script=preprocess_script, name="preprocess") prep_group = CommandGroup( commands=[preprocess], hardware=HardwareConfig(partition="cpu"), @@ -118,76 +72,39 @@ def __post_init__(self): ) prep_job = {"name": "prep", "group": prep_group} - # Job 2: Two different model servers (HETEROGENEOUS SLURM job with 2 het groups) - # 8B model group - server_8b = ServerScript( - server_type="vllm", - model_path="Qwen/Qwen2.5-Math-7B-Instruct", - server_args="--tensor-parallel-size 1" - ) - sandbox_8b = SandboxScript() - client_8b = GenerationClientScript( - output_dir="/results/eval_8b", - extra_arguments="++prompt_config=math", - servers=[server_8b], - model_names=["Qwen/Qwen2.5-Math-7B-Instruct"], - server_types=["vllm"], - sandbox=sandbox_8b, - with_sandbox=True, - ) + # Job 2: Two different model servers (HETEROGENEOUS SLURM job with 2 het components) + # Allocate ports for each server/sandbox pair + from nemo_skills.pipeline.utils.server import get_free_port + server_8b_port = get_free_port(strategy="random") + sandbox_8b_port = get_free_port(strategy="random") + server_32b_port = get_free_port(strategy="random") + sandbox_32b_port = get_free_port(strategy="random") - group_8b = CommandGroup( - commands=[ - Command(script=server_8b, container="vllm", name="server_8b"), - Command(script=sandbox_8b, container="nemo-skills", name="sandbox_8b"), - Command(script=client_8b, container="nemo-skills", name="eval_8b"), - ], - hardware=HardwareConfig(partition="batch", num_gpus=1), - name="eval_8b", - log_dir=log_dir - ) + # Build commands with cluster_config + server_8b_cmd, server_8b_meta = vllm_server_command(cluster_config, model="Qwen/Qwen3-8B", port=server_8b_port) + sandbox_8b_cmd, sandbox_8b_meta = sandbox_command(cluster_config, port=sandbox_8b_port) + server_32b_cmd, server_32b_meta = vllm_server_command(cluster_config, model="Qwen/Qwen3-32B", port=server_32b_port) + sandbox_32b_cmd, sandbox_32b_meta = sandbox_command(cluster_config, port=sandbox_32b_port) - # 32B model group - server_32b = ServerScript( - server_type="vllm", - model_path="Qwen/Qwen2.5-Math-32B-Instruct", - server_args="--tensor-parallel-size 4" - ) - sandbox_32b = SandboxScript() - client_32b = GenerationClientScript( - output_dir="/results/eval_32b", - extra_arguments="++prompt_config=math", - servers=[server_32b], - model_names=["Qwen/Qwen2.5-Math-32B-Instruct"], - server_types=["vllm"], - sandbox=sandbox_32b, - with_sandbox=True, - ) + server_8b = Command(command=server_8b_cmd, gpus=8, name="server_8b", metadata=server_8b_meta) + sandbox_8b = Command(command=sandbox_8b_cmd, name="sandbox_8b", metadata=sandbox_8b_meta) + eval_8b = Command(command="python eval.py --model 8b", gpus=1, name="eval_8b") - group_32b = CommandGroup( - commands=[ - Command(script=server_32b, container="vllm", name="server_32b"), - Command(script=sandbox_32b, container="nemo-skills", name="sandbox_32b"), - Command(script=client_32b, container="nemo-skills", name="eval_32b"), - ], - hardware=HardwareConfig(partition="batch", num_gpus=4), - name="eval_32b", - log_dir=log_dir - ) + server_32b = Command(command=server_32b_cmd, gpus=8, name="server_32b", metadata=server_32b_meta) + sandbox_32b = Command(command=sandbox_32b_cmd, name="sandbox_32b", metadata=sandbox_32b_meta) + eval_32b = Command(command="python eval.py --model 32b", gpus=1, name="eval_32b") + + group_8b = CommandGroup(commands=[server_8b, sandbox_8b, eval_8b], name="eval_8b", log_dir=log_dir) + group_32b = CommandGroup(commands=[server_32b, sandbox_32b, eval_32b], name="eval_32b", log_dir=log_dir) evals_job = {"name": "evals", "groups": [group_8b, group_32b], "dependencies": [prep_job]} # Job 3: Report generation (depends on both evaluations) - @dataclass(kw_only=True) - class ReportScript(Script): - output_file: str - - def __post_init__(self): - self.inline = f"python generate_report.py --output {self.output_file}" - object.__setattr__(self, 'entrypoint', 'bash') - - report_script = ReportScript(output_file="report.txt") - report = Command(script=report_script, name="report") + report = Command( + command="python generate_report.py --output report.txt", + gpus=0, + name="report" + ) report_group = CommandGroup(commands=[report], name="report", log_dir=log_dir) # Create pipeline with dependency graph @@ -204,56 +121,130 @@ def __post_init__(self): pipeline.run() """ +import logging +import shlex +from contextlib import nullcontext +from dataclasses import dataclass, field +from typing import Callable, Dict, List, Optional, Tuple, Union + +import nemo_run as run + +from nemo_skills.pipeline.utils import ( + get_env_variables, + get_executor, + get_exp, + get_exp_handles, + get_tunnel, + run_exp, + temporary_env_update, +) +from nemo_skills.pipeline.utils.commands import wrap_command +from nemo_skills.pipeline.utils.exp import ( + REUSE_CODE_EXP, + get_packaging_job_key, + install_packages_wrap, + tunnel_hash, +) +from nemo_skills.pipeline.utils.mounts import is_mounted_filepath +from nemo_skills.pipeline.utils.packager import get_registered_external_repo +from nemo_skills.utils import get_logger_name + LOG = logging.getLogger(get_logger_name(__file__)) @dataclass class Command: - """Declarative command for running tasks in containers using run.Script objects. + """Declarative command for running tasks in containers. + + The command can be either: + - A string: evaluated immediately + - A callable (lambda): evaluated lazily when the task is prepared - Example: - server = ServerScript(server_type="vllm", model_path="/models/llama", ...) - Command(script=server, container="vllm", name="my_server") + Lambdas are ONLY needed for cross-component references (hostname_ref, meta_ref). + The het_group_index isn't assigned until pipeline execution, so these must be lazy: + # Lambda is ESSENTIAL here - server.hostname_ref() and meta_ref() don't exist yet + client = Command(command=lambda: f"curl {server.hostname_ref()}:{server.meta_ref('port')}") """ - script: run.Script + # Command can be a string or callable (lambda). + # Lambdas are primarily used for cross-component references (hostname_ref, meta_ref). + command: Union[str, Callable] container: str = "nemo-skills" + gpus: Optional[int] = None + nodes: int = 1 name: str = "command" + working_dir: str = "/nemo_run/code" + env_vars: Dict[str, str] = field(default_factory=dict) + installation_command: Optional[str] = None + port: Optional[int] = None # Can be set from metadata + metadata: Dict[str, any] = field(default_factory=dict) # Stores metadata from command builders + het_group_index: Optional[int] = None # Set per-job by Pipeline (not global) + + def __post_init__(self): + # Wrap plain strings with environment setup + if isinstance(self.command, str) and (self.env_vars or self.working_dir): + self.command = wrap_command(self.command, self.working_dir, self.env_vars) + + def hostname_ref(self) -> str: + """Get hostname reference for hetjob cross-component communication.""" + if self.het_group_index is None: + return "127.0.0.1" # Local fallback + # For heterogeneous SLURM jobs, resolve nodelist to actual hostname + return f"$(scontrol show hostnames $SLURM_JOB_NODELIST_HET_GROUP_{self.het_group_index} | head -n1)" + + def meta_ref(self, key: str) -> str: + """Get metadata value (like port). Fails if key not found.""" + if key not in self.metadata: + raise KeyError( + f"Metadata key '{key}' not found in Command '{self.name}'. " + f"Available keys: {list(self.metadata.keys())}" + ) + return str(self.metadata[key]) - def prepare_for_execution(self, cluster_config: Dict) -> Tuple[run.Script, Dict]: - """Prepare script for execution. + def prepare_for_execution(self, cluster_config: Dict) -> Tuple[str, Dict]: + """Prepare command for execution. This method: - 1. Evaluates lazy commands (if script.inline is callable) - 2. Builds execution config from Script fields + 1. Evaluates callables (resolves cross-component references) + 2. Wraps with installation_command if provided Returns: - Tuple of (Script_object, execution_config) + Tuple of (final_command, execution_config) """ - runtime_metadata = {} - - # If script.inline is callable (lazy command building), evaluate it now - if callable(self.script.inline): - result = self.script.inline() + # 1. Evaluate if callable (for cross-component references like hostname_ref) + if callable(self.command): + result = self.command() if isinstance(result, tuple): - evaluated_command, runtime_metadata = result + final_command, runtime_metadata = result + # Deep merge metadata, especially environment dict + for key, value in runtime_metadata.items(): + if key == "environment" and key in self.metadata: + # Merge environment dicts instead of replacing + self.metadata[key].update(value) + else: + self.metadata[key] = value else: - evaluated_command = result + final_command = result + else: + final_command = self.command - # Update script.inline with evaluated command - self.script.set_inline(evaluated_command) + # 2. Wrap with installation_command if provided + if self.installation_command: + final_command = install_packages_wrap(final_command, self.installation_command) - # Build execution config from Script fields + # 3. Build execution config from metadata execution_config = { - "log_prefix": getattr(self.script, "log_prefix", "main"), - "environment": runtime_metadata.get("environment", {}), - "mounts": None, # Mounts not currently exposed by Scripts - "container": self.container, + "num_tasks": self.metadata.get("num_tasks", 1), + "num_gpus": self.metadata.get("gpus", self.gpus or 0), + "num_nodes": self.metadata.get("nodes", self.nodes), + "environment": self.metadata.get("environment", {}), + "log_prefix": self.metadata.get("log_prefix", "main"), + "mounts": self.metadata.get("mounts"), + "container": self.metadata.get("container", self.container), # Use container from metadata if available } - # Return the Script object itself - return self.script, execution_config + return final_command, execution_config def get_name(self) -> str: return self.name @@ -266,7 +257,6 @@ class HardwareConfig: partition: Optional[str] = None num_gpus: Optional[int] = None num_nodes: Optional[int] = None - num_tasks: Optional[int] = 1 sbatch_kwargs: Optional[dict] = None @@ -492,49 +482,16 @@ def run(self, dry_run: bool = False, log_dir: Optional[str] = None, _reuse_exp=N return exp - def _prepare_command(self, command, cluster_config: Dict) -> Tuple[run.Script, Dict]: - """Prepare command for execution. + def _prepare_command(self, command, cluster_config: Dict) -> Tuple[str, Dict]: + """Prepare command and handle mpirun wrapping.""" + final_cmd, exec_config = command.prepare_for_execution(cluster_config) - Returns: - Tuple of (Script_object, exec_config) - """ - script, exec_config = command.prepare_for_execution(cluster_config) - # Only rewrite paths for "none" executor (native execution without containers) - # For "local" executor (Docker), paths should stay as /nemo_run/code/... since - # that's where the code is mounted inside the container - if cluster_config.get("executor") == "none": - script = self._rewrite_local_paths(script) - # Note: mpirun wrapping for multi-task scripts is handled by the executor - return script, exec_config - - def _rewrite_local_paths(self, script: run.Script) -> run.Script: - """For executor='none', replace /nemo_run/code paths with local repo paths.""" - nemo_repo = get_registered_external_repo("nemo_skills") - if nemo_repo is None: - return script - - pkg_path = str(nemo_repo.path) - repo_root = str(nemo_repo.path.parent) - - def _replace(cmd: str) -> str: - return cmd.replace("/nemo_run/code/nemo_skills", pkg_path).replace("/nemo_run/code", repo_root) - - inline_cmd = script.inline - if isinstance(inline_cmd, str): - script.set_inline(_replace(inline_cmd)) - elif callable(inline_cmd): - original_inline = inline_cmd - - def wrapped_inline(): - result = original_inline() - if isinstance(result, tuple): - cmd, metadata = result - return _replace(cmd), metadata - return _replace(result) - - script.set_inline(wrapped_inline) - - return script + # Handle mpirun wrapping for non-SLURM executors + num_tasks = exec_config["num_tasks"] + if cluster_config["executor"] != "slurm" and num_tasks > 1: + final_cmd = f"mpirun --allow-run-as-root -np {num_tasks} bash -c {shlex.quote(final_cmd)}" + + return final_cmd, exec_config def _resolve_container(self, exec_config: Dict, command, cluster_config: Dict) -> str: """Resolve container name to image path.""" @@ -556,7 +513,6 @@ def _create_executor( total_het_groups: int, overlap: bool, dependencies: Optional[List] = None, - job_name_override: Optional[str] = None, ): """Create executor with optional environment update.""" env_context = ( @@ -569,10 +525,10 @@ def _create_executor( return get_executor( cluster_config=cluster_config, container=container_image, - num_nodes=hardware.num_nodes if hardware and hardware.num_nodes is not None else 1, - tasks_per_node=hardware.num_tasks if hardware and hardware.num_tasks is not None else 1, - gpus_per_node=hardware.num_gpus if hardware and hardware.num_gpus is not None else 0, - job_name=job_name_override if job_name_override else command.name, + num_nodes=exec_config["num_nodes"], + tasks_per_node=exec_config["num_tasks"], + gpus_per_node=exec_config["num_gpus"], + job_name=command.name, log_dir=log_dir, log_prefix=exec_config["log_prefix"], partition=hardware.partition if hardware else None, @@ -611,105 +567,81 @@ def _plan_and_add_job( if log_dir is None: raise ValueError(f"CommandGroup '{groups[0].name}' must have log_dir set, or provide it to pipeline.run()") - scripts: List[run.Script] = [] + commands: List[str] = [] executors: List = [] het_group_indices: List[int] = [] - # Assign het_group_index values before evaluating any commands so cross-references - # (e.g., hostname_ref) see the correct indices regardless of processing order. - for het_idx, group in enumerate(groups): - for command in group.commands: - command.script.het_group_index = het_idx if heterogeneous else None - - # Prepare commands once and collect runtime data for a second pass where we - # construct executors. This ensures all scripts have resolved cross-references. - prepared_commands: List[Dict] = [] + # In heterogeneous jobs, collect environment from all commands for cross-component refs shared_env_vars: Dict[str, str] = {} + if heterogeneous: + for het_idx, group in enumerate(groups): + for command in group.commands: + _, exec_config_probe = command.prepare_for_execution(cluster_config) + shared_env_vars.update(exec_config_probe.get("environment", {})) + + # Share packager across executors for efficiency (single-group only) + shared_packager = None + # Build commands and executors for het_idx, group in enumerate(groups): has_multiple_components = len(group.commands) > 1 total_het_groups = ( len(groups) if heterogeneous else (len(group.commands) if has_multiple_components else 1) ) - for comp_idx, command in enumerate(group.commands): - script, exec_config = self._prepare_command(command, cluster_config) - - if isinstance(script.inline, str): - if cluster_config.get("executor") not in ("none", "local"): - script.set_inline(wrap_python_path(script.inline)) - - prepared_commands.append( - { - "het_idx": het_idx, - "comp_idx": comp_idx, - "group": group, - "command": command, - "script": script, - "exec_config": exec_config, - "total_het_groups": total_het_groups, - "overlap": len(group.commands) > 1, - } - ) - - if heterogeneous: - shared_env_vars.update(exec_config.get("environment", {})) - - # Share packager across executors for efficiency (single-group only) - shared_packager = None - - # Build commands and executors using prepared data - for entry in prepared_commands: - het_idx = entry["het_idx"] - comp_idx = entry["comp_idx"] - group = entry["group"] - command = entry["command"] - script = entry["script"] - exec_config = entry["exec_config"] - total_het_groups = entry["total_het_groups"] - overlap = entry["overlap"] - - scripts.append(script) - - # Merge shared environment for heterogeneous jobs - if heterogeneous and shared_env_vars: - exec_config["environment"].update(shared_env_vars) - - # Resolve container and create executor - container_image = self._resolve_container(exec_config, command, cluster_config) - # Pass external dependencies only to the first executor (SLURM doesn't support per-component dependencies in hetjobs) - exec_dependencies = external_deps if (het_idx == 0 and comp_idx == 0) else None - - # Always use group.name for SLURM job name (consistent across all components) - # The group name is set to task_name in generate.py, without component suffixes - # Component names (like {task_name}_server, {task_name}_sandbox) are only used for log_prefix - job_name_for_slurm = group.name - - executor = self._create_executor( - command, - exec_config, - container_image, - cluster_config, - log_dir, - group.hardware, - heterogeneous, - het_idx if heterogeneous else comp_idx, - total_het_groups, - overlap, - dependencies=exec_dependencies, - job_name_override=job_name_for_slurm, + # For single-group jobs with multiple components, allow job-level GPU override for sbatch allocation + job_level_gpus = ( + group.hardware.num_gpus if (not heterogeneous and has_multiple_components and group.hardware) else None ) - # Share packager across executors for single-group jobs - if not heterogeneous: - if comp_idx == 0 and het_idx == 0: - shared_packager = executor.packager + for comp_idx, command in enumerate(group.commands): + # Assign het_group_index ONLY for heterogeneous jobs (per-job, not global) + # Non-heterogeneous jobs use localhost, so het_group_index should remain None + if heterogeneous: + command.het_group_index = het_idx else: - executor.packager = shared_packager + command.het_group_index = None + + final_cmd, exec_config = self._prepare_command(command, cluster_config) + commands.append(final_cmd) + + # Adjust GPU allocation (first component gets job-level GPUs for sbatch) for single-group jobs + exec_config["num_gpus"] = exec_config["num_gpus"] or 0 + if (not heterogeneous) and (comp_idx == 0) and (job_level_gpus is not None): + exec_config["num_gpus"] = job_level_gpus + + # Merge shared environment for heterogeneous jobs + if heterogeneous and shared_env_vars: + exec_config["environment"].update(shared_env_vars) + + # Resolve container and create executor + container_image = self._resolve_container(exec_config, command, cluster_config) + # Pass external dependencies only to the first executor (SLURM doesn't support per-component dependencies in hetjobs) + exec_dependencies = external_deps if (het_idx == 0 and comp_idx == 0) else None + executor = self._create_executor( + command, + exec_config, + container_image, + cluster_config, + log_dir, + group.hardware, + heterogeneous, + het_idx if heterogeneous else comp_idx, + total_het_groups, + (len(group.commands) > 1), + dependencies=exec_dependencies, + ) - executors.append(executor) - if heterogeneous: - het_group_indices.append(het_idx) + # Share packager across executors for single-group jobs + if not heterogeneous: + if comp_idx == 0 and het_idx == 0: + shared_packager = executor.packager + else: + executor.packager = shared_packager + + executors.append(executor) + if heterogeneous: + het_group_indices.append(het_idx) # For heterogeneous jobs, set het_group_indices on the first executor if heterogeneous and executors: @@ -744,7 +676,13 @@ def _plan_and_add_job( # If reuse_code=False, clear cache REUSE_CODE_EXP.pop(tunnel_hash(tunnel), None) - # Note: Path replacements for executor="none" are no longer needed with Script interface + # Handle executor="none" path replacements (single-group only) + if (not heterogeneous) and cluster_config["executor"] == "none": + for idx in range(len(commands)): + commands[idx] = commands[idx].replace( + "/nemo_run/code/nemo_skills", str(get_registered_external_repo("nemo_skills").path) + ) + commands[idx] = commands[idx].replace("/nemo_run/code", "./") # Ray metadata handling if self.with_ray and cluster_config["executor"] == "slurm": @@ -755,24 +693,19 @@ def _plan_and_add_job( # Add to experiment and return task ID # Note: Internal dependencies (task handles from same experiment) go to exp.add() # External dependencies (SLURM job IDs from other experiments) go to executor - if (not heterogeneous) and len(scripts) == 1: - # Single script - pass directly to exp.add() - if metadata: - scripts[0].metadata = metadata + if (not heterogeneous) and len(commands) == 1: task_id = exp.add( - scripts[0], + run.Script(inline=commands[0], metadata=metadata), executor=executors[0], name="nemo-run", dependencies=internal_deps, ) else: - # Multiple scripts or heterogeneous job - # Apply metadata to first script only - if metadata: - scripts[0].metadata = metadata - task_id = exp.add( - scripts, + [ + run.Script(inline=cmd, metadata=(metadata if idx == 0 else None)) + for idx, cmd in enumerate(commands) + ], executor=executors, name="nemo-run", dependencies=internal_deps, diff --git a/nemo_skills/pipeline/utils/generation.py b/nemo_skills/pipeline/utils/generation.py index 8ae4e96bb5..cd576053c1 100644 --- a/nemo_skills/pipeline/utils/generation.py +++ b/nemo_skills/pipeline/utils/generation.py @@ -17,7 +17,6 @@ import shlex import subprocess from collections import defaultdict -from typing import Any, List, Optional, Union from nemo_skills.pipeline.utils.cluster import get_tunnel from nemo_skills.pipeline.utils.mounts import get_unmounted_path @@ -27,81 +26,6 @@ LOG = logging.getLogger(get_logger_name(__file__)) -def normalize_models_config( - model: Optional[Union[str, List[str]]], -) -> List[str]: - """ - Normalize model specification to list. - - Handles both scalar and list inputs: - - CLI (Typer): Converts single values to single-element lists automatically - - Python API: Accepts both strings and lists - - Args: - model: Model path(s) - string or list from Python API, list from CLI - - Returns: - List of model paths - - Raises: - ValueError: If model is None or empty - """ - if model is None: - raise ValueError("Must specify --model") - - # Handle string (Python API with single model) - if isinstance(model, str): - return [model] - - # Handle list - if len(model) == 0: - raise ValueError("Must specify --model") - return list(model) - - -def normalize_parameter( - param_value: Any, - num_models: int, - param_name: str, -) -> List[Any]: - """ - Normalize a parameter to a per-model list. - - Handles both scalar and list inputs for flexible usage: - - CLI (Typer): Converts single values to single-element lists automatically - - Python API: Accepts both scalars and lists directly - - Broadcast logic: - - Scalar value: Broadcast to all models [value] * num_models - - Single-element list: Broadcast to all models - - Multi-element list: Must match num_models exactly - - Args: - param_value: Parameter value (scalar or list) - num_models: Number of models - param_name: Name of parameter (for error messages) - - Returns: - List of parameter values (one per model) - - Raises: - ValueError: If list length doesn't match num_models - """ - if not isinstance(param_value, list): - return [param_value] * num_models - - if len(param_value) == num_models: - return list(param_value) - - if len(param_value) == 1: - return param_value * num_models - - raise ValueError( - f"Parameter {param_name} has {len(param_value)} values but {num_models} models specified. " - f"Must be 1 value (broadcast) or {num_models} values (per-model)." - ) - - def get_chunked_rs_filename( output_dir: str, random_seed: int = None, @@ -370,20 +294,8 @@ def get_generation_cmd( wandb_parameters=None, with_sandbox: bool = False, script: str = "nemo_skills.inference.generate", - # Optional: for multi-model generation - server_addresses: Optional[List[str]] = None, - model_names: Optional[List[str]] = None, - server_types: Optional[List[str]] = None, ): - """Construct the generation command for language model inference. - - Supports both single-model and multi-model generation. For multi-model: - - server_addresses: List of server addresses (one per model) - - model_names: List of model names (one per model) - - server_types: List of server types (one per model) - - For single-model, server config is passed via extra_arguments. - """ + """Construct the generation command for language model inference.""" if input_file is None and input_dir is None: raise ValueError("Either input_file or input_dir must be provided.") if input_file is not None and input_dir is not None: @@ -401,7 +313,6 @@ def get_generation_cmd( output_dir=output_dir, random_seed=random_seed, ) - # Preamble for generation commands: added at executor/declarative level cmd = "export HYDRA_FULL_ERROR=1 && " # Separate Hydra config args (--config-*) from override args (++) @@ -416,22 +327,6 @@ def get_generation_cmd( else: # It's a module name, use -m flag cmd += f"python -m {script} {hydra_config_args} {common_args} " - - # Add multi-model configuration if provided - if server_addresses is not None and model_names is not None: - num_models = len(model_names) - if num_models > 1: - # Multi-model: pass server configuration as lists - model_names_arg = ",".join(model_names) - cmd += f"++server.model=[{model_names_arg}] " - - server_types_arg = ",".join(server_types) - cmd += f"++server.server_type=[{server_types_arg}] " - - server_addresses_arg = ",".join(server_addresses) - cmd += f"++server.base_url=[{server_addresses_arg}] " - # For n=1: server config is already in extra_arguments from configure_client - job_end_cmd = "" if random_seed is not None and input_dir is None: # if input_dir is not None, we default to greedy generations diff --git a/nemo_skills/pipeline/utils/scripts.py b/nemo_skills/pipeline/utils/scripts.py deleted file mode 100644 index 4e37a6b594..0000000000 --- a/nemo_skills/pipeline/utils/scripts.py +++ /dev/null @@ -1,419 +0,0 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -Script classes for NeMo-Skills pipeline components. - -These classes wrap NeMo-Run's run.Script interface to provide typed, reusable -job components (servers, clients, sandboxes) with explicit fields and -cross-component reference support for heterogeneous jobs. - -Example: - # Create a server script with automatic port allocation - server = ServerScript( - server_type="vllm", - model_path="/models/llama-8b", - cluster_config=cluster_config, - num_gpus=8, - ) - - # Create a client that references the server - client = GenerationClientScript( - output_dir="/results", - input_file="/data/input.jsonl", - server=server, # Cross-component reference - ) - - # Use in Command objects - Command(script=server, container="vllm", ...) - Command(script=client, container="nemo-skills", ...) -""" - -import logging -from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Tuple, Union - -import nemo_run as run - -from nemo_skills.pipeline.utils.commands import sandbox_command -from nemo_skills.pipeline.utils.exp import install_packages_wrap -from nemo_skills.pipeline.utils.generation import get_generation_cmd -from nemo_skills.pipeline.utils.server import get_free_port, get_server_command -from nemo_skills.utils import get_logger_name - -if TYPE_CHECKING: - # Avoid circular imports for type hints - pass - -LOG = logging.getLogger(get_logger_name(__file__)) - - -@dataclass -class BaseJobScript(run.Script): - """Base class for job component scripts with heterogeneous job support. - - This class provides: - - het_group_index tracking for cross-component references in heterogeneous SLURM jobs - - hostname_ref() method for getting hostnames in het jobs - - Common pattern for Script initialization - - Attributes: - het_group_index: Index in heterogeneous job group (set by Pipeline at runtime) - """ - - het_group_index: Optional[int] = field(default=None, init=False, repr=False) - installation_command: Optional[str] = None - entrypoint: str = field(default="bash", init=False) - - def __post_init__(self): - """Wrap inline command with installation_command if provided.""" - if not self.installation_command: - return - - if callable(self.inline): - original_inline = self.inline - - def wrapped_inline(): - result = original_inline() - if isinstance(result, tuple): - command, metadata = result - return install_packages_wrap(command, self.installation_command), metadata - return install_packages_wrap(result, self.installation_command) - - self.set_inline(wrapped_inline) - elif isinstance(self.inline, str): - self.set_inline(install_packages_wrap(self.inline, self.installation_command)) - - def set_inline(self, command: Union[str, Callable, run.Script]) -> None: - """Set the inline command safely on frozen dataclass.""" - object.__setattr__(self, "inline", command) - - def hostname_ref(self) -> str: - """Get hostname reference for hetjob cross-component communication. - - Returns a shell variable reference that resolves to the master node hostname - for this het group. Uses environment variables automatically exported by nemo-run: - SLURM_MASTER_NODE_HET_GROUP_0, SLURM_MASTER_NODE_HET_GROUP_1, etc. - - These are set via: - export SLURM_MASTER_NODE_HET_GROUP_N=$(scontrol show hostnames $SLURM_JOB_NODELIST_HET_GROUP_N | head -n1) - """ - if self.het_group_index is None: - return "127.0.0.1" # Local fallback for non-heterogeneous jobs - - # Use the environment variable exported by nemo-run - return f"${{SLURM_MASTER_NODE_HET_GROUP_{self.het_group_index}:-localhost}}" - - -@dataclass(kw_only=True) -class ServerScript(BaseJobScript): - """Script for model inference servers (vLLM, TRT-LLM, SGLang, etc.). - - This script wraps server command builders and provides: - - Automatic port allocation if not specified - - Type-safe server configuration - - Cross-component address sharing (get_address()) - - Resource requirement tracking (num_gpus, num_nodes, num_tasks) - - Attributes: - server_type: Type of server (vllm, trtllm, sglang, megatron, openai, etc.) - model_path: Path to model weights or model name for API services - cluster_config: Cluster configuration dictionary - num_gpus: Number of GPUs required (default: 8) - num_nodes: Number of nodes required (default: 1) - server_args: Additional server-specific arguments - server_entrypoint: Custom server entrypoint script (optional) - port: Server port (allocated automatically if None) - allocate_port: Whether to allocate port automatically (default: True) - num_tasks: Number of MPI tasks (computed in __post_init__) - log_prefix: Prefix for log files (default: "server") - - Example: - # Basic usage - server = ServerScript( - server_type="vllm", - model_path="/models/llama-3-8b", - cluster_config=cluster_config, - num_gpus=8, - ) - - # Access allocated port - print(f"Server will run on port {server.port}") - - # Get full address for client connection - address = server.get_address() # Returns "hostname:port" - """ - - server_type: str - model_path: str - cluster_config: Dict - num_gpus: int = 8 - num_nodes: int = 1 - server_args: str = "" - server_entrypoint: Optional[str] = None # Custom server entrypoint script - port: Optional[int] = None - allocate_port: bool = True - - # Computed fields (set in __post_init__) - num_tasks: int = field(init=False, repr=False) - log_prefix: str = field(default="server", init=False) - - def __post_init__(self): - """Initialize server script. - - - Allocates port if not provided - - Builds server command using get_server_command() - - Sets self.inline to the command string - - Computes num_tasks from server command builder - """ - # Allocate port if not provided - if self.port is None and self.allocate_port: - self.port = get_free_port(strategy="random") - LOG.debug(f"Allocated port {self.port} for {self.server_type} server") - - # Build server command - cmd, self.num_tasks = get_server_command( - server_type=self.server_type, - num_gpus=self.num_gpus, - num_nodes=self.num_nodes, - model_path=self.model_path, - cluster_config=self.cluster_config, - server_port=self.port, - server_args=self.server_args, - server_entrypoint=self.server_entrypoint, - ) - - self.set_inline(cmd) - super().__post_init__() - - def get_address(self) -> str: - """Get server address for client connections. - - Returns hostname:port string that clients can use to connect. - In heterogeneous jobs, hostname_ref() returns a bash expression - that resolves at runtime. - - Returns: - Server address in format "hostname:port" - - Example: - # Use in client command - client_cmd = f"python client.py --server-url http://{server.get_address()}" - """ - return f"{self.hostname_ref()}:{self.port}" - - -@dataclass(kw_only=True) -class SandboxScript(BaseJobScript): - """Script for code execution sandbox container. - - The sandbox provides a secure environment for executing LLM-generated code. - This script wraps sandbox command builders and provides: - - Automatic port allocation - - Mount configuration (can optionally keep mounts, though risky) - - Type-safe sandbox configuration - - Attributes: - cluster_config: Cluster configuration dictionary - port: Sandbox port (allocated automatically if None) - keep_mounts: Whether to keep filesystem mounts (default: False, risky if True). - Note: This is stored for documentation but actually handled at - the executor level, not in the sandbox command itself. - allocate_port: Whether to allocate port automatically (default: True) - log_prefix: Prefix for log files (default: "sandbox") - - Example: - sandbox = SandboxScript( - cluster_config=cluster_config, - keep_mounts=False, # Safer: sandbox has no access to mounted paths - ) - - # Client can reference sandbox port - client = GenerationClientScript(..., sandbox=sandbox) - """ - - cluster_config: Dict - port: Optional[int] = None - keep_mounts: bool = False - allocate_port: bool = True - env_overrides: Optional[List[str]] = None # Extra env vars in KEY=VALUE form - log_prefix: str = field(default="sandbox", init=False) - - def __post_init__(self): - """Initialize sandbox script. - - - Allocates port if not provided - - Builds sandbox command using sandbox_command() - - Sets self.inline to a callable that returns command and environment vars - """ - # Allocate port if not provided - if self.port is None and self.allocate_port: - self.port = get_free_port(strategy="random") - LOG.debug(f"Allocated port {self.port} for sandbox") - - # Build sandbox command and metadata (including environment vars) - # Note: keep_mounts is handled at the executor level, not in the command itself - cmd, metadata = sandbox_command( - cluster_config=self.cluster_config, - port=self.port, - ) - - # Use a callable to return both command and environment variables - # This ensures the sandbox's LISTEN_PORT and NGINX_PORT are properly set - def build_cmd() -> Tuple[str, Dict]: - env = dict(metadata.get("environment", {})) - # Apply user-specified environment overrides - if self.env_overrides: - for override in self.env_overrides: - key, value = override.split("=", 1) - env[key] = value - return cmd, {"environment": env} - - self.set_inline(build_cmd) - super().__post_init__() - - -@dataclass(kw_only=True) -class GenerationClientScript(BaseJobScript): - """Script for LLM generation/inference client. - - This script wraps generation command builders and provides: - - Cross-component references to multiple servers and sandbox - - Lazy command building for runtime hostname resolution - - Type-safe generation configuration - - Environment variable handling for sandbox/server communication - - Attributes: - output_dir: Directory for output files - input_file: Input JSONL file (mutually exclusive with input_dir) - input_dir: Input directory (mutually exclusive with input_file) - extra_arguments: Additional arguments for generation script - random_seed: Random seed for sampling (optional) - chunk_id: Chunk ID for parallel processing (optional) - num_chunks: Total number of chunks (required if chunk_id set) - preprocess_cmd: Command to run before generation (optional) - postprocess_cmd: Command to run after generation (optional) - wandb_parameters: WandB logging configuration (optional) - with_sandbox: Whether sandbox is enabled - script: Module or file path for generation script (default: nemo_skills.inference.generate) - servers: List of ServerScript references (None for pre-hosted servers) - server_addresses_prehosted: Addresses for pre-hosted servers (parallel to servers list) - model_names: Model names for multi-model generation (optional) - server_types: Server types for multi-model generation (optional) - sandbox: Reference to SandboxScript for cross-component communication (optional) - log_prefix: Prefix for log files (default: "main") - - Examples: - # Single server - client = GenerationClientScript( - output_dir="/results", - input_file="/data/input.jsonl", - servers=[server_script], - model_names=["llama-8b"], - server_types=["vllm"], - ) - - # Multi-model with self-hosted and pre-hosted servers - client = GenerationClientScript( - output_dir="/results", - input_file="/data/input.jsonl", - servers=[server1, server2, None], # None = pre-hosted - server_addresses_prehosted=["", "", "https://api.openai.com"], - model_names=["llama-8b", "llama-70b", "gpt-4"], - server_types=["vllm", "vllm", "openai"], - sandbox=sandbox_script, - with_sandbox=True, - ) - """ - - output_dir: str - input_file: Optional[str] = None - input_dir: Optional[str] = None - extra_arguments: str = "" - random_seed: Optional[int] = None - chunk_id: Optional[int] = None - num_chunks: Optional[int] = None - preprocess_cmd: Optional[str] = None - postprocess_cmd: Optional[str] = None - wandb_parameters: Optional[Dict] = None - with_sandbox: bool = False - script: str = "nemo_skills.inference.generate" - - # Cross-component references for single/multi-model - servers: Optional[List[Optional["ServerScript"]]] = None - server_addresses_prehosted: Optional[List[str]] = None - model_names: Optional[List[str]] = None - server_types: Optional[List[str]] = None - sandbox: Optional["SandboxScript"] = None - - log_prefix: str = field(default="main", init=False) - - def __post_init__(self): - """Initialize generation client script with lazy command building. - - Builds command lazily via a callable that is evaluated when het_group_index - is assigned, allowing hostname_ref() to resolve correctly for heterogeneous jobs. - - This works for both cases: - - With cross-refs: Resolves server hostnames and sandbox ports at runtime - - Without cross-refs: Just builds the command string (no runtime resolution needed) - """ - - def build_cmd() -> Tuple[str, Dict]: - """Build command at runtime when cross-refs are resolved.""" - env_vars = {} - - # Add sandbox port to environment if sandbox is referenced - if self.sandbox: - env_vars["NEMO_SKILLS_SANDBOX_PORT"] = str(self.sandbox.port) - - # Build server addresses if servers are provided - server_addresses = None - if self.servers is not None: - server_addresses = [] - for server_idx, server_script in enumerate(self.servers): - if server_script is not None: - # Self-hosted: construct address from hostname and port refs - addr = f"{server_script.hostname_ref()}:{server_script.port}" - else: - # Pre-hosted: use the address from server_addresses_prehosted - addr = self.server_addresses_prehosted[server_idx] - server_addresses.append(addr) - - # Build generation command - cmd = get_generation_cmd( - output_dir=self.output_dir, - input_file=self.input_file, - input_dir=self.input_dir, - extra_arguments=self.extra_arguments, - random_seed=self.random_seed, - chunk_id=self.chunk_id, - num_chunks=self.num_chunks, - preprocess_cmd=self.preprocess_cmd, - postprocess_cmd=self.postprocess_cmd, - wandb_parameters=self.wandb_parameters, - with_sandbox=self.with_sandbox, - script=self.script, - # Multi-model parameters (None for single-model) - server_addresses=server_addresses, - model_names=self.model_names, - server_types=self.server_types, - ) - - # Return command and runtime metadata (environment vars) - return cmd, {"environment": env_vars} - - # Always use lazy command building - self.set_inline(build_cmd) - super().__post_init__() diff --git a/tests/gpu-tests/test_eval.py b/tests/gpu-tests/test_eval.py index 05dccf6b51..31c8f2cccf 100644 --- a/tests/gpu-tests/test_eval.py +++ b/tests/gpu-tests/test_eval.py @@ -44,7 +44,7 @@ "mbpp", "mmau-pro", "asr-leaderboard", - "mrcr", + "aalcr", # Has tokenization mismatch issues "audiobench", "librispeech-pc", } diff --git a/tests/test_declarative_pipeline.py b/tests/test_declarative_pipeline.py index 9d76fd4721..92117e403d 100644 --- a/tests/test_declarative_pipeline.py +++ b/tests/test_declarative_pipeline.py @@ -16,7 +16,6 @@ import json import os -from typing import Callable, Optional from unittest.mock import MagicMock, patch import pytest @@ -27,83 +26,127 @@ from nemo_skills.pipeline.utils.declarative import Command, CommandGroup, HardwareConfig, Pipeline -class DummyScript: - """Minimal run.Script stand-in for unit tests.""" - - def __init__(self, inline: str | Callable | None = "echo test"): - self.inline = inline - self.log_prefix = "main" - self.metadata = {} - self.het_group_index: Optional[int] = None - - def set_inline(self, inline): - self.inline = inline - - def hostname_ref(self) -> str: - if self.het_group_index is None: - return "127.0.0.1" - return f"${{SLURM_MASTER_NODE_HET_GROUP_{self.het_group_index}:-localhost}}" +class TestCommand: + """Test Command class functionality.""" + def test_command_basic_string(self): + """Test creating a Command with a simple string.""" + cmd = Command(command="echo hello", name="test") + assert cmd.name == "test" + assert cmd.container == "nemo-skills" + assert cmd.gpus is None + assert cmd.nodes == 1 -def make_command(*, inline: str | Callable | None = "echo test", name: str = "cmd", script: DummyScript | None = None): - """Helper to build Command objects with DummyScript instances.""" - script_obj = script or DummyScript(inline=inline) - return Command(script=script_obj, name=name) + def test_command_with_metadata(self): + """Test Command with metadata passed separately.""" + cmd = Command(command="echo hello", name="server", metadata={"port": 8080, "log_prefix": "server"}) + assert cmd.metadata["port"] == 8080 + assert cmd.metadata["log_prefix"] == "server" + # Command gets wrapped with working_dir by default + assert "echo hello" in cmd.command + def test_command_with_callable(self): + """Test Command with callable that returns tuple.""" -class TestCommand: - """Tests for the new Script-based Command wrapper.""" + def make_cmd(): + return ("echo world", {"port": 5000}) - def test_command_basic_script(self): - cmd = make_command(inline="echo hello", name="test") - assert cmd.name == "test" - assert cmd.container == "nemo-skills" - assert cmd.script.inline == "echo hello" + cmd = Command(command=make_cmd, name="dynamic") + assert callable(cmd.command) + assert cmd.name == "dynamic" def test_command_prepare_for_execution_string(self): - cmd = make_command(inline="python script.py", name="test") + """Test prepare_for_execution with string command.""" + cmd = Command(command="python script.py", gpus=2, name="test") cluster_config = {"executor": "local", "containers": {}} - script_obj, exec_config = cmd.prepare_for_execution(cluster_config) + final_cmd, exec_config = cmd.prepare_for_execution(cluster_config) - assert script_obj.inline == "python script.py" - assert exec_config["log_prefix"] == "main" - assert exec_config["environment"] == {} + assert "python script.py" in final_cmd + assert exec_config["num_gpus"] == 2 + assert exec_config["num_nodes"] == 1 + assert exec_config["num_tasks"] == 1 def test_command_prepare_for_execution_callable(self): - script = DummyScript(inline=lambda: "echo test") - cmd = make_command(name="test", script=script) + """Test prepare_for_execution with callable command.""" + + def make_cmd(): + return "echo test" + + cmd = Command(command=make_cmd, name="test") cluster_config = {"executor": "local", "containers": {}} - script_obj, _ = cmd.prepare_for_execution(cluster_config) - assert script_obj.inline == "echo test" + final_cmd, exec_config = cmd.prepare_for_execution(cluster_config) + + assert final_cmd == "echo test" def test_command_prepare_for_execution_callable_with_metadata(self): + """Test prepare_for_execution with callable returning tuple.""" + def make_cmd(): - return ("echo metadata", {"environment": {"VAR": "value"}}) + return ("echo metadata", {"num_tasks": 4, "environment": {"VAR": "value"}}) - script = DummyScript(inline=make_cmd) - cmd = make_command(name="test", script=script) + cmd = Command(command=make_cmd, name="test") cluster_config = {"executor": "local", "containers": {}} - _, exec_config = cmd.prepare_for_execution(cluster_config) + final_cmd, exec_config = cmd.prepare_for_execution(cluster_config) + assert final_cmd == "echo metadata" + assert exec_config["num_tasks"] == 4 assert exec_config["environment"]["VAR"] == "value" - def test_command_hostname_ref_none(self): - script = DummyScript() - cmd = make_command(name="test", script=script) + def test_command_meta_ref(self): + """Test meta_ref for accessing metadata.""" + cmd = Command(command="echo test", name="server", metadata={"port": 8080, "host": "localhost"}) + + assert cmd.meta_ref("port") == "8080" + assert cmd.meta_ref("host") == "localhost" + + def test_command_meta_ref_missing_key(self): + """Test meta_ref with missing key raises KeyError.""" + cmd = Command(command="echo test", name="test") - assert script.hostname_ref() == "127.0.0.1" - assert cmd.get_name() == "test" + with pytest.raises(KeyError, match="Metadata key 'port' not found"): + cmd.meta_ref("port") + + def test_command_hostname_ref_none(self): + """Test hostname_ref returns localhost when het_group_index is None.""" + cmd = Command(command="echo test", name="test") + assert cmd.het_group_index is None + assert cmd.hostname_ref() == "127.0.0.1" def test_command_hostname_ref_heterogeneous(self): - script = DummyScript() - script.het_group_index = 2 - make_command(name="test", script=script) + """Test hostname_ref returns SLURM variable when het_group_index is set.""" + cmd = Command(command="echo test", name="test") + cmd.het_group_index = 2 - hostname = script.hostname_ref() - assert "${SLURM_MASTER_NODE_HET_GROUP_2" in hostname + hostname = cmd.hostname_ref() + assert "$SLURM_JOB_NODELIST_HET_GROUP_2" in hostname + assert "scontrol" in hostname + + def test_command_with_installation_command(self): + """Test Command with installation_command.""" + cmd = Command(command="python script.py", installation_command="pip install package", name="test") + cluster_config = {"executor": "local", "containers": {}} + + final_cmd, _ = cmd.prepare_for_execution(cluster_config) + + # Installation command should be wrapped around the main command + assert "pip install package" in final_cmd + assert "python script.py" in final_cmd + + def test_command_env_vars_wrapping(self): + """Test that env_vars and working_dir are applied to string commands.""" + cmd = Command( + command="python script.py", + env_vars={"MY_VAR": "value"}, + working_dir="/custom/path", + name="test", + ) + + # The command should be wrapped with env setup + assert "export MY_VAR=value" in cmd.command + assert "cd /custom/path" in cmd.command class TestCommandGroup: @@ -111,8 +154,8 @@ class TestCommandGroup: def test_commandgroup_basic(self): """Test creating a basic CommandGroup.""" - cmd1 = make_command(inline="echo 1", name="cmd1") - cmd2 = make_command(inline="echo 2", name="cmd2") + cmd1 = Command(command="echo 1", name="cmd1") + cmd2 = Command(command="echo 2", name="cmd2") group = CommandGroup(commands=[cmd1, cmd2], name="test_group") @@ -122,7 +165,7 @@ def test_commandgroup_basic(self): def test_commandgroup_with_hardware(self): """Test CommandGroup with HardwareConfig.""" - cmd = make_command(inline="echo test", name="cmd") + cmd = Command(command="echo test", name="cmd") hardware = HardwareConfig(partition="batch", sbatch_kwargs={"time_min": "01:00:00"}, num_gpus=8) group = CommandGroup(commands=[cmd], hardware=hardware, name="gpu_group") @@ -133,7 +176,7 @@ def test_commandgroup_with_hardware(self): def test_commandgroup_with_log_dir(self): """Test CommandGroup with log_dir.""" - cmd = make_command(inline="echo test", name="cmd") + cmd = Command(command="echo test", name="cmd") group = CommandGroup(commands=[cmd], log_dir="/logs/test", name="group") assert group.log_dir == "/logs/test" @@ -144,7 +187,7 @@ class TestPipeline: def test_pipeline_with_single_job(self): """Test Pipeline with single job.""" - cmd = make_command(inline="echo test", name="cmd") + cmd = Command(command="echo test", name="cmd") group = CommandGroup(commands=[cmd], name="group") cluster_config = {"executor": "local", "containers": {}} @@ -161,10 +204,10 @@ def test_pipeline_with_single_job(self): def test_pipeline_with_jobs(self): """Test Pipeline with jobs parameter (full format with dependencies).""" - cmd1 = make_command(inline="echo 1", name="cmd1") + cmd1 = Command(command="echo 1", name="cmd1") group1 = CommandGroup(commands=[cmd1], name="group1", log_dir="/logs") - cmd2 = make_command(inline="echo 2", name="cmd2") + cmd2 = Command(command="echo 2", name="cmd2") group2 = CommandGroup(commands=[cmd2], name="group2", log_dir="/logs") job1 = {"name": "job1", "group": group1} @@ -189,7 +232,7 @@ def test_pipeline_requires_jobs(self): def test_pipeline_with_run_after(self): """Test Pipeline with run_after parameter.""" - cmd = make_command(inline="echo test", name="cmd") + cmd = Command(command="echo test", name="cmd") group = CommandGroup(commands=[cmd], name="group") cluster_config = {"executor": "local", "containers": {}} @@ -205,7 +248,7 @@ def test_pipeline_with_run_after(self): def test_pipeline_with_run_after_list(self): """Test Pipeline with run_after as list.""" - cmd = make_command(inline="echo test", name="cmd") + cmd = Command(command="echo test", name="cmd") group = CommandGroup(commands=[cmd], name="group") cluster_config = {"executor": "local", "containers": {}} @@ -221,7 +264,7 @@ def test_pipeline_with_run_after_list(self): def test_pipeline_cluster_config_passed_directly(self): """Test that cluster_config is passed directly (no more string resolution).""" - cmd = make_command(inline="echo test", name="cmd") + cmd = Command(command="echo test", name="cmd") group = CommandGroup(commands=[cmd], name="group") cluster_config = {"executor": "local", "containers": {}} @@ -256,7 +299,7 @@ def test_pipeline_run_basic(self, mock_run_exp, mock_env_vars, mock_get_exp): mock_get_exp.return_value.__enter__.return_value = mock_exp # Create pipeline - cmd = make_command(inline="echo test", name="cmd") + cmd = Command(command="echo test", name="cmd") group = CommandGroup(commands=[cmd], name="group", log_dir="/logs") pipeline = Pipeline( name="test", cluster_config=mock_config, jobs=[{"name": "job1", "group": group}], skip_hf_home_check=True @@ -286,10 +329,10 @@ def test_pipeline_run_with_dependencies(self, mock_run_exp, mock_env_vars, mock_ mock_get_exp.return_value.__enter__.return_value = mock_exp # Create pipeline with internal dependencies - cmd1 = make_command(inline="echo 1", name="cmd1") + cmd1 = Command(command="echo 1", name="cmd1") group1 = CommandGroup(commands=[cmd1], name="group1", log_dir="/logs") - cmd2 = make_command(inline="echo 2", name="cmd2") + cmd2 = Command(command="echo 2", name="cmd2") group2 = CommandGroup(commands=[cmd2], name="group2", log_dir="/logs") job1 = {"name": "job1", "group": group1, "dependencies": []} @@ -332,7 +375,7 @@ def test_pipeline_hf_home_validation(self, mock_get_executor, mock_is_mounted, m mock_exp.add.return_value = "handle" mock_get_exp.return_value.__enter__.return_value = mock_exp - cmd = make_command(inline="echo test", name="cmd") + cmd = Command(command="echo test", name="cmd") group = CommandGroup(commands=[cmd], name="group", log_dir="/logs") pipeline = Pipeline(name="test", cluster_config=mock_config, jobs=[{"name": "job1", "group": group}]) @@ -348,7 +391,7 @@ def test_pipeline_hf_home_missing(self, mock_env_vars): mock_config = {"executor": "slurm", "containers": {}} mock_env_vars.return_value = {} # No HF_HOME - cmd = make_command(inline="echo test", name="cmd") + cmd = Command(command="echo test", name="cmd") group = CommandGroup(commands=[cmd], name="group", log_dir="/logs") # Should raise in __init__ now, not run() @@ -363,7 +406,7 @@ def test_pipeline_hf_home_not_mounted(self, mock_is_mounted, mock_env_vars): mock_env_vars.return_value = {"HF_HOME": "/hf"} mock_is_mounted.return_value = False - cmd = make_command(inline="echo test", name="cmd") + cmd = Command(command="echo test", name="cmd") group = CommandGroup(commands=[cmd], name="group", log_dir="/logs") # Should raise in __init__ now, not run() @@ -389,8 +432,8 @@ def test_het_group_index_non_heterogeneous(self, mock_env_vars, mock_get_exp): mock_get_exp.return_value.__enter__.return_value = mock_exp # Create single-group job with multiple components - cmd1 = make_command(inline="echo 1", name="cmd1") - cmd2 = make_command(inline="echo 2", name="cmd2") + cmd1 = Command(command="echo 1", name="cmd1") + cmd2 = Command(command="echo 2", name="cmd2") group = CommandGroup(commands=[cmd1, cmd2], name="group", log_dir="/logs") pipeline = Pipeline( @@ -399,10 +442,10 @@ def test_het_group_index_non_heterogeneous(self, mock_env_vars, mock_get_exp): pipeline.run(dry_run=True) # Both commands should have None het_group_index (localhost communication) - assert cmd1.script.het_group_index is None - assert cmd2.script.het_group_index is None - assert cmd1.script.hostname_ref() == "127.0.0.1" - assert cmd2.script.hostname_ref() == "127.0.0.1" + assert cmd1.het_group_index is None + assert cmd2.het_group_index is None + assert cmd1.hostname_ref() == "127.0.0.1" + assert cmd2.hostname_ref() == "127.0.0.1" @patch("nemo_skills.pipeline.utils.declarative.get_exp") @patch("nemo_skills.pipeline.utils.declarative.get_env_variables") @@ -419,10 +462,10 @@ def test_het_group_index_heterogeneous(self, mock_env_vars, mock_get_exp): mock_get_exp.return_value.__enter__.return_value = mock_exp # Create multi-group heterogeneous job - cmd1 = make_command(inline="echo 1", name="cmd1") + cmd1 = Command(command="echo 1", name="cmd1") group1 = CommandGroup(commands=[cmd1], name="group1", log_dir="/logs") - cmd2 = make_command(inline="echo 2", name="cmd2") + cmd2 = Command(command="echo 2", name="cmd2") group2 = CommandGroup(commands=[cmd2], name="group2", log_dir="/logs") jobs = [{"name": "hetjob", "groups": [group1, group2]}] @@ -430,10 +473,10 @@ def test_het_group_index_heterogeneous(self, mock_env_vars, mock_get_exp): pipeline.run(dry_run=True) # Commands should have het_group_index 0 and 1 - assert cmd1.script.het_group_index == 0 - assert cmd2.script.het_group_index == 1 - assert "SLURM_MASTER_NODE_HET_GROUP_0" in cmd1.script.hostname_ref() - assert "SLURM_MASTER_NODE_HET_GROUP_1" in cmd2.script.hostname_ref() + assert cmd1.het_group_index == 0 + assert cmd2.het_group_index == 1 + assert "$SLURM_JOB_NODELIST_HET_GROUP_0" in cmd1.hostname_ref() + assert "$SLURM_JOB_NODELIST_HET_GROUP_1" in cmd2.hostname_ref() @patch("nemo_skills.pipeline.utils.declarative.get_exp") @patch("nemo_skills.pipeline.utils.declarative.get_env_variables") @@ -450,16 +493,16 @@ def test_het_group_index_per_job_not_global(self, mock_env_vars, mock_get_exp): mock_get_exp.return_value.__enter__.return_value = mock_exp # Create two separate heterogeneous jobs - cmd1 = make_command(inline="echo 1", name="cmd1") + cmd1 = Command(command="echo 1", name="cmd1") group1 = CommandGroup(commands=[cmd1], name="group1", log_dir="/logs") - cmd2 = make_command(inline="echo 2", name="cmd2") + cmd2 = Command(command="echo 2", name="cmd2") group2 = CommandGroup(commands=[cmd2], name="group2", log_dir="/logs") - cmd3 = make_command(inline="echo 3", name="cmd3") + cmd3 = Command(command="echo 3", name="cmd3") group3 = CommandGroup(commands=[cmd3], name="group3", log_dir="/logs") - cmd4 = make_command(inline="echo 4", name="cmd4") + cmd4 = Command(command="echo 4", name="cmd4") group4 = CommandGroup(commands=[cmd4], name="group4", log_dir="/logs") jobs = [ @@ -470,10 +513,10 @@ def test_het_group_index_per_job_not_global(self, mock_env_vars, mock_get_exp): pipeline.run(dry_run=True) # Both jobs should have het_group_index starting from 0 - assert cmd1.script.het_group_index == 0 - assert cmd2.script.het_group_index == 1 - assert cmd3.script.het_group_index == 0 # Starts from 0 again! - assert cmd4.script.het_group_index == 1 + assert cmd1.het_group_index == 0 + assert cmd2.het_group_index == 1 + assert cmd3.het_group_index == 0 # Starts from 0 again! + assert cmd4.het_group_index == 1 class TestDependencyResolution: @@ -493,7 +536,7 @@ def test_dependency_none_handling(self, mock_env_vars, mock_get_exp): mock_exp.add.return_value = "handle" mock_get_exp.return_value.__enter__.return_value = mock_exp - cmd = make_command(inline="echo test", name="cmd") + cmd = Command(command="echo test", name="cmd") group = CommandGroup(commands=[cmd], name="group", log_dir="/logs") jobs = [{"name": "job", "group": group, "dependencies": None}] @@ -516,7 +559,7 @@ def test_pipeline_run_after_applies_to_jobs(self, mock_env_vars, mock_get_exp): mock_exp.add.return_value = "handle" mock_get_exp.return_value.__enter__.return_value = mock_exp - cmd = make_command(inline="echo test", name="cmd") + cmd = Command(command="echo test", name="cmd") group = CommandGroup(commands=[cmd], name="group", log_dir="/logs") pipeline = Pipeline( @@ -546,7 +589,7 @@ def test_pipeline_job_missing_group_or_groups(self): def test_commandgroup_missing_log_dir(self): """Test that CommandGroup without log_dir raises error during execution.""" mock_config = {"executor": "none", "containers": {}} - cmd = make_command(inline="echo test", name="cmd") + cmd = Command(command="echo test", name="cmd") group = CommandGroup(commands=[cmd], name="group") # No log_dir pipeline = Pipeline(name="test", cluster_config=mock_config, jobs=[{"name": "job1", "group": group}]) @@ -583,14 +626,14 @@ def test_multiple_internal_dependencies(self): } # Job 1 and Job 2: independent - cmd1 = make_command(inline="echo job1", name="job1") + cmd1 = Command(command="echo job1", name="job1") group1 = CommandGroup(commands=[cmd1], name="group1", log_dir="/tmp/logs") - cmd2 = make_command(inline="echo job2", name="job2") + cmd2 = Command(command="echo job2", name="job2") group2 = CommandGroup(commands=[cmd2], name="group2", log_dir="/tmp/logs") # Job 3: depends on both job1 and job2 - cmd3 = make_command(inline="echo job3", name="job3") + cmd3 = Command(command="echo job3", name="job3") group3 = CommandGroup(commands=[cmd3], name="group3", log_dir="/tmp/logs") job1_spec = {"name": "job1", "group": group1} @@ -672,11 +715,11 @@ def mock_get_executor(**kwargs): } # Job 1: depends on external experiment - cmd1 = make_command(inline="echo job1", name="job1") + cmd1 = Command(command="echo job1", name="job1") group1 = CommandGroup(commands=[cmd1], name="group1", log_dir="/tmp/logs") # Job 2: depends on job1 (internal) AND external experiment - cmd2 = make_command(inline="echo job2", name="job2") + cmd2 = Command(command="echo job2", name="job2") group2 = CommandGroup(commands=[cmd2], name="group2", log_dir="/tmp/logs") job1_spec = { @@ -888,38 +931,35 @@ def capture_env_update(cluster_config, updates): # Debug: print what we captured print(f"Captured env updates: {env_updates_captured}") - # Verify both sandbox and client environment variables are captured - assert len(env_updates_captured) >= 2, ( - f"Expected at least 2 environment updates (sandbox + client), got {len(env_updates_captured)}: {env_updates_captured}" - ) - - # Find the sandbox and client environment updates - sandbox_env = None + # Find the client and sandbox environment updates client_env = None + sandbox_env = None + for env_update in env_updates_captured: - if "LISTEN_PORT" in env_update and "NGINX_PORT" in env_update: - sandbox_env = env_update if "NEMO_SKILLS_SANDBOX_PORT" in env_update: client_env = env_update + elif "LISTEN_PORT" in env_update and "NGINX_PORT" in env_update: + sandbox_env = env_update - # Verify sandbox got LISTEN_PORT and NGINX_PORT - assert sandbox_env is not None, ( - f"LISTEN_PORT/NGINX_PORT not set for sandbox command: {env_updates_captured}" + # Verify client got NEMO_SKILLS_SANDBOX_PORT (old behavior: exp.py line 493) + # This is the key fix - ensuring sandbox port is passed to client + assert client_env is not None, ( + f"Client environment update not found. Captured updates: {env_updates_captured}\n" + f"This means NEMO_SKILLS_SANDBOX_PORT was not set for the client command, " + f"so the Sandbox class cannot connect to the sandbox server." ) - assert sandbox_env["LISTEN_PORT"] == sandbox_env["NGINX_PORT"], ( - f"LISTEN_PORT and NGINX_PORT should match: {sandbox_env}" + assert "NEMO_SKILLS_SANDBOX_PORT" in client_env, ( + "NEMO_SKILLS_SANDBOX_PORT not set for client command" ) - # Verify client got NEMO_SKILLS_SANDBOX_PORT - assert client_env is not None, ( - f"NEMO_SKILLS_SANDBOX_PORT not set for client command: {env_updates_captured}" + # Verify sandbox got its environment vars (old behavior: exp.py lines 525-538) + assert sandbox_env is not None, ( + f"Sandbox environment update not found. Captured: {env_updates_captured}" ) + assert "LISTEN_PORT" in sandbox_env, "LISTEN_PORT not set for sandbox" + assert "NGINX_PORT" in sandbox_env, "NGINX_PORT not set for sandbox" - # Verify the ports match between sandbox and client - assert client_env["NEMO_SKILLS_SANDBOX_PORT"] == sandbox_env["LISTEN_PORT"], ( - f"Sandbox port mismatch: client has {client_env['NEMO_SKILLS_SANDBOX_PORT']}, " - f"sandbox has {sandbox_env['LISTEN_PORT']}" - ) + # This test verifies the fix works end-to-end through the actual generate() function if __name__ == "__main__": diff --git a/tests/test_generation.py b/tests/test_generation.py index 2693d62241..b69b526a0e 100644 --- a/tests/test_generation.py +++ b/tests/test_generation.py @@ -16,12 +16,12 @@ # running most things through subprocess since that's how it's usually used import subprocess +from unittest.mock import MagicMock import pytest from nemo_skills.evaluation.metrics import ComputeMetrics -from nemo_skills.pipeline.generate import _create_job_unified -from nemo_skills.pipeline.utils.scripts import ServerScript +from nemo_skills.pipeline.generate import _create_commandgroup_from_config def test_eval_gsm8k_api(tmp_path): @@ -153,42 +153,36 @@ def test_generate_openai_format(tmp_path, format): assert len(data[1]["generation"]) > 0 -def test_server_metadata_from_num_tasks(tmp_path): +def test_server_metadata_from_num_tasks(): """Test that metadata dict is properly created from server command returning (cmd, num_tasks).""" + mock_server_fn = MagicMock(return_value=("python server.py", 4)) cluster_config = { - "containers": { - "vllm": "apitest/vllm", - "nemo-skills": "apitest/nemo-skills", - "sandbox": "apitest/sandbox", - }, - "executor": "none", + "containers": {"vllm": "nvcr.io/nvidia/nemo:vllm", "nemo-skills": "nvcr.io/nvidia/nemo:skills"}, + "executor": "slurm", } server_config = { "server_type": "vllm", "num_gpus": 8, "num_nodes": 1, - "model_path": str(tmp_path / "model"), + "model_path": "/models/test", "server_port": 5000, - "server_args": "", } - generation_params = {"output_dir": "/tmp/out"} - groups = _create_job_unified( - models=[server_config["model_path"]], - server_configs=[server_config], - generation_params=generation_params, + cmd_group = _create_commandgroup_from_config( + generation_cmd="python generate.py", + server_config=server_config, + with_sandbox=False, + sandbox_port=None, cluster_config=cluster_config, installation_command=None, - with_sandbox=False, + get_server_command_fn=mock_server_fn, partition=None, keep_mounts_for_sandbox=False, task_name="test-task", log_dir="/tmp/logs", ) - server_cmd = groups[0].commands[0] - assert isinstance(server_cmd.script, ServerScript) - assert server_cmd.script.num_tasks >= 1 - assert server_cmd.script.num_gpus == server_config["num_gpus"] - assert groups[0].hardware.num_gpus == server_config["num_gpus"] - assert groups[0].hardware.num_tasks == server_cmd.script.num_tasks + server_cmd = cmd_group.commands[0] + assert isinstance(server_cmd.metadata, dict) + assert server_cmd.metadata["num_tasks"] == 4 + assert server_cmd.metadata["gpus"] == 8 diff --git a/tests/test_nemo_evaluator_pipeline.py b/tests/test_nemo_evaluator_pipeline.py index 0f333ab748..22ac250882 100644 --- a/tests/test_nemo_evaluator_pipeline.py +++ b/tests/test_nemo_evaluator_pipeline.py @@ -17,14 +17,8 @@ import pytest -from nemo_skills.pipeline.nemo_evaluator import ( - EvaluatorClientScript, -) -from nemo_skills.pipeline.nemo_evaluator import ( - nemo_evaluator as nemo_evaluator_fn, -) +from nemo_skills.pipeline.nemo_evaluator import nemo_evaluator as nemo_evaluator_fn from nemo_skills.pipeline.utils.declarative import Command, CommandGroup -from nemo_skills.pipeline.utils.scripts import ServerScript @pytest.fixture @@ -137,8 +131,9 @@ def test_no_servers_external_urls( # Verify client command client_cmd = group.commands[0] assert isinstance(client_cmd, Command) - assert client_cmd.name.startswith("evaluator-test-client-0") - assert isinstance(client_cmd.script, EvaluatorClientScript) + assert "evaluator-test-0" in client_cmd.name + assert client_cmd.gpus is None # No GPUs when no hosted servers + assert client_cmd.nodes == 1 # Verify hardware config assert group.hardware is not None @@ -186,17 +181,16 @@ def test_main_server_hosted( server_cmd = group.commands[0] assert isinstance(server_cmd, Command) assert "server" in server_cmd.name - assert isinstance(server_cmd.script, ServerScript) - assert server_cmd.script.num_gpus == 8 - assert server_cmd.script.log_prefix == "server" - assert server_cmd.script.port is not None + assert server_cmd.gpus == 8 + assert server_cmd.nodes == 1 + assert "port" in server_cmd.metadata + assert server_cmd.metadata["log_prefix"] == "server" # Verify client command client_cmd = group.commands[1] assert isinstance(client_cmd, Command) assert "client" in client_cmd.name - assert isinstance(client_cmd.script, EvaluatorClientScript) - assert callable(client_cmd.script.inline) # Should be lambda for cross-component refs + assert callable(client_cmd.command) # Should be lambda for cross-component refs # Verify hardware config (should use server GPUs) assert group.hardware.num_gpus == 8 @@ -241,16 +235,14 @@ def test_judge_server_hosted( judge_cmd = group.commands[0] assert isinstance(judge_cmd, Command) assert "judge-server" in judge_cmd.name - assert isinstance(judge_cmd.script, ServerScript) - assert judge_cmd.script.num_gpus == 32 - assert judge_cmd.script.log_prefix == "judge-server" + assert judge_cmd.gpus == 32 + assert judge_cmd.metadata["log_prefix"] == "judge-server" # Verify client command client_cmd = group.commands[1] assert isinstance(client_cmd, Command) assert "client" in client_cmd.name - assert isinstance(client_cmd.script, EvaluatorClientScript) - assert callable(client_cmd.script.inline) # Should be lambda for cross-component refs + assert callable(client_cmd.command) # Should be lambda for cross-component refs # Verify hardware config (should use judge server GPUs) assert group.hardware.num_gpus == 32 @@ -308,22 +300,19 @@ def test_both_servers_hosted_separate_groups( server_cmd = server_group.commands[0] assert isinstance(server_cmd, Command) assert "server" in server_cmd.name - assert isinstance(server_cmd.script, ServerScript) - assert server_cmd.script.num_gpus == 8 + assert server_cmd.gpus == 8 # Verify client command in first group client_cmd = server_group.commands[1] assert isinstance(client_cmd, Command) assert "client" in client_cmd.name - assert isinstance(client_cmd.script, EvaluatorClientScript) - assert callable(client_cmd.script.inline) # Lambda for cross-component refs + assert callable(client_cmd.command) # Lambda for cross-component refs # Verify judge server command in second group judge_cmd = judge_group.commands[0] assert isinstance(judge_cmd, Command) assert "judge-server" in judge_cmd.name - assert isinstance(judge_cmd.script, ServerScript) - assert judge_cmd.script.num_gpus == 32 + assert judge_cmd.gpus == 32 @patch("nemo_skills.pipeline.nemo_evaluator.Pipeline") From 1c433a71204aec187775a3c9ec44c6a0fb27e9bf Mon Sep 17 00:00:00 2001 From: George Armstrong Date: Wed, 17 Dec 2025 19:15:49 -0800 Subject: [PATCH 62/88] Fix: add serialized_output on bad request (#1127) Signed-off-by: George Armstrong Signed-off-by: Cheng-Ping Hsieh --- nemo_skills/inference/model/base.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/nemo_skills/inference/model/base.py b/nemo_skills/inference/model/base.py index 95dbd192b4..9318bfb475 100644 --- a/nemo_skills/inference/model/base.py +++ b/nemo_skills/inference/model/base.py @@ -308,7 +308,12 @@ async def generate_async( continue LOG.error(f"BadRequestError after {max_retries} retries, returning empty response: {e}") - return {"generation": "", "reasoning_content": "", "num_generated_tokens": 0} + return { + "generation": "", + "reasoning_content": "", + "num_generated_tokens": 0, + "serialized_output": [], + } else: raise e From 0e1e7902b0d1ab89c84912a8545052d8d0545e7e Mon Sep 17 00:00:00 2001 From: Wei Du Date: Wed, 17 Dec 2025 21:36:58 -0600 Subject: [PATCH 63/88] update paper link (#1128) Signed-off-by: Wei Du Signed-off-by: Cheng-Ping Hsieh --- docs/releases/index.md | 2 +- docs/releases/nemotron-math-v2/index.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/releases/index.md b/docs/releases/index.md index ae997f352f..4a721e5303 100644 --- a/docs/releases/index.md +++ b/docs/releases/index.md @@ -22,7 +22,7 @@ On this page you can find a list of papers, model and dataset releases that were ## Papers -* [Nemotron-Math: Efficient Long-Context Distillation of Mathematical Reasoning from Multi-Mode Supervision](./nemotron-math-v2/paper.pdf){:target="_blank"} (2025) +* [Nemotron-Math: Efficient Long-Context Distillation of Mathematical Reasoning from Multi-Mode Supervision](https://arxiv.org/abs/2512.15489){:target="_blank"} (2025) * [Scaling Generative Verifiers For Natural Language Mathematical Proof Verification And Selection](https://arxiv.org/abs/2511.13027){:target="_blank"} (2025) diff --git a/docs/releases/nemotron-math-v2/index.md b/docs/releases/nemotron-math-v2/index.md index 20515fff3e..9c80778e04 100644 --- a/docs/releases/nemotron-math-v2/index.md +++ b/docs/releases/nemotron-math-v2/index.md @@ -20,7 +20,7 @@ We used [Qwen2.5-32B-Instruct](https://huggingface.co/Qwen/Qwen2.5-32B-Instruct) -See our [paper](paper.pdf) to learn more details! +See our [paper](https://arxiv.org/abs/2512.15489) to learn more details! ## How to reproduce our results From c2c8a562e0423d40c4fc512ea91f6d650a13e351 Mon Sep 17 00:00:00 2001 From: Stephen Ge Date: Thu, 18 Dec 2025 12:25:25 -0500 Subject: [PATCH 64/88] update paper link, references to dataset, self-correction differences (#1129) Signed-off-by: Stephen Ge Signed-off-by: Cheng-Ping Hsieh --- docs/releases/nemotronmathproofs/index.md | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/docs/releases/nemotronmathproofs/index.md b/docs/releases/nemotronmathproofs/index.md index e9de7e26aa..8be466859b 100644 --- a/docs/releases/nemotronmathproofs/index.md +++ b/docs/releases/nemotronmathproofs/index.md @@ -14,7 +14,9 @@ culminating in Lean 4 proofs. The dataset integrates human-authored problems with systematically generated formalizations and solution traces: * **Natural Language Problems**: ~580k proof problems sourced from AoPS forums, Math StackExchange, and MathOverflow, - semantically deduplicated and decontaminated against popular benchmarks + filtered for proof-based problems that can be formulated as theorems, semantically deduplicated, and decontaminated + against popular benchmarks (removing ~40% of the original dataset). + See [our paper](https://arxiv.org/abs/2512.15489) for details on problem sources and the extraction pipeline. * **Formal Statements**: ~550k Lean 4 theorem statements generated via autoformalization * **Proof Trajectories**: ~900k verified reasoning traces and proofs @@ -30,10 +32,14 @@ We compare a Qwen3-8B model fine-tuned on this dataset against Goedel-Prover-V2- | Model | pass@32 (no self-correction) | pass@32 (with self-correction) | |-------|------------------------------|-------------------------------| -| Goedel-Prover-V2-8B | 84.6% | 86.7% | -| Qwen3-8B SFT on Nemotron-Math-Proofs-v1 | 85.3% | 90.2% | +| Goedel-Prover-V2-8B | 84.6% | 86.7%* | +| Qwen3-8B SFT on Nemotron-Math-Proofs-v1 | 85.3% | 90.2%** | -Nemotron-Nano-v3 (which includes this dataset in its training) achieves the following on miniF2F: +\* Goedel-Prover-V2 uses 3 self-correction attempts (limited by context length). +\*\* Our modified pipeline supports unbounded attempts; we use 8. + +Nemotron-Nano-v3 (which includes this dataset in its training) achieves the following on miniF2F +(all evaluations use 8 self-correction attempts): | Model | pass@32 (no self-correction) | pass@32 (with self-correction) | |-------|------------------------------|-------------------------------| @@ -53,7 +59,8 @@ Browse the sections below to see commands for autoformalization, theorem proving ### Autoformalization The autoformalization pipeline translates natural language theorems into Lean 4 formal statements using an iterative -refinement process with backtranslation verification. The input is natural language math problems—see +refinement process with backtranslation verification. The input is natural language math problems (the `problem` field +from the [dataset](https://huggingface.co/datasets/nvidia/Nemotron-Math-Proofs-v1))—see [OpenMathReasoning dataset construction](../openmathreasoning/dataset.md) for how to prepare these. === "CLI" @@ -128,7 +135,8 @@ The pipeline includes: ### Theorem Proving The prover pipeline generates proofs for formalized statements with iterative error correction. -Input: formal statements from the autoformalization step. +Input: formal statements from the autoformalization step (the `formal_statement` and `lean_header` fields +from the [dataset](https://huggingface.co/datasets/nvidia/Nemotron-Math-Proofs-v1)). === "CLI" @@ -199,7 +207,8 @@ The proving strategy includes: ### Model Training To fine-tune a model on the Nemotron-Math-Proofs dataset. -Input: processed SFT data from the theorem proving step. +Input: processed SFT data from the theorem proving step (the `messages` field from the +[dataset](https://huggingface.co/datasets/nvidia/Nemotron-Math-Proofs-v1) contains verified proof conversations). === "CLI" From cd62bf73fa3f4fbe1e6e3016b85011c4e5c12702 Mon Sep 17 00:00:00 2001 From: George Armstrong Date: Thu, 18 Dec 2025 11:01:18 -0800 Subject: [PATCH 65/88] FIX ioi ignore (#1131) Signed-off-by: George Armstrong Signed-off-by: Cheng-Ping Hsieh --- tests/gpu-tests/test_eval.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/gpu-tests/test_eval.py b/tests/gpu-tests/test_eval.py index 31c8f2cccf..47060a1368 100644 --- a/tests/gpu-tests/test_eval.py +++ b/tests/gpu-tests/test_eval.py @@ -33,8 +33,7 @@ "livebench_coding", "livecodebench-pro", "livecodebench-cpp", - "ioi24", - "ioi25", + "ioi", "bfcl_v3", "bfcl_v4", "swe-bench", From 71d15b6ab36afeb11f80927a0cd1fc9579947b8c Mon Sep 17 00:00:00 2001 From: anowaczynski-nvidia Date: Thu, 18 Dec 2025 21:19:10 +0100 Subject: [PATCH 66/88] download AA-LCR_extracted-text.zip via hf_hub_download (#1126) Signed-off-by: Arkadiusz Nowaczynski Co-authored-by: George Armstrong Signed-off-by: Cheng-Ping Hsieh --- nemo_skills/dataset/aalcr/prepare.py | 30 +++++++++++++++++----------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/nemo_skills/dataset/aalcr/prepare.py b/nemo_skills/dataset/aalcr/prepare.py index 81f2b724a1..e9b021000d 100644 --- a/nemo_skills/dataset/aalcr/prepare.py +++ b/nemo_skills/dataset/aalcr/prepare.py @@ -15,10 +15,13 @@ import json import logging import os +import tempfile +import zipfile from pathlib import Path import tiktoken from datasets import load_dataset +from huggingface_hub import hf_hub_download from tqdm import tqdm from nemo_skills.utils import get_logger_name, setup_logging @@ -203,24 +206,27 @@ def write_data_to_file(output_file, data, txt_file_folder, max_context_window, t def prepare_aalcr_data(max_context_window, setup, tokenizer_name): # download the provied extracted text files # https://huggingface.co/datasets/ArtificialAnalysis/AA-LCR/resolve/main/extracted_text/AA-LCR_extracted-text.zip + extracted_text_zip_path = hf_hub_download( + repo_id="ArtificialAnalysis/AA-LCR", + filename="extracted_text/AA-LCR_extracted-text.zip", + repo_type="dataset", + ) - if not os.path.exists(Path(__file__).absolute().parent / "lcr"): - import zipfile - - import wget + extracted_text_zip_path = Path(extracted_text_zip_path) + assert extracted_text_zip_path.exists() and extracted_text_zip_path.is_file() - wget.download(URL) - zipfile.ZipFile("AA-LCR_extracted-text.zip").extractall(Path(__file__).absolute().parent) - os.remove("AA-LCR_extracted-text.zip") + with tempfile.TemporaryDirectory() as tmpdir: + zipfile.ZipFile(extracted_text_zip_path).extractall(tmpdir) - txt_file_folder = Path(__file__).absolute().parent / "lcr" + txt_file_folder = Path(tmpdir) / "lcr" + assert txt_file_folder.exists() and txt_file_folder.is_dir() - dataset = load_dataset("ArtificialAnalysis/AA-LCR")["test"] + dataset = load_dataset("ArtificialAnalysis/AA-LCR")["test"] - data_dir = Path(__file__).absolute().parent + data_dir = Path(__file__).absolute().parent + output_file = data_dir / f"{setup}.jsonl" - output_file = data_dir / f"{setup}.jsonl" - write_data_to_file(output_file, dataset, txt_file_folder, max_context_window, tokenizer_name) + write_data_to_file(output_file, dataset, txt_file_folder, max_context_window, tokenizer_name) if __name__ == "__main__": From 8ddcdf49ccd8be9d6e8bc6e732340d27938e104d Mon Sep 17 00:00:00 2001 From: Wasi Ahmad Date: Thu, 18 Dec 2025 19:03:33 -0800 Subject: [PATCH 67/88] Evaluation on Livecodebench-pro (#1115) Signed-off-by: wasiahmad Signed-off-by: Cheng-Ping Hsieh --- docs/evaluation/code.md | 32 +++++++++ .../dataset/livecodebench-pro/__init__.py | 5 +- .../dataset/livecodebench-pro/prepare.py | 72 ++++++++++++++++--- nemo_skills/evaluation/evaluator/code.py | 71 ++++++++++++++---- nemo_skills/evaluation/metrics/map_metrics.py | 1 + 5 files changed, 155 insertions(+), 26 deletions(-) diff --git a/docs/evaluation/code.md b/docs/evaluation/code.md index 1b4a60a14e..68decc1645 100644 --- a/docs/evaluation/code.md +++ b/docs/evaluation/code.md @@ -328,6 +328,38 @@ Due to variance between runs, you can automatically repeat the evaluation and av - Benchmark is defined in [`nemo_skills/dataset/livecodebench-pro/__init__.py`](https://github.com/NVIDIA-NeMo/Skills/blob/main/nemo_skills/dataset/livecodebench-pro/__init__.py) - Original benchmark source is [here](https://github.com/GavinZhengOI/LiveCodeBench-Pro). +#### Data Preparation + +First, prepare the dataset by running the `ns prepare_data` command. The arguments below will generate `test_24q4.jsonl`, `test_25q1.jsonl`, `test_25q2.jsonl`, and `test_25q3.jsonl` files. + +``` +ns prepare_data livecodebench-pro --cluster=local --data_dir=/workspace/ns-data +``` + +Note that, this will also download testcases and keep it at `/workspace/ns-data/livecodebench-pro/testcases`. We recommend using a cluster data location since the testcases directory would be of size 15GB. + +#### Running the Evaluation + +``` +ns eval \ + --cluster= \ + --model=nvidia/OpenReasoning-Nemotron-32B \ + --server_type=vllm \ + --server_args="--async-scheduling" \ + --server_nodes=1 \ + --server_gpus=8 \ + --benchmarks=livecodebench-pro \ + --split=test_25q2 \ + --data_dir=/workspace/ns-data/livecodebench-pro \ + --output_dir= \ + ++parse_reasoning=True \ + ++eval_config.test_file=/workspace/ns-data/livecodebench-pro/test_25q2.jsonl \ + ++eval_config.test_dir=/workspace/ns-data/livecodebench-pro/testcases \ + ++inference.temperature=0.6 \ + ++inference.top_p=0.95 \ + ++inference.tokens_to_generate=65536 +``` + ### human-eval - Benchmark is defined in [`nemo_skills/dataset/human-eval/__init__.py`](https://github.com/NVIDIA-NeMo/Skills/blob/main/nemo_skills/dataset/human-eval/__init__.py) diff --git a/nemo_skills/dataset/livecodebench-pro/__init__.py b/nemo_skills/dataset/livecodebench-pro/__init__.py index a071794767..7aa6e5f0ed 100644 --- a/nemo_skills/dataset/livecodebench-pro/__init__.py +++ b/nemo_skills/dataset/livecodebench-pro/__init__.py @@ -14,5 +14,6 @@ # settings that define how evaluation should be done by default (all can be changed from cmdline) DATASET_GROUP = "code" -METRICS_TYPE = "code" -GENERATION_ARGS = "++prompt_config=eval/livecodebench/python_codegen ++eval_type=livecodebench_pro" +METRICS_TYPE = "livecodebench_pro" +EVAL_SPLIT = "test_25q2" +GENERATION_ARGS = "++prompt_config=eval/livecodebench/cpp_codegen ++eval_type=livecodebench_pro" diff --git a/nemo_skills/dataset/livecodebench-pro/prepare.py b/nemo_skills/dataset/livecodebench-pro/prepare.py index 0f648d0a2e..52703532de 100644 --- a/nemo_skills/dataset/livecodebench-pro/prepare.py +++ b/nemo_skills/dataset/livecodebench-pro/prepare.py @@ -13,19 +13,71 @@ # limitations under the License. import json +import os from pathlib import Path from datasets import load_dataset +from huggingface_hub import snapshot_download + +TESTCASE_REPO = "QAQAQAQAQ/LiveCodeBench-Pro-Testcase" +PROBLEM_REPO = "QAQAQAQAQ/LiveCodeBench-Pro" +DEFAULT_SPLITS = [ + ("24q4", "quater_2024_10_12", 207), + ("25q1", "quater_2025_1_3", 166), + ("25q2", "quater_2025_4_6", 167), + ("25q3", "quater_2025_7_9", 144), +] + + +def download_testcases(local_dir, token): + """ + Downloads the large testcase dataset (~15GB) to the specified directory. + """ + print(f"Downloading testcases from {TESTCASE_REPO} to {local_dir}...") + try: + path = snapshot_download(repo_id=TESTCASE_REPO, repo_type="dataset", local_dir=local_dir, token=token) + print(f"Testcases successfully downloaded to: {path}") + except Exception as e: + print(f"Failed to download testcases: {e}") + raise + + +def process_problem_splits(output_dir, token): + """ + Downloads problem descriptions, converts them to JSONL, and saves them. + """ + print(f"Processing problem splits from {PROBLEM_REPO}...") + + for tag, split, sample_size in DEFAULT_SPLITS: + print(f" - Processing split: {split} -> test_{tag}.jsonl") + + try: + dataset = load_dataset(PROBLEM_REPO, split=split, token=token) + if len(dataset) != sample_size: + print(f" WARNING: Expected {sample_size} samples for {split}, but got {len(dataset)}.") + + output_file = output_dir / f"test_{tag}.jsonl" + + with open(output_file, "w", encoding="utf-8") as f: + for row in dataset: + output_record = dict(row) + output_record["question"] = row["problem_statement"] + output_record["subset_for_metrics"] = row["difficulty"] + + f.write(json.dumps(output_record) + "\n") + + except Exception as e: + print(f" Error processing split {split}: {e}") + if __name__ == "__main__": + hf_token = os.environ.get("HF_TOKEN") + if not hf_token: + print("Error: HF_TOKEN environment variable is required.") + print("Please export it: export HF_TOKEN='hf_...'") + exit(1) + data_dir = Path(__file__).absolute().parent - output_file = str(data_dir / "test.jsonl") - - dataset = load_dataset("anonymous1926/anonymous_dataset") - with open(output_file, "w") as f: - for split_name, split in dataset.items(): - for row in split: - row["task_id"] = row.pop("problem_id") - row["question"] = row.pop("problem_statement") - row["split"] = split_name - f.write(json.dumps(row) + "\n") + testcase_dir = data_dir / "testcases" + download_testcases(local_dir=testcase_dir, token=hf_token) + process_problem_splits(output_dir=data_dir, token=hf_token) diff --git a/nemo_skills/evaluation/evaluator/code.py b/nemo_skills/evaluation/evaluator/code.py index 3800495d28..f24bae422d 100644 --- a/nemo_skills/evaluation/evaluator/code.py +++ b/nemo_skills/evaluation/evaluator/code.py @@ -115,14 +115,14 @@ async def eval_full(self): # type: ignore[override] LOG.info("Full evaluation completed successfully") -def preprocess_code(generation_dict: dict, language="python", strip_whitespace=True): - completion = generation_dict.get("generation", "") or "" +def preprocess_code(generation_dict: dict, language: str = "python", strip_whitespace: bool = True): + completion = generation_dict.get("generation", "") completion = completion.replace("\r", "") # --------------------------------------------------------- # 1. Handle reasoning traces: ... # --------------------------------------------------------- - if "" in completion: + if "" in completion: # partition is faster than regex and avoids imports _, separator, post_thought = completion.partition("") if separator: @@ -194,7 +194,7 @@ def eval_evalplus(cfg): jsonl_file = cfg.input_file with open(jsonl_file) as f: - samples = [preprocess_code(json.loads(line)) for line in f] + samples = [preprocess_code(json.loads(line), language="python") for line in f] # all changes will be done with a new key "completion", so it's ok to write to the same file with open(jsonl_file, "wt", encoding="utf-8") as f: for sample in samples: @@ -236,20 +236,63 @@ def install_requirements(url): print(f"Error during installation: {e}") +@nested_dataclass(kw_only=True) +class LiveCodeBenchProEvaluatorConfig(BaseEvaluatorConfig): + sandbox: dict = field(default_factory=lambda: {"sandbox_type": "local"}) + language: str = "cpp" # use either "python" or "cpp" + test_file: str = None + test_dir: str = None # path to the unit tests directory + timeout: int = 6 + num_processes: int = 12 + + def eval_livecodebench_pro(cfg): - cfg = BaseEvaluatorConfig(**cfg) + cfg = LiveCodeBenchProEvaluatorConfig(**cfg) + try: + from livecodebench.evaluate import evaluate + except ImportError: + LOG.info("Package 'livecodebench' not found. Attempting to install...") + install_from_git("git+https://github.com/wasiahmad/livecodebench.git@livecodebench_pro") + try: + from livecodebench.evaluate import evaluate + except ImportError: + LOG.info("Failed to install 'livecodebench'. Please install it manually.") + raise + jsonl_file = cfg.input_file + samples = [] with open(jsonl_file) as f: - samples = [preprocess_code(json.loads(line), "python") for line in f] - for sample in samples: - sample["problem_id"] = sample.pop("task_id") - sample["text_response"] = sample.pop("completion") - sample["response_meta"] = None + for line in f: + sample = json.loads(line) + sample = preprocess_code(sample, language=cfg.language, strip_whitespace=True) + sample["code_list"] = [sample["completion"]] + samples.append(sample) with open(jsonl_file, "wt", encoding="utf-8") as f: for sample in samples: f.write(json.dumps(sample) + "\n") + evaluate( + custom_output_file=jsonl_file, + language=cfg.language, + test_file=cfg.test_file, + test_dir=cfg.test_dir, + k_list=[1], + num_process_evaluate=cfg.num_processes, + timeout=cfg.timeout, + ) + + with open(jsonl_file[:-6] + "_eval_results.json", "rt", encoding="utf-8") as fin: + eval_grades = json.load(fin) + with open(jsonl_file, "wt", encoding="utf-8") as f: + for sample in samples: + if sample["problem_id"] in eval_grades["eval"]: + sample["graded_list"] = eval_grades["eval"][sample["problem_id"]]["graded_list"] + f.write(json.dumps(sample) + "\n") + + # moving eval file to ensure metrics are recomputed + shutil.move(jsonl_file[:-6] + "_eval_results.json", jsonl_file[:-6] + "_eval_results-saved.json") + def eval_livebench_coding(cfg): cfg = BaseEvaluatorConfig(**cfg) @@ -271,12 +314,12 @@ def eval_livebench_coding(cfg): sample = json.loads(line) if sample["task"] == "coding_completion": assert len(sample["partial_solution"]) > 0 - sample = preprocess_code(sample, strip_whitespace=False) + sample = preprocess_code(sample, language="python", strip_whitespace=False) sample["completion"] = sample["completion"].replace("\t", " ") full_solution = sample["partial_solution"] + "\n" + sample["completion"] sample["code_list"] = [full_solution] else: - sample = preprocess_code(sample, strip_whitespace=True) + sample = preprocess_code(sample, language="python", strip_whitespace=True) sample["code_list"] = [sample["completion"]] samples.append(sample) @@ -332,7 +375,7 @@ def eval_bigcodebench(cfg): samples = [] with open(jsonl_file) as f: for line in f: - generation_dict = preprocess_code(json.loads(line)) + generation_dict = preprocess_code(json.loads(line), language="python") generation_dict["solution"] = generation_dict.pop("completion") samples.append(generation_dict) with open(jsonl_file, "wt", encoding="utf-8") as f: @@ -417,7 +460,7 @@ def postprocess_code(sample): elif data_split != sample["split"]: raise ValueError(f"All samples should have the same split, but got {data_split} and {sample['split']}") - sample = preprocess_code(sample, strip_whitespace=False) + sample = preprocess_code(sample, language="python", strip_whitespace=False) sample["original_completion"] = sample["completion"] sample = postprocess_code(sample) samples.append(sample) diff --git a/nemo_skills/evaluation/metrics/map_metrics.py b/nemo_skills/evaluation/metrics/map_metrics.py index 94cf9b8c73..1c66a95bd7 100644 --- a/nemo_skills/evaluation/metrics/map_metrics.py +++ b/nemo_skills/evaluation/metrics/map_metrics.py @@ -58,6 +58,7 @@ "ruler": RulerMetrics, "ruler2": RulerMetrics, "livecodebench": LiveCodeBenchMetrics, + "livecodebench_pro": LiveCodeBenchMetrics, "swe-bench": SweBenchMetrics, "scicode": SciCodeMetrics, "bigcodebench": BigCodeBenchMetrics, From 3a50f7fb02ac0e3ddcd2488d08fe0923ba917bcf Mon Sep 17 00:00:00 2001 From: Wasi Ahmad Date: Wed, 24 Dec 2025 09:46:37 -0800 Subject: [PATCH 68/88] Evaluation support for SWE-rebench (#1102) Signed-off-by: wasiahmad Signed-off-by: Nikolai Ludwig Signed-off-by: George Armstrong Signed-off-by: i-vainn Signed-off-by: Grigor Nalbandyan Signed-off-by: bzantium Signed-off-by: Stephen Ge Signed-off-by: Jiacheng Xu Signed-off-by: George Zelenfroind Co-authored-by: Nick Ludwig Co-authored-by: George Armstrong Co-authored-by: Ivan Co-authored-by: Wojciech Prazuch Co-authored-by: gnalbandyan <153070076+gnalbandyan@users.noreply.github.com> Co-authored-by: Minho Ryu Co-authored-by: Stephen Ge Co-authored-by: Jiacheng Xu Co-authored-by: Jiacheng Xu Co-authored-by: George <37293288+Jorjeous@users.noreply.github.com> Co-authored-by: Sanyam Kapoor Signed-off-by: Cheng-Ping Hsieh --- nemo_skills/dataset/swe-rebench/__init__.py | 21 ++++ nemo_skills/dataset/swe-rebench/prepare.py | 100 ++++++++++++++++++++ 2 files changed, 121 insertions(+) create mode 100644 nemo_skills/dataset/swe-rebench/__init__.py create mode 100644 nemo_skills/dataset/swe-rebench/prepare.py diff --git a/nemo_skills/dataset/swe-rebench/__init__.py b/nemo_skills/dataset/swe-rebench/__init__.py new file mode 100644 index 0000000000..ceff4e2e07 --- /dev/null +++ b/nemo_skills/dataset/swe-rebench/__init__.py @@ -0,0 +1,21 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# settings that define how evaluation should be done by default (all can be changed from cmdline) +EVAL_SPLIT = "default" +DATASET_GROUP = "code" +METRICS_TYPE = "swe-bench" +# evaluation is fused with generation for efficiency +GENERATION_MODULE = "nemo_skills.inference.eval.swebench" +GENERATION_ARGS = "++eval_harness_repo=https://github.com/wasiahmad/SWE-rebench.git " diff --git a/nemo_skills/dataset/swe-rebench/prepare.py b/nemo_skills/dataset/swe-rebench/prepare.py new file mode 100644 index 0000000000..45bdd1614c --- /dev/null +++ b/nemo_skills/dataset/swe-rebench/prepare.py @@ -0,0 +1,100 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import json +from pathlib import Path + +import datasets + + +def get_date_range(start_str, end_str): + """Generates a list of YYYY_MM strings between start and end inclusive.""" + start_year, start_month = map(int, start_str.split("_")) + end_year, end_month = map(int, end_str.split("_")) + + dates = [] + current_year, current_month = start_year, start_month + + while (current_year < end_year) or (current_year == end_year and current_month <= end_month): + dates.append(f"{current_year}_{current_month:02d}") + + current_month += 1 + if current_month > 12: + current_month = 1 + current_year += 1 + return dates + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--container_formatter", + type=str, + default="docker://{docker_image}", + help="Container formatter string. You can download .sif containers and store them in a mounted " + "directory which you can reference here to avoid redownloading all the time.", + ) + parser.add_argument("--start_date", type=str, help="Start date in YYYY_MM format") + parser.add_argument("--end_date", type=str, help="End date in YYYY_MM format") + parser.add_argument( + "--setup", type=str, default="default", help="Setup name (used as nemo-skills split parameter)." + ) + parser.add_argument( + "--dataset_name", + type=str, + default="nebius/SWE-rebench-leaderboard", + help="Dataset name to load", + ) + args = parser.parse_args() + + dataset_name = args.dataset_name + output_file = Path(__file__).parent / f"{args.setup}.jsonl" + + if args.start_date and args.end_date: + splits_to_load = get_date_range(args.start_date, args.end_date) + else: + print("Start/End date not provided. Defaulting to 'test' split.") + splits_to_load = ["test"] + + all_data = [] + global_id_counter = 0 + + for split in splits_to_load: + print(f"Loading split: {split}...") + try: + ds = datasets.load_dataset(path=dataset_name, split=split) + for item in ds: + docker_image = item["docker_image"] + if args.container_formatter.endswith(".sif"): + docker_image = item["docker_image"].replace("/", "_").replace(":", "_") + container_formatter = args.container_formatter.format(docker_image=docker_image) + processed_item = { + **item, + "container_formatter": container_formatter, + "container_id": global_id_counter, + "dataset_name": dataset_name, + "split": split, + } + all_data.append(processed_item) + global_id_counter += 1 + + except Exception as e: + print(f"Warning: Could not load split {split}. Error: {e}") + + with open(output_file, "w", encoding="utf-8") as f: + for entry in all_data: + f.write(json.dumps(entry) + "\n") + + print(f"Successfully saved {len(all_data)} samples to {output_file}") From 26ab83423aafdba777acf8a16fa82b5909872450 Mon Sep 17 00:00:00 2001 From: Igor Gitman Date: Fri, 26 Dec 2025 21:19:13 -0800 Subject: [PATCH 69/88] Trust remote code in tokenizer (#1146) Signed-off-by: fzyzcjy <5236035+fzyzcjy@users.noreply.github.com> Signed-off-by: Igor Gitman Co-authored-by: fzyzcjy <5236035+fzyzcjy@users.noreply.github.com> Signed-off-by: Cheng-Ping Hsieh --- nemo_skills/inference/eval/bfcl.py | 2 +- nemo_skills/inference/generate.py | 2 +- nemo_skills/inference/model/parallel_thinking.py | 2 +- nemo_skills/inference/model/utils.py | 2 +- nemo_skills/inference/prover.py | 2 +- nemo_skills/prompt/utils.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/nemo_skills/inference/eval/bfcl.py b/nemo_skills/inference/eval/bfcl.py index 7a83825236..f476581532 100644 --- a/nemo_skills/inference/eval/bfcl.py +++ b/nemo_skills/inference/eval/bfcl.py @@ -138,7 +138,7 @@ def _validate_and_setup_client_parsing(self): # Initialize the prompt formatter # While BFCL model_handler also has the _format_prompt method, we found errors in it's implementation # So we use the tokenizer to format the prompt instead which uses the chat template directly - tokenizer = AutoTokenizer.from_pretrained(model_handler.model_name_huggingface) + tokenizer = AutoTokenizer.from_pretrained(model_handler.model_name_huggingface, trust_remote_code=True) self.message_formatter = partial(tokenizer.apply_chat_template, tokenize=False, add_generation_prompt=True) def create_response_parser(self, native_response_parser): diff --git a/nemo_skills/inference/generate.py b/nemo_skills/inference/generate.py index 136375db46..29e4e0084c 100644 --- a/nemo_skills/inference/generate.py +++ b/nemo_skills/inference/generate.py @@ -338,7 +338,7 @@ def __init__(self, cfg: GenerateSolutionsConfig): # Setup hf_tokenizer for counting prompt tokens self.hf_tokenizer = None if self.cfg.count_prompt_tokens: - self.hf_tokenizer = AutoTokenizer.from_pretrained(self.tokenizer) + self.hf_tokenizer = AutoTokenizer.from_pretrained(self.tokenizer, trust_remote_code=True) if self.hf_tokenizer is None: raise ValueError("Tokenizer could not be initialized. Needed for counting prompt tokens.") diff --git a/nemo_skills/inference/model/parallel_thinking.py b/nemo_skills/inference/model/parallel_thinking.py index 8993916a52..95400b15c2 100644 --- a/nemo_skills/inference/model/parallel_thinking.py +++ b/nemo_skills/inference/model/parallel_thinking.py @@ -105,7 +105,7 @@ def __init__(self, model: BaseModel, tokenizer: str | None, orig_prompt_filler, raise ValueError(f"Invalid parallel thinking mode: {self.cfg.mode}") if self.cfg.count_prompt_tokens: - self.hf_tokenizer = AutoTokenizer.from_pretrained(self.tokenizer) + self.hf_tokenizer = AutoTokenizer.from_pretrained(self.tokenizer, trust_remote_code=True) if self.hf_tokenizer is None: raise ValueError("Tokenizer could not be initialized. Needed for counting prompt tokens.") diff --git a/nemo_skills/inference/model/utils.py b/nemo_skills/inference/model/utils.py index d589175316..27bf917e74 100644 --- a/nemo_skills/inference/model/utils.py +++ b/nemo_skills/inference/model/utils.py @@ -88,7 +88,7 @@ class WrapperAutoTokenizer: def __init__(self, model_name: str): LOG.info(f"Initializing tokenizer from string: {model_name}") - self.tokenizer = AutoTokenizer.from_pretrained(model_name) + self.tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True) def encode(self, prompt: str | list[dict], tools=None) -> list[int]: """Encode the prompt using the tokenizer.""" diff --git a/nemo_skills/inference/prover.py b/nemo_skills/inference/prover.py index 711b5fe9aa..19bebe1c6e 100644 --- a/nemo_skills/inference/prover.py +++ b/nemo_skills/inference/prover.py @@ -107,7 +107,7 @@ def __init__(self, cfg: ProverConfig): # Initialize tokenizer for chat template application tokenizer_path = self.cfg.tokenizer or self.cfg.server.get("model") - self.hf_tokenizer = AutoTokenizer.from_pretrained(tokenizer_path) + self.hf_tokenizer = AutoTokenizer.from_pretrained(tokenizer_path, trust_remote_code=True) if self.cfg.refinement: self.setup_refine_prompt() diff --git a/nemo_skills/prompt/utils.py b/nemo_skills/prompt/utils.py index 932d170f3b..6975570bfa 100644 --- a/nemo_skills/prompt/utils.py +++ b/nemo_skills/prompt/utils.py @@ -112,7 +112,7 @@ def __init__(self, config, tokenizer): if self.tokenizer: # assuming it's the object already if not str if isinstance(self.tokenizer, str): - self.tokenizer = AutoTokenizer.from_pretrained(self.tokenizer) + self.tokenizer = AutoTokenizer.from_pretrained(self.tokenizer, trust_remote_code=True) def build_filled_example(self, example_dict: Dict[str, Any]) -> str: """Builds a filled example string based on the example dictionary.""" From 956e8e8a425c86c92ed02b4d8201c49c705216ad Mon Sep 17 00:00:00 2001 From: Sanyam Kapoor <3909933+activatedgeek@users.noreply.github.com> Date: Mon, 5 Jan 2026 23:45:53 +0530 Subject: [PATCH 70/88] Resolve broken links in docs (#1150) Signed-off-by: Sanyam Kapoor Signed-off-by: Cheng-Ping Hsieh --- docs/evaluation/index.md | 4 ++-- docs/index.md | 3 +-- docs/pipelines/generation.md | 4 +--- docs/pipelines/start-server.md | 2 +- mkdocs.yml | 1 + 5 files changed, 6 insertions(+), 8 deletions(-) diff --git a/docs/evaluation/index.md b/docs/evaluation/index.md index b63943e869..bb35c4d52b 100644 --- a/docs/evaluation/index.md +++ b/docs/evaluation/index.md @@ -9,7 +9,7 @@ We support many popular benchmarks and it's easy to add new in the future. The f - [**Instruction following**](./instruction-following.md): e.g. [ifbench](./instruction-following.md#ifbench), [ifeval](./instruction-following.md#ifeval) - [**Long-context**](./long-context.md): e.g. [ruler](./long-context.md#ruler), [mrcr](./long-context.md#mrcr) - [**Tool-calling**](./tool-calling.md): e.g. [bfcl_v3](./tool-calling.md#bfcl_v3) -- [**Multilingual**](./multilingual.md): e.g. [mmlu-prox](./multilingual.md#mmlu-prox), [flores-200](./multilingual.md#FLORES-200), [wmt24pp](./multilingual.md#wmt24pp) +- [**Multilingual**](./multilingual.md): e.g. [mmlu-prox](./multilingual.md#mmlu-prox), [flores-200](./multilingual.md#flores-200), [wmt24pp](./multilingual.md#wmt24pp) - [**Speech & Audio**](./speech-audio.md): e.g. [asr-leaderboard](./speech-audio.md#asr-leaderboard), [mmau-pro](./speech-audio.md#mmau-pro) See [nemo_skills/dataset](https://github.com/NVIDIA-NeMo/Skills/blob/main/nemo_skills/dataset) where each folder is a benchmark we support. @@ -177,7 +177,7 @@ code execution timeout for scicode benchmark !!! tip "Passing Main Arguments with Config Files" For parameters that are difficult to escape on the command line (like `end_reasoning_string=''`), - you can use YAML config files instead. See [Passing Main Arguments with Config Files](../pipelines/index.md###passing-main-arguments-with-config-files) for details. + you can use YAML config files instead. See [Passing Main Arguments with Config Files](../pipelines/index.md#passing-main-arguments-with-config-files) for details. ## Using data on cluster diff --git a/docs/index.md b/docs/index.md index ce53e256c3..a17dce8f81 100644 --- a/docs/index.md +++ b/docs/index.md @@ -21,7 +21,7 @@ Here are some of the features we support: - [**Instruction following**](./evaluation/instruction-following.md): e.g. [ifbench](./evaluation/instruction-following.md#ifbench), [ifeval](./evaluation/instruction-following.md#ifeval) - [**Long-context**](./evaluation/long-context.md): e.g. [ruler](./evaluation/long-context.md#ruler), [mrcr](./evaluation/long-context.md#mrcr) - [**Tool-calling**](./evaluation/tool-calling.md): e.g. [bfcl_v3](./evaluation/tool-calling.md#bfcl_v3) - - [**Multilingual capabilities**](./evaluation/multilingual.md): e.g. [mmlu-prox](./evaluation/multilingual.md#mmlu-prox), [flores-200](./evaluation/multilingual.md#FLORES-200), [wmt24pp](./evaluation/multilingual.md#wmt24pp) + - [**Multilingual capabilities**](./evaluation/multilingual.md): e.g. [mmlu-prox](./evaluation/multilingual.md#mmlu-prox), [flores-200](./evaluation/multilingual.md#flores-200), [wmt24pp](./evaluation/multilingual.md#wmt24pp) - [**Speech & Audio**](./evaluation/speech-audio.md): e.g. [asr-leaderboard](./evaluation/speech-audio.md#asr-leaderboard), [mmau-pro](./evaluation/speech-audio.md#mmau-pro) - [**Robustness evaluation**](./evaluation/robustness.md): Evaluate model sensitvity against changes in prompt. - Easily parallelize each evaluation across many Slurm jobs, self-host LLM judges, bring your own prompts or change benchmark configuration in any other way. @@ -36,4 +36,3 @@ You can find more examples of how to use Nemo-Skills in the [tutorials](./tutori We've built and released many popular models and datasets using Nemo-Skills. See all of them in the [Papers & Releases](./releases/index.md) documentation. We support many popular benchmarks and it's easy to add new in the future. The following categories of benchmarks are supported - diff --git a/docs/pipelines/generation.md b/docs/pipelines/generation.md index dc299ee9dd..058c176e6c 100644 --- a/docs/pipelines/generation.md +++ b/docs/pipelines/generation.md @@ -98,7 +98,7 @@ See [nemo_skills/inference/generate.py](https://github.com/NVIDIA-NeMo/Skills/bl !!! tip "Passing Main Arguments with Config Files" For parameters that are difficult to escape on the command line (like `end_reasoning_string=''`), - you can use YAML config files instead. See [Passing Main Arguments with Config Files](index.md###passing-main-arguments-with-config-files) for details. + you can use YAML config files instead. See [Passing Main Arguments with Config Files](index.md#passing-main-arguments-with-config-files) for details. ## Sampling multiple generations @@ -470,5 +470,3 @@ We support three methods for automatic trimming of generation budget or context: ++server.enable_soft_fail=True ++server.context_limit_retry_strategy=reduce_prompt_from_end ``` - - diff --git a/docs/pipelines/start-server.md b/docs/pipelines/start-server.md index 36d9534022..fd3474a24e 100644 --- a/docs/pipelines/start-server.md +++ b/docs/pipelines/start-server.md @@ -64,7 +64,7 @@ Similarly, the local port for the sandbox server can be changed using `--sandbox ## Using the Server -To use this started server in [Evaluation](/Skills/pipelines/evaluation/) or [Generation](/Skills/pipelines/generation/), +To use this started server in [Evaluation](evaluation.md) or [Generation](generation.md), all the model-related arguments can now be replaced with `--server_type=openai` and `server_address` arguments. For instance, for the vLLM model server above, the `eval` pipeline arguments can be modified as, diff --git a/mkdocs.yml b/mkdocs.yml index 6ef479a929..2d6118732b 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,3 +1,4 @@ +strict: true site_name: Nemo-Skills site_url: https://nvidia-nemo.github.io/Skills extra_css: From 7343902b14d6e3a3ddaf68e8e91c2b21de06c08a Mon Sep 17 00:00:00 2001 From: Valentin Mendelev Date: Tue, 6 Jan 2026 01:37:33 +0100 Subject: [PATCH 71/88] Introduced vLLM_multimodal model to save multimodal outputs (#1136) Signed-off-by: Valentin Mendelev Co-authored-by: Nikolay Karpov Signed-off-by: Cheng-Ping Hsieh --- nemo_skills/inference/generate.py | 20 +++- nemo_skills/inference/model/__init__.py | 2 + nemo_skills/inference/model/base.py | 5 + .../inference/model/vllm_multimodal.py | 110 ++++++++++++++++++ 4 files changed, 135 insertions(+), 2 deletions(-) create mode 100644 nemo_skills/inference/model/vllm_multimodal.py diff --git a/nemo_skills/inference/generate.py b/nemo_skills/inference/generate.py index 29e4e0084c..52a176bc25 100644 --- a/nemo_skills/inference/generate.py +++ b/nemo_skills/inference/generate.py @@ -399,8 +399,20 @@ def setup_prompt(self): def setup_llm(self): self.sandbox = get_sandbox(**self.cfg.sandbox) if self.cfg.sandbox is not None else None + self.data_dir = None + if "data_dir" in self.cfg.eval_config and not isinstance(self.cfg.eval_config.get("data_dir"), type(None)): + self.data_dir = self.cfg.eval_config["data_dir"] + + output_dir = str(Path(self.cfg.output_file).parent) + if self.cfg.code_execution: - llm = get_code_execution_model(**self.cfg.server, tokenizer=self.tokenizer, sandbox=self.sandbox) + llm = get_code_execution_model( + **self.cfg.server, + tokenizer=self.tokenizer, + sandbox=self.sandbox, + data_dir=self.data_dir or "", + output_dir=output_dir, + ) elif self.cfg.tool_modules is not None: llm = get_tool_calling_model( **self.cfg.server, @@ -409,9 +421,13 @@ def setup_llm(self): schema_overrides=self.cfg.schema_overrides, tokenizer=self.tokenizer, additional_config={"sandbox": self.cfg.sandbox}, + data_dir=self.data_dir or "", + output_dir=output_dir, ) else: - llm = get_model(**self.cfg.server, tokenizer=self.tokenizer) + llm = get_model( + **self.cfg.server, tokenizer=self.tokenizer, data_dir=self.data_dir or "", output_dir=output_dir + ) if self.cfg.parallel_thinking.mode is not None: # We don't want to override these key variables which overlap with self.cfg diff --git a/nemo_skills/inference/model/__init__.py b/nemo_skills/inference/model/__init__.py index 164d92fcc8..595d8fd3ee 100644 --- a/nemo_skills/inference/model/__init__.py +++ b/nemo_skills/inference/model/__init__.py @@ -39,6 +39,7 @@ # Utilities from .vllm import VLLMModel +from .vllm_multimodal import VLLMMultimodalModel # Model implementations @@ -51,6 +52,7 @@ "azureopenai": AzureOpenAIModel, "gemini": GeminiModel, "vllm": VLLMModel, + "vllm_multimodal": VLLMMultimodalModel, "sglang": SGLangModel, "tts_nim": TTSNIMModel, "asr_nim": ASRNIMModel, diff --git a/nemo_skills/inference/model/base.py b/nemo_skills/inference/model/base.py index 9318bfb475..117096b4c7 100644 --- a/nemo_skills/inference/model/base.py +++ b/nemo_skills/inference/model/base.py @@ -75,9 +75,14 @@ def __init__( enable_soft_fail: bool = False, context_limit_retry_strategy: str | None = None, num_special_tokens_budget: int = 100, + # Directory paths for data and output + data_dir: str = "", + output_dir: str | None = None, ): self._tunnel = None self.model_name_or_path = model + self.data_dir = data_dir + self.output_dir = output_dir self.server_host = host self.server_port = port self.ssh_server = ssh_server diff --git a/nemo_skills/inference/model/vllm_multimodal.py b/nemo_skills/inference/model/vllm_multimodal.py new file mode 100644 index 0000000000..0569c9efd9 --- /dev/null +++ b/nemo_skills/inference/model/vllm_multimodal.py @@ -0,0 +1,110 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import base64 +import json +import logging +import os +import re + +from nemo_skills.utils import get_logger_name + +from .vllm import VLLMModel + +LOG = logging.getLogger(get_logger_name(__file__)) + +# Pattern to extract debug_info from content +DEBUG_INFO_PATTERN = re.compile(r"\n?(.*?)", re.DOTALL) + + +class VLLMMultimodalModel(VLLMModel): + """VLLMModel with support for saving audio responses to disk. + + When the server returns audio in the response, this model will: + 1. Save the audio bytes to a file in output_dir/audio/ + 2. Replace the base64 data with the file path in the result + """ + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.output_audio_dir = None + if self.output_dir: + self.output_audio_dir = os.path.join(self.output_dir, "audio") + os.makedirs(self.output_audio_dir, exist_ok=True) + LOG.info(f"Audio responses will be saved to: {self.output_audio_dir}") + + def _parse_chat_completion_response(self, response, include_response: bool = False, **kwargs) -> dict: + """Parse chat completion response and save any audio to disk.""" + result = super()._parse_chat_completion_response(response, include_response=include_response, **kwargs) + + # Extract debug_info from content (embedded as JSON in tags) + if "generation" in result and result["generation"]: + match = DEBUG_INFO_PATTERN.search(result["generation"]) + if match: + try: + result["debug_info"] = json.loads(match.group(1)) + # Strip debug_info from generation + result["generation"] = DEBUG_INFO_PATTERN.sub("", result["generation"]) + except json.JSONDecodeError: + LOG.warning("Failed to parse debug_info JSON from content") + + choice = response.choices[0] + if hasattr(choice.message, "audio") and choice.message.audio: + audio_result = self._process_audio_response(choice.message.audio, response.id) + result["audio"] = audio_result + + # Strip audio data from serialized_output to avoid duplication + if "serialized_output" in result: + for item in result["serialized_output"]: + if isinstance(item, dict) and "audio" in item: + # Keep only metadata, remove base64 data + if isinstance(item["audio"], dict) and "data" in item["audio"]: + del item["audio"]["data"] + # Also strip debug_info from serialized content + if isinstance(item, dict) and "content" in item and item["content"]: + item["content"] = DEBUG_INFO_PATTERN.sub("", item["content"]) + + return result + + def _process_audio_response(self, audio_data, response_id: str) -> dict: + """Process audio data: save to file and return metadata with path.""" + audio_info = { + "format": getattr(audio_data, "format", "wav"), + "sample_rate": getattr(audio_data, "sample_rate", 22050), + "transcript": getattr(audio_data, "transcript", None), + } + + audio_base64 = getattr(audio_data, "data", None) + if not audio_base64: + return audio_info + + if self.output_audio_dir: + try: + audio_bytes = base64.b64decode(audio_base64) + filename = f"{response_id}.wav" + filepath = os.path.join(self.output_audio_dir, filename) + + with open(filepath, "wb") as f: + f.write(audio_bytes) + + audio_info["path"] = filepath + audio_info["size_bytes"] = len(audio_bytes) + LOG.info(f"Saved audio: {filepath} ({len(audio_bytes)} bytes)") + except Exception as e: + LOG.warning(f"Failed to save audio: {e}") + audio_info["data"] = audio_base64 + else: + audio_info["data"] = audio_base64 + + return audio_info From 32205913bd212cd9fb6a2f005f58daa196abad19 Mon Sep 17 00:00:00 2001 From: George Armstrong Date: Tue, 6 Jan 2026 12:08:38 -0800 Subject: [PATCH 72/88] add swe-rebench to excluded datasets (#1154) Signed-off-by: George Armstrong Signed-off-by: Cheng-Ping Hsieh --- tests/gpu-tests/test_eval.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/gpu-tests/test_eval.py b/tests/gpu-tests/test_eval.py index 47060a1368..422fdfa830 100644 --- a/tests/gpu-tests/test_eval.py +++ b/tests/gpu-tests/test_eval.py @@ -37,6 +37,7 @@ "bfcl_v3", "bfcl_v4", "swe-bench", + "swe-rebench", "aai", "human-eval", "human-eval-infilling", From ba27f631a838b4ff739535f9a6f88f91e750d906 Mon Sep 17 00:00:00 2001 From: George Armstrong Date: Tue, 6 Jan 2026 14:22:58 -0800 Subject: [PATCH 73/88] Fix run.Script refactor (#1133) Signed-off-by: George Armstrong Signed-off-by: Cheng-Ping Hsieh --- .github/workflows/gpu_tests.yml | 10 +- nemo_skills/pipeline/generate.py | 452 ++++++++++------- nemo_skills/pipeline/nemo_evaluator.py | 217 ++++----- nemo_skills/pipeline/utils/__init__.py | 2 + nemo_skills/pipeline/utils/declarative.py | 560 ++++++++++++---------- nemo_skills/pipeline/utils/generation.py | 106 +++- nemo_skills/pipeline/utils/scripts.py | 428 +++++++++++++++++ tests/gpu-tests/test_eval.py | 2 +- tests/test_declarative_pipeline.py | 274 +++++------ tests/test_generation.py | 40 +- tests/test_nemo_evaluator_pipeline.py | 41 +- 11 files changed, 1413 insertions(+), 719 deletions(-) create mode 100644 nemo_skills/pipeline/utils/scripts.py diff --git a/.github/workflows/gpu_tests.yml b/.github/workflows/gpu_tests.yml index 16f77633a8..a500fc59b2 100644 --- a/.github/workflows/gpu_tests.yml +++ b/.github/workflows/gpu_tests.yml @@ -52,7 +52,15 @@ jobs: cd ${{ github.run_id }} nvidia-smi set -o pipefail # this will make sure next line returns non-0 exit code if tests fail - ./tests/gpu-tests/run_qwen.sh + # Run heartbeat in background, capture its PID, and ensure cleanup + (while true; do sleep 60; echo "[HEARTBEAT] $(date '+%Y-%m-%d %H:%M:%S') - still running..."; done) & + HEARTBEAT_PID=$! + # Run tests and capture exit code + EXIT_CODE=0 + ./tests/gpu-tests/run_qwen.sh || EXIT_CODE=$? + # Kill heartbeat and exit with test result + kill $HEARTBEAT_PID 2>/dev/null || true + exit $EXIT_CODE - name: Cleanup if: always() run: | diff --git a/nemo_skills/pipeline/generate.py b/nemo_skills/pipeline/generate.py index f33796d05c..90ec987bca 100644 --- a/nemo_skills/pipeline/generate.py +++ b/nemo_skills/pipeline/generate.py @@ -14,7 +14,7 @@ import importlib import logging import os -from typing import Callable, Dict, List, Optional +from typing import Dict, List, Optional import typer @@ -23,14 +23,17 @@ from nemo_skills.inference import GENERATION_MODULE_MAP, GenerationType from nemo_skills.pipeline.app import app, typer_unpacker from nemo_skills.pipeline.utils.cluster import parse_kwargs -from nemo_skills.pipeline.utils.commands import sandbox_command from nemo_skills.pipeline.utils.declarative import ( Command, CommandGroup, HardwareConfig, Pipeline, ) -from nemo_skills.pipeline.utils.server import get_free_port +from nemo_skills.pipeline.utils.scripts import ( + GenerationClientScript, + SandboxScript, + ServerScript, +) from nemo_skills.utils import ( compute_chunk_ids, get_logger_name, @@ -44,118 +47,160 @@ # TODO: add num_jobs here for consistency with eval? -def _create_commandgroup_from_config( - generation_cmd: str, - server_config: Optional[Dict], - with_sandbox: bool, - sandbox_port: Optional[int], +def _create_job_unified( + models: List[str], + server_configs: List[Optional[Dict]], + generation_params: Dict, cluster_config: Dict, installation_command: Optional[str], - get_server_command_fn: Callable, + with_sandbox: bool, partition: Optional[str], keep_mounts_for_sandbox: bool, task_name: str, log_dir: str, sbatch_kwargs: Optional[Dict] = None, sandbox_env_overrides: Optional[List[str]] = None, -) -> CommandGroup: - """Create a CommandGroup from server_config. - - Component ordering: - 1. Server (if server_config provided) - 2. Client command - 3. Sandbox (if with_sandbox=True) +) -> List[CommandGroup]: """ + Create CommandGroups for n models (unified for n=1 and n>1). + + Structure: + - Group 0: Model 0 server + client + (optional sandbox) + - Group 1: Model 1 server (if n>1) + - Group N: Model N server (if n>1) + + For n=1, returns a single-element list. The Pipeline automatically + optimizes single-group lists to efficient single-group jobs. + + Args: + models: List of model paths + server_configs: List of server configurations (one per model, None if not hosting) + generation_params: Dict of parameters for generation (output_dir, etc.) + cluster_config: Cluster configuration + installation_command: Installation command to run before client + with_sandbox: Whether to include sandbox + partition: Slurm partition + keep_mounts_for_sandbox: Whether to keep mounts for sandbox + task_name: Name for the task + log_dir: Directory for logs + sbatch_kwargs: Additional sbatch kwargs + + Returns: + List of CommandGroup objects (one per het group) + """ + num_models = len(models) + groups = [] + server_scripts = [] # Track server Script objects for cross-component references + + for model_idx, (model_path, server_config) in enumerate(zip(models, server_configs)): + components = [] + server_script = None + + # Track GPU/node requirements for this group (from server config) + group_gpus = 0 + group_nodes = 1 + + # 1. Add server if needed + if server_config is not None and int(server_config.get("num_gpus", 0)) > 0: + server_type = server_config["server_type"] + server_container = server_config.get("container") or cluster_config["containers"][server_type] - components = [] + # Create ServerScript + server_script = ServerScript( + server_type=server_type, + model_path=server_config["model_path"], + cluster_config=cluster_config, + num_gpus=server_config["num_gpus"], + num_nodes=server_config["num_nodes"], + server_args=server_config.get("server_args", ""), + server_entrypoint=server_config.get("server_entrypoint"), + port=server_config.get("server_port"), + allocate_port=(server_config.get("server_port") is None), + ) - # 1. Add server if server_config is provided - if server_config is not None and int(server_config["num_gpus"]) > 0: - server_type = server_config["server_type"] - # Get container from server_config if provided, otherwise fall back to cluster config - if "container" in server_config: - server_container = server_config.pop("container") + # Set group GPU/node requirements from server config + group_gpus = server_config["num_gpus"] + group_nodes = server_config["num_nodes"] + + server_cmd = Command( + script=server_script, + container=server_container, + name=f"{task_name}_model_{model_idx}_server" if num_models > 1 else f"{task_name}_server", + ) + components.append(server_cmd) + server_scripts.append(server_script) else: - server_container = cluster_config["containers"][server_type] + # No server for this model (pre-hosted) + server_scripts.append(None) + + # 2. Group 0 gets the client and sandbox + if model_idx == 0: + # Create sandbox script (if with_sandbox) + sandbox_script = None + if with_sandbox: + sandbox_script = SandboxScript( + cluster_config=cluster_config, + keep_mounts=keep_mounts_for_sandbox, + allocate_port=True, # Always allocate port for sandbox + env_overrides=sandbox_env_overrides, + ) + + sandbox_cmd = Command( + script=sandbox_script, + container=cluster_config["containers"]["sandbox"], + name=f"{task_name}_sandbox", + ) + components.append(sandbox_cmd) + + # Create client script with cross-component references to all servers + client_script = GenerationClientScript( + output_dir=generation_params["output_dir"], + input_file=generation_params.get("input_file"), + input_dir=generation_params.get("input_dir"), + extra_arguments=generation_params.get("extra_arguments", ""), + random_seed=generation_params.get("random_seed"), + chunk_id=generation_params.get("chunk_id"), + num_chunks=generation_params.get("num_chunks"), + preprocess_cmd=generation_params.get("preprocess_cmd"), + postprocess_cmd=generation_params.get("postprocess_cmd"), + wandb_parameters=generation_params.get("wandb_parameters"), + with_sandbox=with_sandbox, + script=generation_params.get("script", "nemo_skills.inference.generate"), + # Multi-server support (works for single and multi-model) + servers=server_scripts if server_scripts else None, + server_addresses_prehosted=generation_params.get("server_addresses_prehosted"), + model_names=generation_params.get("model_names"), + server_types=generation_params.get("server_types"), + sandbox=sandbox_script, + installation_command=installation_command, + ) - # Call server command builder directly with cluster_config - cmd, num_tasks = get_server_command_fn(**server_config, cluster_config=cluster_config) + client_cmd = Command( + script=client_script, + container=cluster_config["containers"]["nemo-skills"], + name=f"{task_name}", + ) + components.append(client_cmd) - # Create metadata dict - metadata = { - "num_tasks": num_tasks, - "gpus": server_config["num_gpus"], - "nodes": server_config["num_nodes"], - "log_prefix": "server", - } + # Only create group if it has components (skip empty groups for pre-hosted models) + if components: + group_tasks = server_script.num_tasks if (server_config and server_script) else 1 - server_cmd = Command( - command=cmd, - container=server_container, - gpus=server_config["num_gpus"], - nodes=server_config["num_nodes"], - name=task_name, - metadata=metadata, - ) - components.append(server_cmd) - - # 2. Add main generation command - # Note: General cluster config env vars are automatically added by get_env_variables() in get_executor() - client_env = {} - if with_sandbox and sandbox_port is not None: - client_env["NEMO_SKILLS_SANDBOX_PORT"] = str(sandbox_port) - - client_cmd = Command( - command=generation_cmd, - container=cluster_config["containers"]["nemo-skills"], - name=task_name, - installation_command=installation_command, - metadata={ - "log_prefix": "main", - "environment": client_env, - }, - ) - components.append(client_cmd) - - # 3. Add sandbox if requested - if with_sandbox: - # Call sandbox command builder directly with cluster_config - cmd, metadata = sandbox_command(cluster_config=cluster_config, port=sandbox_port) - metadata["log_prefix"] = "sandbox" - - # Apply user-specified environment overrides for the sandbox - if sandbox_env_overrides: - sandbox_env = metadata.get("environment", {}) - for override in sandbox_env_overrides: - key, value = override.split("=", 1) - sandbox_env[key] = value - metadata["environment"] = sandbox_env - - sandbox_cmd = Command( - command=cmd, - container=cluster_config["containers"]["sandbox"], - name=task_name, - metadata=metadata, - ) + group = CommandGroup( + commands=components, + hardware=HardwareConfig( + partition=partition, + num_gpus=group_gpus, + num_nodes=group_nodes, + num_tasks=group_tasks, + sbatch_kwargs=sbatch_kwargs, + ), + name=f"{task_name}_model_{model_idx}_group" if num_models > 1 else task_name, + log_dir=log_dir, + ) + groups.append(group) - components.append(sandbox_cmd) - - # Find maximum GPUs/nodes needed by any component for the HardwareConfig - # The job-level resource request must be the maximum across all components - max_gpus = max((comp.gpus or 0) for comp in components) - max_nodes = max((comp.nodes or 1) for comp in components) - - return CommandGroup( - commands=components, - hardware=HardwareConfig( - partition=partition, - num_gpus=max_gpus, - num_nodes=max_nodes, - sbatch_kwargs=sbatch_kwargs, - ), - name=task_name, - log_dir=log_dir, - ) + return groups @app.command(context_settings={"allow_extra_args": True, "ignore_unknown_options": True}) @@ -186,21 +231,45 @@ def generate( "If not specified, will use the registered generation module for the " "generation type (which is required in this case).", ), - model: str = typer.Option(None, help="Path to the model or model name in API"), - server_address: str = typer.Option( - None, help="Use ip:port for self-hosted models or the API url if using model providers" + model: List[str] = typer.Option( + None, + help="Path to the model(s). CLI: space-separated. Python API: string or list. " + "Single value broadcasts to all models for multi-model generation.", + ), + server_address: List[str] = typer.Option( + None, + help="Server address(es). CLI: space-separated. Python API: string or list. " + "Single value broadcasts to all models.", + ), + server_type: List[pipeline_utils.SupportedServers] = typer.Option( + ..., + help="Server type(s). CLI: space-separated. Python API: string or list. " + "Single value broadcasts to all models.", ), - server_type: pipeline_utils.SupportedServers = typer.Option(..., help="Type of server to use"), - server_gpus: int = typer.Option(None, help="Number of GPUs to use if hosting the model"), - server_nodes: int = typer.Option(1, help="Number of nodes required for hosting LLM server"), - server_args: str = typer.Option("", help="Any extra arguments to pass to the server"), - server_entrypoint: str = typer.Option( + server_gpus: List[int] = typer.Option( None, - help="Path to the entrypoint of the server. " - "If not specified, will use the default entrypoint for the server type.", + help="Number of GPUs per model. CLI: space-separated ints. Python API: int or list. " + "Single value broadcasts to all models.", ), - server_container: str = typer.Option( - None, help="Override container image for the hosted server (if server_gpus is set)" + server_nodes: List[int] = typer.Option( + [1], + help="Number of nodes per model. CLI: space-separated ints. Python API: int or list. " + "Single value broadcasts to all models.", + ), + server_args: List[str] = typer.Option( + [""], + help="Server arguments per model. CLI: space-separated. Python API: string or list. " + "Single value broadcasts to all models.", + ), + server_entrypoint: List[str] = typer.Option( + None, + help="Server entrypoint(s). CLI: space-separated. Python API: string or list. " + "Single value broadcasts to all models.", + ), + server_container: List[str] = typer.Option( + None, + help="Container image(s). CLI: space-separated. Python API: string or list. " + "Single value broadcasts to all models.", ), dependent_jobs: int = typer.Option(0, help="Specify this to launch that number of dependent jobs"), mount_paths: str = typer.Option(None, help="Comma separated list of paths to mount on the remote machine"), @@ -296,7 +365,18 @@ def generate( None, help="Internal option to specify task dependencies.", hidden=True ), ): - """Generate LLM completions for a given input file. + """Generate LLM completions for single or multiple models. + + Supports both single-model and multi-model generation through a unified interface. + + Parameter Types: + Multi-model parameters (model, server_*, etc.) use List[T] type hints for Typer CLI + compatibility, but accept both scalars and lists when called from Python: + - CLI: --model m1 m2 (space-separated) → Typer converts to ["m1", "m2"] + - Python API: model="m1" or model=["m1", "m2"] → Both work (normalized internally) + - Single values broadcast to all models: server_gpus=8 → [8, 8, 8] for 3 models + + Multi-model usage requires either --generation-type or --generation-module. Run `python -m nemo_skills.inference.generate --help` for other supported arguments (need to be prefixed with ++, since we use Hydra for that script). @@ -306,10 +386,42 @@ def generate( LOG.info("Starting generation job") LOG.info("Extra arguments that will be passed to the underlying script: %s", extra_arguments) - try: - server_type = server_type.value - except AttributeError: - pass + # Normalize model configuration to list + models_list = pipeline_utils.normalize_models_config(model) + num_models = len(models_list) + + LOG.info(f"Number of models: {num_models}") + for model_idx, model_name in enumerate(models_list): + LOG.info(f" Model {model_idx}: {model_name}") + + # Convert server_type enum values to strings + def convert_server_type_to_string(server_type): + return server_type.value if hasattr(server_type, "value") else server_type + + if isinstance(server_type, list): + server_type = [convert_server_type_to_string(st) for st in server_type] + else: + server_type = convert_server_type_to_string(server_type) + + # Normalize all server parameters to per-model lists + server_types_list = pipeline_utils.normalize_parameter(server_type, num_models, "server_type") + server_gpus_list = pipeline_utils.normalize_parameter(server_gpus, num_models, "server_gpus") + server_nodes_list = pipeline_utils.normalize_parameter(server_nodes, num_models, "server_nodes") + server_args_list = pipeline_utils.normalize_parameter(server_args, num_models, "server_args") + server_entrypoints_list = pipeline_utils.normalize_parameter(server_entrypoint, num_models, "server_entrypoint") + server_containers_list = pipeline_utils.normalize_parameter(server_container, num_models, "server_container") + + if server_address is not None: + server_addresses_list = pipeline_utils.normalize_parameter(server_address, num_models, "server_address") + else: + server_addresses_list = [None] * num_models + + # Validate multi-model requirements + if num_models > 1: + if generation_type is None and generation_module is None: + raise ValueError( + "Multi-model generation requires either --generation-type or --generation-module to be specified" + ) if log_samples: wandb_parameters = { @@ -325,8 +437,6 @@ def generate( else: wandb_parameters = None - get_random_port = pipeline_utils.should_get_random_port(server_gpus, exclusive) - if random_seeds and num_random_seeds: raise ValueError("Cannot specify both random_seeds and num_random_seeds") if num_random_seeds: @@ -355,8 +465,6 @@ def generate( check_mounted_paths=check_mounted_paths, ) - original_server_address = server_address - if generation_module is not None and generation_type is not None: raise ValueError("Cannot specify both generation_module and generation_type. ") if generation_module is None: @@ -407,36 +515,36 @@ def generate( chunk_id=None, ) for chunk_id in chunk_ids: - # Configure client (same as before) - server_config, server_address, extra_arguments = pipeline_utils.configure_client( - model=model, - server_type=server_type, - server_address=original_server_address, - server_gpus=server_gpus, - server_nodes=server_nodes, - server_args=server_args, - server_entrypoint=server_entrypoint, - server_container=server_container, - extra_arguments=extra_arguments_original, - get_random_port=get_random_port, - ) + # Configure clients for each model + server_configs = [] + server_addresses_resolved = [] + # For single model: configure_client returns extra_args with server config appended + # For multi-model: use original extra_args (server config added as lists in get_generation_cmd) + extra_arguments = extra_arguments_original + + for model_idx in range(num_models): + get_random_port_for_server = pipeline_utils.should_get_random_port( + server_gpus_list[model_idx], exclusive + ) - # Build generation command (same as before) - cmd = pipeline_utils.get_generation_cmd( - input_file=input_file, - input_dir=input_dir, - random_seed=seed, - output_dir=output_dir, - extra_arguments=extra_arguments, - chunk_id=chunk_id, - num_chunks=num_chunks, - preprocess_cmd=preprocess_cmd, - postprocess_cmd=postprocess_cmd, - wandb_parameters=wandb_parameters if seed_idx == 0 else None, - script=generation_module, - with_sandbox=with_sandbox, - ) - cmd = pipeline_utils.wrap_python_path(cmd=cmd) + srv_config, srv_address, srv_extra_args = pipeline_utils.configure_client( + model=models_list[model_idx], + server_type=server_types_list[model_idx], + server_address=server_addresses_list[model_idx], + server_gpus=server_gpus_list[model_idx], + server_nodes=server_nodes_list[model_idx], + server_args=server_args_list[model_idx], + server_entrypoint=server_entrypoints_list[model_idx], + server_container=server_containers_list[model_idx], + extra_arguments=extra_arguments_original if model_idx == 0 else "", + get_random_port=get_random_port_for_server, + ) + server_configs.append(srv_config) + server_addresses_resolved.append(srv_address) + + # For single model, capture the extra_args with server config from configure_client + if model_idx == 0 and num_models == 1: + extra_arguments = srv_extra_args # Base task name (shared across all dependent jobs in the chain) task_name = f"{expname}-rs{seed}" if seed is not None else expname @@ -448,22 +556,35 @@ def generate( prev_job = None for dep_idx in range(dependent_jobs + 1): - # Allocate sandbox port if needed - # This must be done BEFORE creating CommandGroup so client knows the port - if with_sandbox: - current_sandbox_port = get_free_port(strategy="random") if get_random_port else 6000 - else: - current_sandbox_port = None + # Build generation parameters dict for Script + generation_params = { + "output_dir": output_dir, + "input_file": input_file, + "input_dir": input_dir, + "extra_arguments": extra_arguments, + "random_seed": seed, + "chunk_id": chunk_id, + "num_chunks": num_chunks, + "preprocess_cmd": preprocess_cmd, + "postprocess_cmd": postprocess_cmd, + "wandb_parameters": wandb_parameters if seed_idx == 0 else None, + "script": generation_module, + # Multi-model specific fields + "server_addresses_prehosted": server_addresses_resolved, + "model_names": models_list, + "server_types": server_types_list, + } - # Create CommandGroup for this task - cmd_group = _create_commandgroup_from_config( - generation_cmd=cmd, - server_config=server_config.copy() if server_config else None, - with_sandbox=with_sandbox, - sandbox_port=current_sandbox_port, + # Create CommandGroup(s) using Script objects + # For multi-model, this creates multiple CommandGroups (one per model + one for client) + # For single-model, this creates a single CommandGroup + job_groups = _create_job_unified( + models=models_list, + server_configs=[cfg.copy() if cfg else None for cfg in server_configs], + generation_params=generation_params, cluster_config=cluster_config, installation_command=installation_command, - get_server_command_fn=generation_task.get_server_command_fn(), + with_sandbox=with_sandbox, partition=partition, keep_mounts_for_sandbox=keep_mounts_for_sandbox, task_name=task_name, @@ -487,11 +608,16 @@ def generate( # Subsequent jobs in chain depend on previous job (use job object, not string) job_deps = [prev_job] + # For multi-group jobs, use "groups" key; for single-group, use "group" key job_spec = { "name": internal_job_name, - "group": cmd_group, "dependencies": job_deps, } + if len(job_groups) > 1: + job_spec["groups"] = job_groups + else: + job_spec["group"] = job_groups[0] + jobs.append(job_spec) prev_job = job_spec # Track for next iteration diff --git a/nemo_skills/pipeline/nemo_evaluator.py b/nemo_skills/pipeline/nemo_evaluator.py index 020162692a..39838737ed 100644 --- a/nemo_skills/pipeline/nemo_evaluator.py +++ b/nemo_skills/pipeline/nemo_evaluator.py @@ -89,7 +89,7 @@ import copy import logging -from dataclasses import dataclass +from dataclasses import dataclass, field from pathlib import Path from typing import Dict, List, Optional @@ -97,12 +97,12 @@ from nemo_evaluator_launcher.api import RunConfig from nemo_evaluator_launcher.common.helpers import get_eval_factory_command from nemo_evaluator_launcher.common.mapping import get_task_from_mapping, load_tasks_mapping -from omegaconf import DictConfig, OmegaConf +from omegaconf import OmegaConf import nemo_skills.pipeline.utils as pipeline_utils from nemo_skills.pipeline.app import app, typer_unpacker -from nemo_skills.pipeline.utils.commands import vllm_server_command from nemo_skills.pipeline.utils.declarative import Command, CommandGroup, HardwareConfig, Pipeline +from nemo_skills.pipeline.utils.scripts import BaseJobScript, ServerScript from nemo_skills.utils import get_logger_name, setup_logging LOG = logging.getLogger(get_logger_name(__file__)) @@ -289,8 +289,8 @@ def nemo_evaluator( expname=expname, idx=idx, task_name=task.name, - launcher_run_cfg=launcher_run_cfg, - task_cfg=task, + launcher_run_cfg=OmegaConf.to_container(launcher_run_cfg, resolve=True), + task_cfg=OmegaConf.to_container(task, resolve=True), task_definition=task_definition, base_output_root=base_output_root, eval_image=eval_image, @@ -443,10 +443,8 @@ def _create_serving_command_obj( idx: int, task_name: str, ) -> Command: - """Create a Command object for a hosted serving component (main or judge server). + """Create a `Command` backed by a `ServerScript` for a hosted serving component. - This function wraps vllm_server_command and standardizes container selection, - logging prefixes, and metadata for both main and judge servers. Args: cluster_config: Cluster configuration dictionary @@ -464,54 +462,53 @@ def _create_serving_command_obj( task_name: Task name for naming Returns: - Command object configured for the serving component + Command: A Command object whose `script` is a configured `ServerScript`. """ stype = (server_type or "vllm").lower() - sargs = args or "" if stype != "vllm": LOG.warning("Only vllm server_type is supported currently; got %s", stype) - cmd_str, meta = vllm_server_command( + server_script = ServerScript( + server_type=stype, + model_path=model or "", cluster_config=cluster_config, - model=model, # type: ignore[arg-type] + num_gpus=gpus, + num_nodes=nodes or 1, + server_args=args or "", + server_entrypoint=entrypoint, port=port, - server_type=stype, - gpus=gpus, - nodes=nodes, - args=sargs, - entrypoint=entrypoint, + allocate_port=port is None, ) - # Resolve container fallback when not explicitly provided + # Judge servers get a distinct log prefix for clarity + if is_judge: + server_script.log_prefix = "judge-server" + if not container: container = cluster_config["containers"][stype] - log_prefix = "judge-server" if is_judge else "server" name_role = "judge-server" if is_judge else "server" return Command( - command=cmd_str, + script=server_script, container=container, - gpus=gpus, - nodes=nodes or 1, name=f"{expname}-{name_role}-{idx}-{task_name}", - metadata={ - **meta, - "gpus": gpus, - "log_prefix": log_prefix, - }, ) @dataclass class _TaskCreationContext: - """Local helper to pass around the information about the task and easier logic sharing.""" + """Local helper to pass around the information about the task and easier logic sharing. + + Note: launcher_run_cfg and task_cfg are stored as plain dicts (not OmegaConf) to allow + serialization by nemo_run/fiddle. Convert back to DictConfig if OmegaConf operations are needed. + """ expname: str idx: int task_name: str - launcher_run_cfg: RunConfig - task_cfg: DictConfig + launcher_run_cfg: dict # Stored as plain dict for serialization compatibility + task_cfg: dict # Stored as plain dict for serialization compatibility task_definition: dict base_output_root: Optional[str] eval_image: str @@ -630,12 +627,7 @@ def _build_judge_server_if_needed(ctx: _TaskCreationContext) -> Optional[Command def _build_client_command( ctx: _TaskCreationContext, main_server_cmd: Optional[Command], judge_server_cmd: Optional[Command] ) -> Command: - """Build Command for evaluator client. - - The client command behavior depends on server hosting: - - If servers are co-hosted: Uses lambda factory to resolve runtime URLs via hostname_ref/meta_ref - - If using external servers: Uses static URLs from server_base_url/judge_server_base_url - - If no servers: Uses URLs from evaluator config or defaults + """Create the evaluator client `Command` using `EvaluatorClientScript`. Args: ctx: Task creation context with all configuration @@ -643,100 +635,26 @@ def _build_client_command( judge_server_cmd: Judge server Command if self-hosted, None otherwise Returns: - Command object for evaluator client + Command: A Command whose script builds the evaluator CLI at runtime """ - if ctx.hosting_server or ctx.hosting_judge: - # Co-hosted servers: Use lambda factory to resolve runtime URLs - # The lambda is evaluated at execution time when het_group_index is assigned - def _client_cmd_factory(): - waits: List[str] = [] - target_url: Optional[str] = None - judge_url: Optional[str] = None - # Build main server URL from runtime references - if ctx.hosting_server and main_server_cmd is not None: - server_host = main_server_cmd.hostname_ref() - server_port_val = main_server_cmd.meta_ref("port") - base_url = f"http://{server_host}:{server_port_val}" - waits.append(pipeline_utils.get_server_wait_cmd(f"{base_url}{ctx.server_health_path}")) - target_url = f"{base_url}{ctx.server_api_path}" - - # Build judge server URL from runtime references - if ctx.hosting_judge and judge_server_cmd is not None: - jhost = judge_server_cmd.hostname_ref() - jport = judge_server_cmd.meta_ref("port") - jbase = f"http://{jhost}:{jport}" - waits.append(pipeline_utils.get_server_wait_cmd(f"{jbase}{ctx.judge_server_health_path}")) - judge_url = f"{jbase}{ctx.judge_server_api_path}" - - # Wait for servers to be ready, then run evaluator - wait_cmd = " && ".join(waits) if waits else "true" - cmd = _build_task_cmd( - task_name=ctx.task_name, - launcher_run_cfg=ctx.launcher_run_cfg, - task_cfg=ctx.task_cfg, - task_definition=ctx.task_definition, - expname=ctx.expname, - base_output_root=ctx.base_output_root, - url_override=target_url, - model_id=ctx.server_model, - judge_url_override=judge_url, - judge_model_id=ctx.judge_server_model, - ) - return f"{wait_cmd} && {cmd}" - - return Command( - command=_client_cmd_factory, - container=ctx.eval_image, - gpus=ctx.job_gpus or None, - nodes=ctx.job_nodes or 1, - name=f"{ctx.expname}-client-{ctx.idx}-{ctx.task_name}", - metadata={ - "log_prefix": "main", - "environment": ctx.env_vars, - "gpus": ctx.job_gpus or None, - }, - ) - - # No hosted servers: Use external URLs or config defaults - server_url = None - if ctx.with_external_server and ctx.server_base_url: - server_url = ctx.server_base_url.rstrip("/") + ctx.server_api_path - judge_url = None - if ctx.with_external_judge and ctx.judge_server_base_url: - judge_url = ctx.judge_server_base_url.rstrip("/") + ctx.judge_server_api_path - - eval_cmd = _build_task_cmd( - task_name=ctx.task_name, - launcher_run_cfg=ctx.launcher_run_cfg, - task_cfg=ctx.task_cfg, - task_definition=ctx.task_definition, - expname=ctx.expname, - base_output_root=ctx.base_output_root, - url_override=server_url, - model_id=ctx.server_model, - judge_url_override=judge_url, - judge_model_id=ctx.judge_server_model, + client_script = EvaluatorClientScript( + ctx=ctx, + main_server_script=main_server_cmd.script if main_server_cmd else None, + judge_server_script=judge_server_cmd.script if judge_server_cmd else None, ) return Command( - command=eval_cmd, + script=client_script, container=ctx.eval_image, - gpus=None, - nodes=ctx.job_nodes or 1, - name=f"{ctx.expname}-{ctx.idx}-{ctx.task_name}", - metadata={ - "log_prefix": "main", - "environment": ctx.env_vars, - "gpus": ctx.job_gpus or None, - }, + name=f"{ctx.expname}-client-{ctx.idx}-{ctx.task_name}", ) def _build_task_cmd( task_name: str, - launcher_run_cfg: DictConfig, - task_cfg: DictConfig, + launcher_run_cfg: dict, + task_cfg: dict, task_definition: dict, expname: str, base_output_root: Optional[str], @@ -752,8 +670,8 @@ def _build_task_cmd( Args: task_name: Task identifier (e.g., "ifeval", "gpqa_diamond") - launcher_run_cfg: Global evaluator configuration from RunConfig - task_cfg: Task-specific configuration (may include task-level overrides) + launcher_run_cfg: Global evaluator configuration (as plain dict) + task_cfg: Task-specific configuration (as plain dict, may include task-level overrides) task_definition: Task definition from mapping (container, harness info) expname: Experiment name for output directory structure base_output_root: Base directory for task outputs @@ -771,7 +689,9 @@ def _build_task_cmd( - Judge: config.params.extra.judge.url Output directory is set to: {base_output_root}/{expname}/nemo-evaluator-results/{task_name} """ - task_cfg_copy = copy.deepcopy(task_cfg) + # Convert back to DictConfig for OmegaConf operations + launcher_run_cfg = OmegaConf.create(launcher_run_cfg) + task_cfg_copy = OmegaConf.create(copy.deepcopy(task_cfg)) if url_override: OmegaConf.update(task_cfg_copy, "overrides", {"target.api_endpoint.url": url_override}, force_add=True) @@ -806,3 +726,56 @@ def _build_task_cmd( cmd_struct = get_eval_factory_command(launcher_run_cfg, task_cfg_copy, task_definition) return cmd_struct.cmd + + +@dataclass(kw_only=True) +class EvaluatorClientScript(BaseJobScript): + """run.Script implementation for nemo-evaluator client with runtime server resolution.""" + + ctx: _TaskCreationContext + main_server_script: Optional[ServerScript] = None + judge_server_script: Optional[ServerScript] = None + log_prefix: str = field(default="main", init=False) + + def __post_init__(self): + def build_command(): + waits: List[str] = [] + target_url: Optional[str] = None + judge_url: Optional[str] = None + + if self.ctx.hosting_server and self.main_server_script is not None: + server_host = self.main_server_script.hostname_ref() + base_url = f"http://{server_host}:{self.main_server_script.port}" + waits.append(pipeline_utils.get_server_wait_cmd(f"{base_url}{self.ctx.server_health_path}")) + target_url = f"{base_url}{self.ctx.server_api_path}" + elif self.ctx.with_external_server and self.ctx.server_base_url: + target_url = self.ctx.server_base_url.rstrip("/") + self.ctx.server_api_path + + if self.ctx.hosting_judge and self.judge_server_script is not None: + judge_host = self.judge_server_script.hostname_ref() + judge_base = f"http://{judge_host}:{self.judge_server_script.port}" + waits.append(pipeline_utils.get_server_wait_cmd(f"{judge_base}{self.ctx.judge_server_health_path}")) + judge_url = f"{judge_base}{self.ctx.judge_server_api_path}" + elif self.ctx.with_external_judge and self.ctx.judge_server_base_url: + judge_url = self.ctx.judge_server_base_url.rstrip("/") + self.ctx.judge_server_api_path + + cmd = _build_task_cmd( + task_name=self.ctx.task_name, + launcher_run_cfg=self.ctx.launcher_run_cfg, + task_cfg=self.ctx.task_cfg, + task_definition=self.ctx.task_definition, + expname=self.ctx.expname, + base_output_root=self.ctx.base_output_root, + url_override=target_url, + model_id=self.ctx.server_model, + judge_url_override=judge_url, + judge_model_id=self.ctx.judge_server_model, + ) + + wait_cmd = " && ".join(waits) if waits else None + final_cmd = f"{wait_cmd} && {cmd}" if wait_cmd else cmd + env_vars = copy.deepcopy(self.ctx.env_vars) + return final_cmd, {"environment": env_vars} + + self.set_inline(build_command) + super().__post_init__() diff --git a/nemo_skills/pipeline/utils/__init__.py b/nemo_skills/pipeline/utils/__init__.py index 1e470f3539..3e738a530f 100644 --- a/nemo_skills/pipeline/utils/__init__.py +++ b/nemo_skills/pipeline/utils/__init__.py @@ -49,6 +49,8 @@ get_chunked_rs_filename, get_generation_cmd, get_remaining_jobs, + normalize_models_config, + normalize_parameter, wrap_cmd, ) from nemo_skills.pipeline.utils.mounts import ( diff --git a/nemo_skills/pipeline/utils/declarative.py b/nemo_skills/pipeline/utils/declarative.py index e294a3ed82..7029dcc638 100644 --- a/nemo_skills/pipeline/utils/declarative.py +++ b/nemo_skills/pipeline/utils/declarative.py @@ -12,39 +12,71 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import annotations + +import logging +from contextlib import nullcontext +from dataclasses import dataclass +from typing import Dict, List, Optional, Tuple, Union + +import nemo_run as run + +from nemo_skills.pipeline.utils import ( + get_env_variables, + get_executor, + get_exp, + get_exp_handles, + get_registered_external_repo, + get_tunnel, + run_exp, + temporary_env_update, +) +from nemo_skills.pipeline.utils.exp import ( + REUSE_CODE_EXP, + get_packaging_job_key, + tunnel_hash, +) +from nemo_skills.pipeline.utils.mounts import is_mounted_filepath +from nemo_skills.pipeline.utils.server import wrap_python_path +from nemo_skills.utils import get_logger_name + """ -Simplified declarative pipeline system using only Command for all task types. +Simplified declarative pipeline system using Command with run.Script objects. Basic Example (Single job with multiple commands): - from nemo_skills.pipeline.utils.commands import vllm_server_command, sandbox_command + from nemo_skills.pipeline.utils.scripts import ServerScript, SandboxScript, GenerationClientScript from nemo_skills.pipeline.utils.declarative import Command, CommandGroup, HardwareConfig, Pipeline - from nemo_skills.pipeline.utils.server import get_free_port - - # Allocate ports for server and sandbox - server_port = get_free_port(strategy="random") - sandbox_port = get_free_port(strategy="random") - - # Commands that run together in one SLURM job - # Note: Lambdas are needed for cross-component references (hostname_ref, meta_ref) - # which aren't resolved until het_group_index is assigned at pipeline execution time. - server_cmd, server_meta = vllm_server_command(cluster_cfg, model="Qwen/Qwen3-8B", port=server_port) - server = Command(command=server_cmd, gpus=8, name="server", metadata=server_meta) - - sandbox_cmd, sandbox_meta = sandbox_command(cluster_cfg, port=sandbox_port) - sandbox = Command(command=sandbox_cmd, name="sandbox", metadata=sandbox_meta) - - # This lambda is ESSENTIAL - server.hostname_ref() and meta_ref() aren't available until runtime - # Client needs NEMO_SKILLS_SANDBOX_PORT to connect to sandbox - client = Command( - command=lambda: f"curl {server.hostname_ref()}:{server.meta_ref('port')}/health", - name="client", - metadata={"environment": {"NEMO_SKILLS_SANDBOX_PORT": str(sandbox_port)}} + + # Create Script objects for server and sandbox + # Scripts handle port allocation, cross-component references, and command building + server_script = ServerScript( + server_type="vllm", + model_path="Qwen/Qwen2.5-Math-7B-Instruct", + server_args="--tensor-parallel-size 1" ) + sandbox_script = SandboxScript() + + # Create generation client that references server and sandbox + # Cross-component references (hostname_ref, port) are resolved at runtime + client_script = GenerationClientScript( + output_dir="/results/inference", + extra_arguments="++prompt_config=math ++split=test", + servers=[server_script], # References server for hostname/port + model_names=["Qwen/Qwen2.5-Math-7B-Instruct"], + server_types=["vllm"], + sandbox=sandbox_script, # References sandbox for port + with_sandbox=True, + ) + + # Wrap Scripts in Commands with container and resource info + server = Command(script=server_script, container="vllm", name="server") + sandbox = Command(script=sandbox_script, container="nemo-skills", name="sandbox") + client = Command(script=client_script, container="nemo-skills", name="client") - # Group them together + # Group them together (they run in one SLURM job) inference_group = CommandGroup( commands=[server, sandbox, client], - hardware=HardwareConfig(partition="batch"), + hardware=HardwareConfig(partition="batch", num_gpus=1), name="inference" ) @@ -57,13 +89,27 @@ pipeline.run() Advanced Example (Multiple jobs with dependencies and heterogeneous components): + from nemo_skills.pipeline.utils.scripts import ServerScript, SandboxScript, GenerationClientScript + from nemo_run import Script + log_dir = "/experiments/full_pipeline/logs" - # Job 1: Preprocessing - preprocess = Command( - command="python preprocess.py --input data.jsonl --output processed.jsonl", - gpus=0, - name="preprocess" + + # Job 1: Preprocessing with custom Script + @dataclass(kw_only=True) + class PreprocessScript(Script): + input_file: str + output_file: str + + def __post_init__(self): + cmd = f"python preprocess.py --input {self.input_file} --output {self.output_file}" + self.inline = cmd + object.__setattr__(self, 'entrypoint', 'bash') + + preprocess_script = PreprocessScript( + input_file="data.jsonl", + output_file="processed.jsonl" ) + preprocess = Command(script=preprocess_script, name="preprocess") prep_group = CommandGroup( commands=[preprocess], hardware=HardwareConfig(partition="cpu"), @@ -72,39 +118,76 @@ ) prep_job = {"name": "prep", "group": prep_group} - # Job 2: Two different model servers (HETEROGENEOUS SLURM job with 2 het components) - # Allocate ports for each server/sandbox pair - from nemo_skills.pipeline.utils.server import get_free_port - server_8b_port = get_free_port(strategy="random") - sandbox_8b_port = get_free_port(strategy="random") - server_32b_port = get_free_port(strategy="random") - sandbox_32b_port = get_free_port(strategy="random") - - # Build commands with cluster_config - server_8b_cmd, server_8b_meta = vllm_server_command(cluster_config, model="Qwen/Qwen3-8B", port=server_8b_port) - sandbox_8b_cmd, sandbox_8b_meta = sandbox_command(cluster_config, port=sandbox_8b_port) - server_32b_cmd, server_32b_meta = vllm_server_command(cluster_config, model="Qwen/Qwen3-32B", port=server_32b_port) - sandbox_32b_cmd, sandbox_32b_meta = sandbox_command(cluster_config, port=sandbox_32b_port) + # Job 2: Two different model servers (HETEROGENEOUS SLURM job with 2 het groups) + # 8B model group + server_8b = ServerScript( + server_type="vllm", + model_path="Qwen/Qwen2.5-Math-7B-Instruct", + server_args="--tensor-parallel-size 1" + ) + sandbox_8b = SandboxScript() + client_8b = GenerationClientScript( + output_dir="/results/eval_8b", + extra_arguments="++prompt_config=math", + servers=[server_8b], + model_names=["Qwen/Qwen2.5-Math-7B-Instruct"], + server_types=["vllm"], + sandbox=sandbox_8b, + with_sandbox=True, + ) - server_8b = Command(command=server_8b_cmd, gpus=8, name="server_8b", metadata=server_8b_meta) - sandbox_8b = Command(command=sandbox_8b_cmd, name="sandbox_8b", metadata=sandbox_8b_meta) - eval_8b = Command(command="python eval.py --model 8b", gpus=1, name="eval_8b") + group_8b = CommandGroup( + commands=[ + Command(script=server_8b, container="vllm", name="server_8b"), + Command(script=sandbox_8b, container="nemo-skills", name="sandbox_8b"), + Command(script=client_8b, container="nemo-skills", name="eval_8b"), + ], + hardware=HardwareConfig(partition="batch", num_gpus=1), + name="eval_8b", + log_dir=log_dir + ) - server_32b = Command(command=server_32b_cmd, gpus=8, name="server_32b", metadata=server_32b_meta) - sandbox_32b = Command(command=sandbox_32b_cmd, name="sandbox_32b", metadata=sandbox_32b_meta) - eval_32b = Command(command="python eval.py --model 32b", gpus=1, name="eval_32b") + # 32B model group + server_32b = ServerScript( + server_type="vllm", + model_path="Qwen/Qwen2.5-Math-32B-Instruct", + server_args="--tensor-parallel-size 4" + ) + sandbox_32b = SandboxScript() + client_32b = GenerationClientScript( + output_dir="/results/eval_32b", + extra_arguments="++prompt_config=math", + servers=[server_32b], + model_names=["Qwen/Qwen2.5-Math-32B-Instruct"], + server_types=["vllm"], + sandbox=sandbox_32b, + with_sandbox=True, + ) - group_8b = CommandGroup(commands=[server_8b, sandbox_8b, eval_8b], name="eval_8b", log_dir=log_dir) - group_32b = CommandGroup(commands=[server_32b, sandbox_32b, eval_32b], name="eval_32b", log_dir=log_dir) + group_32b = CommandGroup( + commands=[ + Command(script=server_32b, container="vllm", name="server_32b"), + Command(script=sandbox_32b, container="nemo-skills", name="sandbox_32b"), + Command(script=client_32b, container="nemo-skills", name="eval_32b"), + ], + hardware=HardwareConfig(partition="batch", num_gpus=4), + name="eval_32b", + log_dir=log_dir + ) evals_job = {"name": "evals", "groups": [group_8b, group_32b], "dependencies": [prep_job]} # Job 3: Report generation (depends on both evaluations) - report = Command( - command="python generate_report.py --output report.txt", - gpus=0, - name="report" - ) + @dataclass(kw_only=True) + class ReportScript(Script): + output_file: str + + def __post_init__(self): + self.inline = f"python generate_report.py --output {self.output_file}" + object.__setattr__(self, 'entrypoint', 'bash') + + report_script = ReportScript(output_file="report.txt") + report = Command(script=report_script, name="report") report_group = CommandGroup(commands=[report], name="report", log_dir=log_dir) # Create pipeline with dependency graph @@ -121,130 +204,56 @@ pipeline.run() """ -import logging -import shlex -from contextlib import nullcontext -from dataclasses import dataclass, field -from typing import Callable, Dict, List, Optional, Tuple, Union - -import nemo_run as run - -from nemo_skills.pipeline.utils import ( - get_env_variables, - get_executor, - get_exp, - get_exp_handles, - get_tunnel, - run_exp, - temporary_env_update, -) -from nemo_skills.pipeline.utils.commands import wrap_command -from nemo_skills.pipeline.utils.exp import ( - REUSE_CODE_EXP, - get_packaging_job_key, - install_packages_wrap, - tunnel_hash, -) -from nemo_skills.pipeline.utils.mounts import is_mounted_filepath -from nemo_skills.pipeline.utils.packager import get_registered_external_repo -from nemo_skills.utils import get_logger_name - LOG = logging.getLogger(get_logger_name(__file__)) @dataclass class Command: - """Declarative command for running tasks in containers. - - The command can be either: - - A string: evaluated immediately - - A callable (lambda): evaluated lazily when the task is prepared + """Declarative command for running tasks in containers using run.Script objects. - Lambdas are ONLY needed for cross-component references (hostname_ref, meta_ref). - The het_group_index isn't assigned until pipeline execution, so these must be lazy: - # Lambda is ESSENTIAL here - server.hostname_ref() and meta_ref() don't exist yet - client = Command(command=lambda: f"curl {server.hostname_ref()}:{server.meta_ref('port')}") + Example: + server = ServerScript(server_type="vllm", model_path="/models/llama", ...) + Command(script=server, container="vllm", name="my_server") """ - # Command can be a string or callable (lambda). - # Lambdas are primarily used for cross-component references (hostname_ref, meta_ref). - command: Union[str, Callable] + script: run.Script container: str = "nemo-skills" - gpus: Optional[int] = None - nodes: int = 1 name: str = "command" - working_dir: str = "/nemo_run/code" - env_vars: Dict[str, str] = field(default_factory=dict) - installation_command: Optional[str] = None - port: Optional[int] = None # Can be set from metadata - metadata: Dict[str, any] = field(default_factory=dict) # Stores metadata from command builders - het_group_index: Optional[int] = None # Set per-job by Pipeline (not global) - - def __post_init__(self): - # Wrap plain strings with environment setup - if isinstance(self.command, str) and (self.env_vars or self.working_dir): - self.command = wrap_command(self.command, self.working_dir, self.env_vars) - - def hostname_ref(self) -> str: - """Get hostname reference for hetjob cross-component communication.""" - if self.het_group_index is None: - return "127.0.0.1" # Local fallback - # For heterogeneous SLURM jobs, resolve nodelist to actual hostname - return f"$(scontrol show hostnames $SLURM_JOB_NODELIST_HET_GROUP_{self.het_group_index} | head -n1)" - - def meta_ref(self, key: str) -> str: - """Get metadata value (like port). Fails if key not found.""" - if key not in self.metadata: - raise KeyError( - f"Metadata key '{key}' not found in Command '{self.name}'. " - f"Available keys: {list(self.metadata.keys())}" - ) - return str(self.metadata[key]) - def prepare_for_execution(self, cluster_config: Dict) -> Tuple[str, Dict]: - """Prepare command for execution. + def prepare_for_execution(self, cluster_config: Dict) -> Tuple[run.Script, Dict]: + """Prepare script for execution. This method: - 1. Evaluates callables (resolves cross-component references) - 2. Wraps with installation_command if provided + 1. Evaluates lazy commands (if script.inline is callable) + 2. Builds execution config from Script fields Returns: - Tuple of (final_command, execution_config) + Tuple of (Script_object, execution_config) """ - # 1. Evaluate if callable (for cross-component references like hostname_ref) - if callable(self.command): - result = self.command() + runtime_metadata = {} + + # If script.inline is callable (lazy command building), evaluate it now + if callable(self.script.inline): + result = self.script.inline() if isinstance(result, tuple): - final_command, runtime_metadata = result - # Deep merge metadata, especially environment dict - for key, value in runtime_metadata.items(): - if key == "environment" and key in self.metadata: - # Merge environment dicts instead of replacing - self.metadata[key].update(value) - else: - self.metadata[key] = value + evaluated_command, runtime_metadata = result else: - final_command = result - else: - final_command = self.command + evaluated_command = result - # 2. Wrap with installation_command if provided - if self.installation_command: - final_command = install_packages_wrap(final_command, self.installation_command) + # Update script.inline with evaluated command + self.script.set_inline(evaluated_command) - # 3. Build execution config from metadata + # Build execution config from Script fields execution_config = { - "num_tasks": self.metadata.get("num_tasks", 1), - "num_gpus": self.metadata.get("gpus", self.gpus or 0), - "num_nodes": self.metadata.get("nodes", self.nodes), - "environment": self.metadata.get("environment", {}), - "log_prefix": self.metadata.get("log_prefix", "main"), - "mounts": self.metadata.get("mounts"), - "container": self.metadata.get("container", self.container), # Use container from metadata if available + "log_prefix": getattr(self.script, "log_prefix", "main"), + "environment": runtime_metadata.get("environment", {}), + "mounts": None, # Mounts not currently exposed by Scripts + "container": self.container, } - return final_command, execution_config + # Return the Script object itself + return self.script, execution_config def get_name(self) -> str: return self.name @@ -257,6 +266,7 @@ class HardwareConfig: partition: Optional[str] = None num_gpus: Optional[int] = None num_nodes: Optional[int] = None + num_tasks: Optional[int] = 1 sbatch_kwargs: Optional[dict] = None @@ -482,16 +492,49 @@ def run(self, dry_run: bool = False, log_dir: Optional[str] = None, _reuse_exp=N return exp - def _prepare_command(self, command, cluster_config: Dict) -> Tuple[str, Dict]: - """Prepare command and handle mpirun wrapping.""" - final_cmd, exec_config = command.prepare_for_execution(cluster_config) - - # Handle mpirun wrapping for non-SLURM executors - num_tasks = exec_config["num_tasks"] - if cluster_config["executor"] != "slurm" and num_tasks > 1: - final_cmd = f"mpirun --allow-run-as-root -np {num_tasks} bash -c {shlex.quote(final_cmd)}" + def _prepare_command(self, command, cluster_config: Dict) -> Tuple[run.Script, Dict]: + """Prepare command for execution. - return final_cmd, exec_config + Returns: + Tuple of (Script_object, exec_config) + """ + script, exec_config = command.prepare_for_execution(cluster_config) + # Only rewrite paths for "none" executor (native execution without containers) + # For "local" executor (Docker), paths should stay as /nemo_run/code/... since + # that's where the code is mounted inside the container + if cluster_config.get("executor") == "none": + script = self._rewrite_local_paths(script) + # Note: mpirun wrapping for multi-task scripts is handled by the executor + return script, exec_config + + def _rewrite_local_paths(self, script: run.Script) -> run.Script: + """For executor='none', replace /nemo_run/code paths with local repo paths.""" + nemo_repo = get_registered_external_repo("nemo_skills") + if nemo_repo is None: + return script + + pkg_path = str(nemo_repo.path) + repo_root = str(nemo_repo.path.parent) + + def _replace(cmd: str) -> str: + return cmd.replace("/nemo_run/code/nemo_skills", pkg_path).replace("/nemo_run/code", repo_root) + + inline_cmd = script.inline + if isinstance(inline_cmd, str): + script.set_inline(_replace(inline_cmd)) + elif callable(inline_cmd): + original_inline = inline_cmd + + def wrapped_inline(): + result = original_inline() + if isinstance(result, tuple): + cmd, metadata = result + return _replace(cmd), metadata + return _replace(result) + + script.set_inline(wrapped_inline) + + return script def _resolve_container(self, exec_config: Dict, command, cluster_config: Dict) -> str: """Resolve container name to image path.""" @@ -513,6 +556,7 @@ def _create_executor( total_het_groups: int, overlap: bool, dependencies: Optional[List] = None, + job_name_override: Optional[str] = None, ): """Create executor with optional environment update.""" env_context = ( @@ -521,14 +565,23 @@ def _create_executor( else nullcontext() ) + # Check if the script should span all nodes from the group's HardwareConfig. + # Scripts with span_group_nodes=True (e.g., ServerScript) use the group's num_nodes. + # Scripts with span_group_nodes=False (default) run on 1 node - important for multi-node + # setups with --overlap where client/sandbox should only run on the master node. + span_group_nodes = getattr(command.script, "span_group_nodes", False) + num_nodes = 1 + if span_group_nodes and hardware and hardware.num_nodes is not None: + num_nodes = hardware.num_nodes + with env_context: return get_executor( cluster_config=cluster_config, container=container_image, - num_nodes=exec_config["num_nodes"], - tasks_per_node=exec_config["num_tasks"], - gpus_per_node=exec_config["num_gpus"], - job_name=command.name, + num_nodes=num_nodes, + tasks_per_node=hardware.num_tasks if hardware and hardware.num_tasks is not None else 1, + gpus_per_node=hardware.num_gpus if hardware and hardware.num_gpus is not None else 0, + job_name=job_name_override if job_name_override else command.name, log_dir=log_dir, log_prefix=exec_config["log_prefix"], partition=hardware.partition if hardware else None, @@ -567,81 +620,105 @@ def _plan_and_add_job( if log_dir is None: raise ValueError(f"CommandGroup '{groups[0].name}' must have log_dir set, or provide it to pipeline.run()") - commands: List[str] = [] + scripts: List[run.Script] = [] executors: List = [] het_group_indices: List[int] = [] - # In heterogeneous jobs, collect environment from all commands for cross-component refs - shared_env_vars: Dict[str, str] = {} - if heterogeneous: - for het_idx, group in enumerate(groups): - for command in group.commands: - _, exec_config_probe = command.prepare_for_execution(cluster_config) - shared_env_vars.update(exec_config_probe.get("environment", {})) + # Assign het_group_index values before evaluating any commands so cross-references + # (e.g., hostname_ref) see the correct indices regardless of processing order. + for het_idx, group in enumerate(groups): + for command in group.commands: + command.script.het_group_index = het_idx if heterogeneous else None - # Share packager across executors for efficiency (single-group only) - shared_packager = None + # Prepare commands once and collect runtime data for a second pass where we + # construct executors. This ensures all scripts have resolved cross-references. + prepared_commands: List[Dict] = [] + shared_env_vars: Dict[str, str] = {} - # Build commands and executors for het_idx, group in enumerate(groups): has_multiple_components = len(group.commands) > 1 total_het_groups = ( len(groups) if heterogeneous else (len(group.commands) if has_multiple_components else 1) ) - # For single-group jobs with multiple components, allow job-level GPU override for sbatch allocation - job_level_gpus = ( - group.hardware.num_gpus if (not heterogeneous and has_multiple_components and group.hardware) else None - ) - for comp_idx, command in enumerate(group.commands): - # Assign het_group_index ONLY for heterogeneous jobs (per-job, not global) - # Non-heterogeneous jobs use localhost, so het_group_index should remain None - if heterogeneous: - command.het_group_index = het_idx - else: - command.het_group_index = None - - final_cmd, exec_config = self._prepare_command(command, cluster_config) - commands.append(final_cmd) - - # Adjust GPU allocation (first component gets job-level GPUs for sbatch) for single-group jobs - exec_config["num_gpus"] = exec_config["num_gpus"] or 0 - if (not heterogeneous) and (comp_idx == 0) and (job_level_gpus is not None): - exec_config["num_gpus"] = job_level_gpus - - # Merge shared environment for heterogeneous jobs - if heterogeneous and shared_env_vars: - exec_config["environment"].update(shared_env_vars) - - # Resolve container and create executor - container_image = self._resolve_container(exec_config, command, cluster_config) - # Pass external dependencies only to the first executor (SLURM doesn't support per-component dependencies in hetjobs) - exec_dependencies = external_deps if (het_idx == 0 and comp_idx == 0) else None - executor = self._create_executor( - command, - exec_config, - container_image, - cluster_config, - log_dir, - group.hardware, - heterogeneous, - het_idx if heterogeneous else comp_idx, - total_het_groups, - (len(group.commands) > 1), - dependencies=exec_dependencies, + script, exec_config = self._prepare_command(command, cluster_config) + + if isinstance(script.inline, str): + if cluster_config.get("executor") not in ("none", "local"): + script.set_inline(wrap_python_path(script.inline)) + + prepared_commands.append( + { + "het_idx": het_idx, + "comp_idx": comp_idx, + "group": group, + "command": command, + "script": script, + "exec_config": exec_config, + "total_het_groups": total_het_groups, + "overlap": len(group.commands) > 1, + } ) - # Share packager across executors for single-group jobs - if not heterogeneous: - if comp_idx == 0 and het_idx == 0: - shared_packager = executor.packager - else: - executor.packager = shared_packager - - executors.append(executor) if heterogeneous: - het_group_indices.append(het_idx) + shared_env_vars.update(exec_config.get("environment", {})) + + # Share packager across executors for efficiency (single-group only) + shared_packager = None + + # Build commands and executors using prepared data + for entry in prepared_commands: + het_idx = entry["het_idx"] + comp_idx = entry["comp_idx"] + group = entry["group"] + command = entry["command"] + script = entry["script"] + exec_config = entry["exec_config"] + total_het_groups = entry["total_het_groups"] + overlap = entry["overlap"] + + scripts.append(script) + + # Merge shared environment for heterogeneous jobs + if heterogeneous and shared_env_vars: + exec_config["environment"].update(shared_env_vars) + + # Resolve container and create executor + container_image = self._resolve_container(exec_config, command, cluster_config) + # Pass external dependencies only to the first executor (SLURM doesn't support per-component dependencies in hetjobs) + exec_dependencies = external_deps if (het_idx == 0 and comp_idx == 0) else None + + # Always use group.name for SLURM job name (consistent across all components) + # The group name is set to task_name in generate.py, without component suffixes + # Component names (like {task_name}_server, {task_name}_sandbox) are only used for log_prefix + job_name_for_slurm = group.name + + executor = self._create_executor( + command, + exec_config, + container_image, + cluster_config, + log_dir, + group.hardware, + heterogeneous, + het_idx if heterogeneous else comp_idx, + total_het_groups, + overlap, + dependencies=exec_dependencies, + job_name_override=job_name_for_slurm, + ) + + # Share packager across executors for single-group jobs + if not heterogeneous: + if comp_idx == 0 and het_idx == 0: + shared_packager = executor.packager + else: + executor.packager = shared_packager + + executors.append(executor) + if heterogeneous: + het_group_indices.append(het_idx) # For heterogeneous jobs, set het_group_indices on the first executor if heterogeneous and executors: @@ -676,13 +753,7 @@ def _plan_and_add_job( # If reuse_code=False, clear cache REUSE_CODE_EXP.pop(tunnel_hash(tunnel), None) - # Handle executor="none" path replacements (single-group only) - if (not heterogeneous) and cluster_config["executor"] == "none": - for idx in range(len(commands)): - commands[idx] = commands[idx].replace( - "/nemo_run/code/nemo_skills", str(get_registered_external_repo("nemo_skills").path) - ) - commands[idx] = commands[idx].replace("/nemo_run/code", "./") + # Note: Path replacements for executor="none" are no longer needed with Script interface # Ray metadata handling if self.with_ray and cluster_config["executor"] == "slurm": @@ -693,19 +764,24 @@ def _plan_and_add_job( # Add to experiment and return task ID # Note: Internal dependencies (task handles from same experiment) go to exp.add() # External dependencies (SLURM job IDs from other experiments) go to executor - if (not heterogeneous) and len(commands) == 1: + if (not heterogeneous) and len(scripts) == 1: + # Single script - pass directly to exp.add() + if metadata: + scripts[0].metadata = metadata task_id = exp.add( - run.Script(inline=commands[0], metadata=metadata), + scripts[0], executor=executors[0], name="nemo-run", dependencies=internal_deps, ) else: + # Multiple scripts or heterogeneous job + # Apply metadata to first script only + if metadata: + scripts[0].metadata = metadata + task_id = exp.add( - [ - run.Script(inline=cmd, metadata=(metadata if idx == 0 else None)) - for idx, cmd in enumerate(commands) - ], + scripts, executor=executors, name="nemo-run", dependencies=internal_deps, diff --git a/nemo_skills/pipeline/utils/generation.py b/nemo_skills/pipeline/utils/generation.py index cd576053c1..863b196e20 100644 --- a/nemo_skills/pipeline/utils/generation.py +++ b/nemo_skills/pipeline/utils/generation.py @@ -17,6 +17,7 @@ import shlex import subprocess from collections import defaultdict +from typing import Any, List, Optional, Union from nemo_skills.pipeline.utils.cluster import get_tunnel from nemo_skills.pipeline.utils.mounts import get_unmounted_path @@ -26,6 +27,81 @@ LOG = logging.getLogger(get_logger_name(__file__)) +def normalize_models_config( + model: Optional[Union[str, List[str]]], +) -> List[str]: + """ + Normalize model specification to list. + + Handles both scalar and list inputs: + - CLI (Typer): Converts single values to single-element lists automatically + - Python API: Accepts both strings and lists + + Args: + model: Model path(s) - string or list from Python API, list from CLI + + Returns: + List of model paths + + Raises: + ValueError: If model is None or empty + """ + if model is None: + raise ValueError("Must specify --model") + + # Handle string (Python API with single model) + if isinstance(model, str): + return [model] + + # Handle list + if len(model) == 0: + raise ValueError("Must specify --model") + return list(model) + + +def normalize_parameter( + param_value: Any, + num_models: int, + param_name: str, +) -> List[Any]: + """ + Normalize a parameter to a per-model list. + + Handles both scalar and list inputs for flexible usage: + - CLI (Typer): Converts single values to single-element lists automatically + - Python API: Accepts both scalars and lists directly + + Broadcast logic: + - Scalar value: Broadcast to all models [value] * num_models + - Single-element list: Broadcast to all models + - Multi-element list: Must match num_models exactly + + Args: + param_value: Parameter value (scalar or list) + num_models: Number of models + param_name: Name of parameter (for error messages) + + Returns: + List of parameter values (one per model) + + Raises: + ValueError: If list length doesn't match num_models + """ + if not isinstance(param_value, list): + return [param_value] * num_models + + if len(param_value) == num_models: + return list(param_value) + + if len(param_value) == 1: + return param_value * num_models + + raise ValueError( + f"Parameter {param_name} has {len(param_value)} values but {num_models} models specified. " + f"Must be 1 value (broadcast) or {num_models} values (per-model)." + ) + + def get_chunked_rs_filename( output_dir: str, random_seed: int = None, @@ -294,8 +370,20 @@ def get_generation_cmd( wandb_parameters=None, with_sandbox: bool = False, script: str = "nemo_skills.inference.generate", + # Optional: for multi-model generation + server_addresses: Optional[List[str]] = None, + model_names: Optional[List[str]] = None, + server_types: Optional[List[str]] = None, ): - """Construct the generation command for language model inference.""" + """Construct the generation command for language model inference. + + Supports both single-model and multi-model generation. For multi-model: + - server_addresses: List of server addresses (one per model) + - model_names: List of model names (one per model) + - server_types: List of server types (one per model) + + For single-model, server config is passed via extra_arguments. + """ if input_file is None and input_dir is None: raise ValueError("Either input_file or input_dir must be provided.") if input_file is not None and input_dir is not None: @@ -313,6 +401,7 @@ def get_generation_cmd( output_dir=output_dir, random_seed=random_seed, ) + # Preamble for generation commands: added at executor/declarative level cmd = "export HYDRA_FULL_ERROR=1 && " # Separate Hydra config args (--config-*) from override args (++) @@ -327,6 +416,21 @@ def get_generation_cmd( else: # It's a module name, use -m flag cmd += f"python -m {script} {hydra_config_args} {common_args} " + + # Add multi-model configuration if provided + if server_addresses is not None and model_names is not None: + num_models = len(model_names) + if num_models > 1: + # Multi-model: pass server configuration as lists + model_names_arg = ",".join(model_names) + cmd += f"++server.model=[{model_names_arg}] " + + server_types_arg = ",".join(server_types) + cmd += f"++server.server_type=[{server_types_arg}] " + + server_addresses_arg = ",".join(server_addresses) + cmd += f"++server.base_url=[{server_addresses_arg}] " + job_end_cmd = "" if random_seed is not None and input_dir is None: # if input_dir is not None, we default to greedy generations diff --git a/nemo_skills/pipeline/utils/scripts.py b/nemo_skills/pipeline/utils/scripts.py new file mode 100644 index 0000000000..0657eec98b --- /dev/null +++ b/nemo_skills/pipeline/utils/scripts.py @@ -0,0 +1,428 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Script classes for NeMo-Skills pipeline components. + +These classes wrap NeMo-Run's run.Script interface to provide typed, reusable +job components (servers, clients, sandboxes) with explicit fields and +cross-component reference support for heterogeneous jobs. + +Example: + # Create a server script with automatic port allocation + server = ServerScript( + server_type="vllm", + model_path="/models/llama-8b", + cluster_config=cluster_config, + num_gpus=8, + ) + + # Create a client that references the server + client = GenerationClientScript( + output_dir="/results", + input_file="/data/input.jsonl", + server=server, # Cross-component reference + ) + + # Use in Command objects + Command(script=server, container="vllm", ...) + Command(script=client, container="nemo-skills", ...) +""" + +import logging +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Tuple, Union + +import nemo_run as run + +from nemo_skills.pipeline.utils.commands import sandbox_command +from nemo_skills.pipeline.utils.exp import install_packages_wrap +from nemo_skills.pipeline.utils.generation import get_generation_cmd +from nemo_skills.pipeline.utils.server import get_free_port, get_server_command +from nemo_skills.utils import get_logger_name + +if TYPE_CHECKING: + # Avoid circular imports for type hints + pass + +LOG = logging.getLogger(get_logger_name(__file__)) + + +@dataclass +class BaseJobScript(run.Script): + """Base class for job component scripts with heterogeneous job support. + + This class provides: + - het_group_index tracking for cross-component references in heterogeneous SLURM jobs + - hostname_ref() method for getting hostnames in het jobs + - Common pattern for Script initialization + + Attributes: + het_group_index: Index in heterogeneous job group (set by Pipeline at runtime) + span_group_nodes: Whether to span all nodes from the group's HardwareConfig. + When False (default), the script runs on 1 node regardless of group config. + When True, the script spans all nodes specified in the group's num_nodes. + This is important for multi-node setups with --overlap where the server + needs multiple nodes but client/sandbox should run on the master node only. + """ + + het_group_index: Optional[int] = field(default=None, init=False, repr=False) + span_group_nodes: bool = False # Default: run on 1 node + installation_command: Optional[str] = None + entrypoint: str = field(default="bash", init=False) + + def __post_init__(self): + """Wrap inline command with installation_command if provided.""" + if not self.installation_command: + return + + if callable(self.inline): + original_inline = self.inline + + def wrapped_inline(): + result = original_inline() + if isinstance(result, tuple): + command, metadata = result + return install_packages_wrap(command, self.installation_command), metadata + return install_packages_wrap(result, self.installation_command) + + self.set_inline(wrapped_inline) + elif isinstance(self.inline, str): + self.set_inline(install_packages_wrap(self.inline, self.installation_command)) + + def set_inline(self, command: Union[str, Callable, run.Script]) -> None: + """Set the inline command safely on frozen dataclass.""" + object.__setattr__(self, "inline", command) + + def hostname_ref(self) -> str: + """Get hostname reference for hetjob cross-component communication. + + Returns a shell variable reference that resolves to the master node hostname + for this het group. Uses environment variables automatically exported by nemo-run: + SLURM_MASTER_NODE_HET_GROUP_0, SLURM_MASTER_NODE_HET_GROUP_1, etc. + + These are set via: + export SLURM_MASTER_NODE_HET_GROUP_N=$(scontrol show hostnames $SLURM_JOB_NODELIST_HET_GROUP_N | head -n1) + """ + if self.het_group_index is None: + return "127.0.0.1" # Local fallback for non-heterogeneous jobs + + # Use the environment variable exported by nemo-run + return f"${{SLURM_MASTER_NODE_HET_GROUP_{self.het_group_index}:-localhost}}" + + +@dataclass(kw_only=True) +class ServerScript(BaseJobScript): + """Script for model inference servers (vLLM, TRT-LLM, SGLang, etc.). + + This script wraps server command builders and provides: + - Automatic port allocation if not specified + - Type-safe server configuration + - Cross-component address sharing (get_address()) + - Resource requirement tracking (num_gpus, num_nodes, num_tasks) + + Attributes: + server_type: Type of server (vllm, trtllm, sglang, megatron, openai, etc.) + model_path: Path to model weights or model name for API services + cluster_config: Cluster configuration dictionary + num_gpus: Number of GPUs required (default: 8) + num_nodes: Number of nodes required (default: 1) + server_args: Additional server-specific arguments + server_entrypoint: Custom server entrypoint script (optional) + port: Server port (allocated automatically if None) + allocate_port: Whether to allocate port automatically (default: True) + num_tasks: Number of MPI tasks (computed in __post_init__) + log_prefix: Prefix for log files (default: "server") + + Example: + # Basic usage + server = ServerScript( + server_type="vllm", + model_path="/models/llama-3-8b", + cluster_config=cluster_config, + num_gpus=8, + ) + + # Access allocated port + print(f"Server will run on port {server.port}") + + # Get full address for client connection + address = server.get_address() # Returns "hostname:port" + """ + + server_type: str + model_path: str + cluster_config: Dict + num_gpus: int = 8 + num_nodes: int = 1 + server_args: str = "" + server_entrypoint: Optional[str] = None # Custom server entrypoint script + port: Optional[int] = None + allocate_port: bool = True + + # Server spans all group nodes (e.g., for distributed inference) + span_group_nodes: bool = True + + # Computed fields (set in __post_init__) + num_tasks: int = field(init=False, repr=False) + log_prefix: str = field(default="server", init=False) + + def __post_init__(self): + """Initialize server script. + + - Allocates port if not provided + - Builds server command using get_server_command() + - Sets self.inline to the command string + - Computes num_tasks from server command builder + """ + # Allocate port if not provided + if self.port is None and self.allocate_port: + self.port = get_free_port(strategy="random") + LOG.debug(f"Allocated port {self.port} for {self.server_type} server") + + # Build server command + cmd, self.num_tasks = get_server_command( + server_type=self.server_type, + num_gpus=self.num_gpus, + num_nodes=self.num_nodes, + model_path=self.model_path, + cluster_config=self.cluster_config, + server_port=self.port, + server_args=self.server_args, + server_entrypoint=self.server_entrypoint, + ) + + self.set_inline(cmd) + super().__post_init__() + + def get_address(self) -> str: + """Get server address for client connections. + + Returns hostname:port string that clients can use to connect. + In heterogeneous jobs, hostname_ref() returns a bash expression + that resolves at runtime. + + Returns: + Server address in format "hostname:port" + + Example: + # Use in client command + client_cmd = f"python client.py --server-url http://{server.get_address()}" + """ + return f"{self.hostname_ref()}:{self.port}" + + +@dataclass(kw_only=True) +class SandboxScript(BaseJobScript): + """Script for code execution sandbox container. + + The sandbox provides a secure environment for executing LLM-generated code. + This script wraps sandbox command builders and provides: + - Automatic port allocation + - Mount configuration (can optionally keep mounts, though risky) + - Type-safe sandbox configuration + + Attributes: + cluster_config: Cluster configuration dictionary + port: Sandbox port (allocated automatically if None) + keep_mounts: Whether to keep filesystem mounts (default: False, risky if True). + Note: This is stored for documentation but actually handled at + the executor level, not in the sandbox command itself. + allocate_port: Whether to allocate port automatically (default: True) + log_prefix: Prefix for log files (default: "sandbox") + + Example: + sandbox = SandboxScript( + cluster_config=cluster_config, + keep_mounts=False, # Safer: sandbox has no access to mounted paths + ) + + # Client can reference sandbox port + client = GenerationClientScript(..., sandbox=sandbox) + """ + + cluster_config: Dict + port: Optional[int] = None + keep_mounts: bool = False + allocate_port: bool = True + env_overrides: Optional[List[str]] = None # Extra env vars in KEY=VALUE form + log_prefix: str = field(default="sandbox", init=False) + + def __post_init__(self): + """Initialize sandbox script. + + - Allocates port if not provided + - Builds sandbox command using sandbox_command() + - Sets self.inline to a callable that returns command and environment vars + """ + # Allocate port if not provided + if self.port is None and self.allocate_port: + self.port = get_free_port(strategy="random") + LOG.debug(f"Allocated port {self.port} for sandbox") + + # Build sandbox command and metadata (including environment vars) + # Note: keep_mounts is handled at the executor level, not in the command itself + cmd, metadata = sandbox_command( + cluster_config=self.cluster_config, + port=self.port, + ) + + # Use a callable to return both command and environment variables + # This ensures the sandbox's LISTEN_PORT and NGINX_PORT are properly set + def build_cmd() -> Tuple[str, Dict]: + env = dict(metadata.get("environment", {})) + # Apply user-specified environment overrides + if self.env_overrides: + for override in self.env_overrides: + key, value = override.split("=", 1) + env[key] = value + return cmd, {"environment": env} + + self.set_inline(build_cmd) + super().__post_init__() + + +@dataclass(kw_only=True) +class GenerationClientScript(BaseJobScript): + """Script for LLM generation/inference client. + + This script wraps generation command builders and provides: + - Cross-component references to multiple servers and sandbox + - Lazy command building for runtime hostname resolution + - Type-safe generation configuration + - Environment variable handling for sandbox/server communication + + Attributes: + output_dir: Directory for output files + input_file: Input JSONL file (mutually exclusive with input_dir) + input_dir: Input directory (mutually exclusive with input_file) + extra_arguments: Additional arguments for generation script + random_seed: Random seed for sampling (optional) + chunk_id: Chunk ID for parallel processing (optional) + num_chunks: Total number of chunks (required if chunk_id set) + preprocess_cmd: Command to run before generation (optional) + postprocess_cmd: Command to run after generation (optional) + wandb_parameters: WandB logging configuration (optional) + with_sandbox: Whether sandbox is enabled + script: Module or file path for generation script (default: nemo_skills.inference.generate) + servers: List of ServerScript references (None for pre-hosted servers) + server_addresses_prehosted: Addresses for pre-hosted servers (parallel to servers list) + model_names: Model names for multi-model generation (optional) + server_types: Server types for multi-model generation (optional) + sandbox: Reference to SandboxScript for cross-component communication (optional) + log_prefix: Prefix for log files (default: "main") + + Examples: + # Single server + client = GenerationClientScript( + output_dir="/results", + input_file="/data/input.jsonl", + servers=[server_script], + model_names=["llama-8b"], + server_types=["vllm"], + ) + + # Multi-model with self-hosted and pre-hosted servers + client = GenerationClientScript( + output_dir="/results", + input_file="/data/input.jsonl", + servers=[server1, server2, None], # None = pre-hosted + server_addresses_prehosted=["", "", "https://api.openai.com"], + model_names=["llama-8b", "llama-70b", "gpt-4"], + server_types=["vllm", "vllm", "openai"], + sandbox=sandbox_script, + with_sandbox=True, + ) + """ + + output_dir: str + input_file: Optional[str] = None + input_dir: Optional[str] = None + extra_arguments: str = "" + random_seed: Optional[int] = None + chunk_id: Optional[int] = None + num_chunks: Optional[int] = None + preprocess_cmd: Optional[str] = None + postprocess_cmd: Optional[str] = None + wandb_parameters: Optional[Dict] = None + with_sandbox: bool = False + script: str = "nemo_skills.inference.generate" + + # Cross-component references for single/multi-model + servers: Optional[List[Optional["ServerScript"]]] = None + server_addresses_prehosted: Optional[List[str]] = None + model_names: Optional[List[str]] = None + server_types: Optional[List[str]] = None + sandbox: Optional["SandboxScript"] = None + + log_prefix: str = field(default="main", init=False) + + def __post_init__(self): + """Initialize generation client script with lazy command building. + + Builds command lazily via a callable that is evaluated when het_group_index + is assigned, allowing hostname_ref() to resolve correctly for heterogeneous jobs. + + This works for both cases: + - With cross-refs: Resolves server hostnames and sandbox ports at runtime + - Without cross-refs: Just builds the command string (no runtime resolution needed) + """ + + def build_cmd() -> Tuple[str, Dict]: + """Build command at runtime when cross-refs are resolved.""" + env_vars = {} + + # Add sandbox port to environment if sandbox is referenced + if self.sandbox: + env_vars["NEMO_SKILLS_SANDBOX_PORT"] = str(self.sandbox.port) + + # Build server addresses if servers are provided + server_addresses = None + if self.servers is not None: + server_addresses = [] + for server_idx, server_script in enumerate(self.servers): + if server_script is not None: + # Self-hosted: construct address from hostname and port refs + addr = f"{server_script.hostname_ref()}:{server_script.port}" + else: + # Pre-hosted: use the address from server_addresses_prehosted + addr = self.server_addresses_prehosted[server_idx] + server_addresses.append(addr) + + # Build generation command + cmd = get_generation_cmd( + output_dir=self.output_dir, + input_file=self.input_file, + input_dir=self.input_dir, + extra_arguments=self.extra_arguments, + random_seed=self.random_seed, + chunk_id=self.chunk_id, + num_chunks=self.num_chunks, + preprocess_cmd=self.preprocess_cmd, + postprocess_cmd=self.postprocess_cmd, + wandb_parameters=self.wandb_parameters, + with_sandbox=self.with_sandbox, + script=self.script, + # Multi-model parameters (None for single-model) + server_addresses=server_addresses, + model_names=self.model_names, + server_types=self.server_types, + ) + + # Return command and runtime metadata (environment vars) + return cmd, {"environment": env_vars} + + # Always use lazy command building + self.set_inline(build_cmd) + super().__post_init__() diff --git a/tests/gpu-tests/test_eval.py b/tests/gpu-tests/test_eval.py index 422fdfa830..a753cb7f53 100644 --- a/tests/gpu-tests/test_eval.py +++ b/tests/gpu-tests/test_eval.py @@ -44,7 +44,7 @@ "mbpp", "mmau-pro", "asr-leaderboard", - "aalcr", # Has tokenization mismatch issues + "mrcr", "audiobench", "librispeech-pc", } diff --git a/tests/test_declarative_pipeline.py b/tests/test_declarative_pipeline.py index 92117e403d..9d76fd4721 100644 --- a/tests/test_declarative_pipeline.py +++ b/tests/test_declarative_pipeline.py @@ -16,6 +16,7 @@ import json import os +from typing import Callable, Optional from unittest.mock import MagicMock, patch import pytest @@ -26,127 +27,83 @@ from nemo_skills.pipeline.utils.declarative import Command, CommandGroup, HardwareConfig, Pipeline -class TestCommand: - """Test Command class functionality.""" +class DummyScript: + """Minimal run.Script stand-in for unit tests.""" - def test_command_basic_string(self): - """Test creating a Command with a simple string.""" - cmd = Command(command="echo hello", name="test") - assert cmd.name == "test" - assert cmd.container == "nemo-skills" - assert cmd.gpus is None - assert cmd.nodes == 1 + def __init__(self, inline: str | Callable | None = "echo test"): + self.inline = inline + self.log_prefix = "main" + self.metadata = {} + self.het_group_index: Optional[int] = None - def test_command_with_metadata(self): - """Test Command with metadata passed separately.""" - cmd = Command(command="echo hello", name="server", metadata={"port": 8080, "log_prefix": "server"}) - assert cmd.metadata["port"] == 8080 - assert cmd.metadata["log_prefix"] == "server" - # Command gets wrapped with working_dir by default - assert "echo hello" in cmd.command + def set_inline(self, inline): + self.inline = inline - def test_command_with_callable(self): - """Test Command with callable that returns tuple.""" + def hostname_ref(self) -> str: + if self.het_group_index is None: + return "127.0.0.1" + return f"${{SLURM_MASTER_NODE_HET_GROUP_{self.het_group_index}:-localhost}}" - def make_cmd(): - return ("echo world", {"port": 5000}) - cmd = Command(command=make_cmd, name="dynamic") - assert callable(cmd.command) - assert cmd.name == "dynamic" +def make_command(*, inline: str | Callable | None = "echo test", name: str = "cmd", script: DummyScript | None = None): + """Helper to build Command objects with DummyScript instances.""" + script_obj = script or DummyScript(inline=inline) + return Command(script=script_obj, name=name) + + +class TestCommand: + """Tests for the new Script-based Command wrapper.""" + + def test_command_basic_script(self): + cmd = make_command(inline="echo hello", name="test") + assert cmd.name == "test" + assert cmd.container == "nemo-skills" + assert cmd.script.inline == "echo hello" def test_command_prepare_for_execution_string(self): - """Test prepare_for_execution with string command.""" - cmd = Command(command="python script.py", gpus=2, name="test") + cmd = make_command(inline="python script.py", name="test") cluster_config = {"executor": "local", "containers": {}} - final_cmd, exec_config = cmd.prepare_for_execution(cluster_config) + script_obj, exec_config = cmd.prepare_for_execution(cluster_config) - assert "python script.py" in final_cmd - assert exec_config["num_gpus"] == 2 - assert exec_config["num_nodes"] == 1 - assert exec_config["num_tasks"] == 1 + assert script_obj.inline == "python script.py" + assert exec_config["log_prefix"] == "main" + assert exec_config["environment"] == {} def test_command_prepare_for_execution_callable(self): - """Test prepare_for_execution with callable command.""" - - def make_cmd(): - return "echo test" - - cmd = Command(command=make_cmd, name="test") + script = DummyScript(inline=lambda: "echo test") + cmd = make_command(name="test", script=script) cluster_config = {"executor": "local", "containers": {}} - final_cmd, exec_config = cmd.prepare_for_execution(cluster_config) - - assert final_cmd == "echo test" + script_obj, _ = cmd.prepare_for_execution(cluster_config) + assert script_obj.inline == "echo test" def test_command_prepare_for_execution_callable_with_metadata(self): - """Test prepare_for_execution with callable returning tuple.""" - def make_cmd(): - return ("echo metadata", {"num_tasks": 4, "environment": {"VAR": "value"}}) + return ("echo metadata", {"environment": {"VAR": "value"}}) - cmd = Command(command=make_cmd, name="test") + script = DummyScript(inline=make_cmd) + cmd = make_command(name="test", script=script) cluster_config = {"executor": "local", "containers": {}} - final_cmd, exec_config = cmd.prepare_for_execution(cluster_config) + _, exec_config = cmd.prepare_for_execution(cluster_config) - assert final_cmd == "echo metadata" - assert exec_config["num_tasks"] == 4 assert exec_config["environment"]["VAR"] == "value" - def test_command_meta_ref(self): - """Test meta_ref for accessing metadata.""" - cmd = Command(command="echo test", name="server", metadata={"port": 8080, "host": "localhost"}) - - assert cmd.meta_ref("port") == "8080" - assert cmd.meta_ref("host") == "localhost" - - def test_command_meta_ref_missing_key(self): - """Test meta_ref with missing key raises KeyError.""" - cmd = Command(command="echo test", name="test") - - with pytest.raises(KeyError, match="Metadata key 'port' not found"): - cmd.meta_ref("port") - def test_command_hostname_ref_none(self): - """Test hostname_ref returns localhost when het_group_index is None.""" - cmd = Command(command="echo test", name="test") - assert cmd.het_group_index is None - assert cmd.hostname_ref() == "127.0.0.1" - - def test_command_hostname_ref_heterogeneous(self): - """Test hostname_ref returns SLURM variable when het_group_index is set.""" - cmd = Command(command="echo test", name="test") - cmd.het_group_index = 2 - - hostname = cmd.hostname_ref() - assert "$SLURM_JOB_NODELIST_HET_GROUP_2" in hostname - assert "scontrol" in hostname - - def test_command_with_installation_command(self): - """Test Command with installation_command.""" - cmd = Command(command="python script.py", installation_command="pip install package", name="test") - cluster_config = {"executor": "local", "containers": {}} - - final_cmd, _ = cmd.prepare_for_execution(cluster_config) + script = DummyScript() + cmd = make_command(name="test", script=script) - # Installation command should be wrapped around the main command - assert "pip install package" in final_cmd - assert "python script.py" in final_cmd + assert script.hostname_ref() == "127.0.0.1" + assert cmd.get_name() == "test" - def test_command_env_vars_wrapping(self): - """Test that env_vars and working_dir are applied to string commands.""" - cmd = Command( - command="python script.py", - env_vars={"MY_VAR": "value"}, - working_dir="/custom/path", - name="test", - ) + def test_command_hostname_ref_heterogeneous(self): + script = DummyScript() + script.het_group_index = 2 + make_command(name="test", script=script) - # The command should be wrapped with env setup - assert "export MY_VAR=value" in cmd.command - assert "cd /custom/path" in cmd.command + hostname = script.hostname_ref() + assert "${SLURM_MASTER_NODE_HET_GROUP_2" in hostname class TestCommandGroup: @@ -154,8 +111,8 @@ class TestCommandGroup: def test_commandgroup_basic(self): """Test creating a basic CommandGroup.""" - cmd1 = Command(command="echo 1", name="cmd1") - cmd2 = Command(command="echo 2", name="cmd2") + cmd1 = make_command(inline="echo 1", name="cmd1") + cmd2 = make_command(inline="echo 2", name="cmd2") group = CommandGroup(commands=[cmd1, cmd2], name="test_group") @@ -165,7 +122,7 @@ def test_commandgroup_basic(self): def test_commandgroup_with_hardware(self): """Test CommandGroup with HardwareConfig.""" - cmd = Command(command="echo test", name="cmd") + cmd = make_command(inline="echo test", name="cmd") hardware = HardwareConfig(partition="batch", sbatch_kwargs={"time_min": "01:00:00"}, num_gpus=8) group = CommandGroup(commands=[cmd], hardware=hardware, name="gpu_group") @@ -176,7 +133,7 @@ def test_commandgroup_with_hardware(self): def test_commandgroup_with_log_dir(self): """Test CommandGroup with log_dir.""" - cmd = Command(command="echo test", name="cmd") + cmd = make_command(inline="echo test", name="cmd") group = CommandGroup(commands=[cmd], log_dir="/logs/test", name="group") assert group.log_dir == "/logs/test" @@ -187,7 +144,7 @@ class TestPipeline: def test_pipeline_with_single_job(self): """Test Pipeline with single job.""" - cmd = Command(command="echo test", name="cmd") + cmd = make_command(inline="echo test", name="cmd") group = CommandGroup(commands=[cmd], name="group") cluster_config = {"executor": "local", "containers": {}} @@ -204,10 +161,10 @@ def test_pipeline_with_single_job(self): def test_pipeline_with_jobs(self): """Test Pipeline with jobs parameter (full format with dependencies).""" - cmd1 = Command(command="echo 1", name="cmd1") + cmd1 = make_command(inline="echo 1", name="cmd1") group1 = CommandGroup(commands=[cmd1], name="group1", log_dir="/logs") - cmd2 = Command(command="echo 2", name="cmd2") + cmd2 = make_command(inline="echo 2", name="cmd2") group2 = CommandGroup(commands=[cmd2], name="group2", log_dir="/logs") job1 = {"name": "job1", "group": group1} @@ -232,7 +189,7 @@ def test_pipeline_requires_jobs(self): def test_pipeline_with_run_after(self): """Test Pipeline with run_after parameter.""" - cmd = Command(command="echo test", name="cmd") + cmd = make_command(inline="echo test", name="cmd") group = CommandGroup(commands=[cmd], name="group") cluster_config = {"executor": "local", "containers": {}} @@ -248,7 +205,7 @@ def test_pipeline_with_run_after(self): def test_pipeline_with_run_after_list(self): """Test Pipeline with run_after as list.""" - cmd = Command(command="echo test", name="cmd") + cmd = make_command(inline="echo test", name="cmd") group = CommandGroup(commands=[cmd], name="group") cluster_config = {"executor": "local", "containers": {}} @@ -264,7 +221,7 @@ def test_pipeline_with_run_after_list(self): def test_pipeline_cluster_config_passed_directly(self): """Test that cluster_config is passed directly (no more string resolution).""" - cmd = Command(command="echo test", name="cmd") + cmd = make_command(inline="echo test", name="cmd") group = CommandGroup(commands=[cmd], name="group") cluster_config = {"executor": "local", "containers": {}} @@ -299,7 +256,7 @@ def test_pipeline_run_basic(self, mock_run_exp, mock_env_vars, mock_get_exp): mock_get_exp.return_value.__enter__.return_value = mock_exp # Create pipeline - cmd = Command(command="echo test", name="cmd") + cmd = make_command(inline="echo test", name="cmd") group = CommandGroup(commands=[cmd], name="group", log_dir="/logs") pipeline = Pipeline( name="test", cluster_config=mock_config, jobs=[{"name": "job1", "group": group}], skip_hf_home_check=True @@ -329,10 +286,10 @@ def test_pipeline_run_with_dependencies(self, mock_run_exp, mock_env_vars, mock_ mock_get_exp.return_value.__enter__.return_value = mock_exp # Create pipeline with internal dependencies - cmd1 = Command(command="echo 1", name="cmd1") + cmd1 = make_command(inline="echo 1", name="cmd1") group1 = CommandGroup(commands=[cmd1], name="group1", log_dir="/logs") - cmd2 = Command(command="echo 2", name="cmd2") + cmd2 = make_command(inline="echo 2", name="cmd2") group2 = CommandGroup(commands=[cmd2], name="group2", log_dir="/logs") job1 = {"name": "job1", "group": group1, "dependencies": []} @@ -375,7 +332,7 @@ def test_pipeline_hf_home_validation(self, mock_get_executor, mock_is_mounted, m mock_exp.add.return_value = "handle" mock_get_exp.return_value.__enter__.return_value = mock_exp - cmd = Command(command="echo test", name="cmd") + cmd = make_command(inline="echo test", name="cmd") group = CommandGroup(commands=[cmd], name="group", log_dir="/logs") pipeline = Pipeline(name="test", cluster_config=mock_config, jobs=[{"name": "job1", "group": group}]) @@ -391,7 +348,7 @@ def test_pipeline_hf_home_missing(self, mock_env_vars): mock_config = {"executor": "slurm", "containers": {}} mock_env_vars.return_value = {} # No HF_HOME - cmd = Command(command="echo test", name="cmd") + cmd = make_command(inline="echo test", name="cmd") group = CommandGroup(commands=[cmd], name="group", log_dir="/logs") # Should raise in __init__ now, not run() @@ -406,7 +363,7 @@ def test_pipeline_hf_home_not_mounted(self, mock_is_mounted, mock_env_vars): mock_env_vars.return_value = {"HF_HOME": "/hf"} mock_is_mounted.return_value = False - cmd = Command(command="echo test", name="cmd") + cmd = make_command(inline="echo test", name="cmd") group = CommandGroup(commands=[cmd], name="group", log_dir="/logs") # Should raise in __init__ now, not run() @@ -432,8 +389,8 @@ def test_het_group_index_non_heterogeneous(self, mock_env_vars, mock_get_exp): mock_get_exp.return_value.__enter__.return_value = mock_exp # Create single-group job with multiple components - cmd1 = Command(command="echo 1", name="cmd1") - cmd2 = Command(command="echo 2", name="cmd2") + cmd1 = make_command(inline="echo 1", name="cmd1") + cmd2 = make_command(inline="echo 2", name="cmd2") group = CommandGroup(commands=[cmd1, cmd2], name="group", log_dir="/logs") pipeline = Pipeline( @@ -442,10 +399,10 @@ def test_het_group_index_non_heterogeneous(self, mock_env_vars, mock_get_exp): pipeline.run(dry_run=True) # Both commands should have None het_group_index (localhost communication) - assert cmd1.het_group_index is None - assert cmd2.het_group_index is None - assert cmd1.hostname_ref() == "127.0.0.1" - assert cmd2.hostname_ref() == "127.0.0.1" + assert cmd1.script.het_group_index is None + assert cmd2.script.het_group_index is None + assert cmd1.script.hostname_ref() == "127.0.0.1" + assert cmd2.script.hostname_ref() == "127.0.0.1" @patch("nemo_skills.pipeline.utils.declarative.get_exp") @patch("nemo_skills.pipeline.utils.declarative.get_env_variables") @@ -462,10 +419,10 @@ def test_het_group_index_heterogeneous(self, mock_env_vars, mock_get_exp): mock_get_exp.return_value.__enter__.return_value = mock_exp # Create multi-group heterogeneous job - cmd1 = Command(command="echo 1", name="cmd1") + cmd1 = make_command(inline="echo 1", name="cmd1") group1 = CommandGroup(commands=[cmd1], name="group1", log_dir="/logs") - cmd2 = Command(command="echo 2", name="cmd2") + cmd2 = make_command(inline="echo 2", name="cmd2") group2 = CommandGroup(commands=[cmd2], name="group2", log_dir="/logs") jobs = [{"name": "hetjob", "groups": [group1, group2]}] @@ -473,10 +430,10 @@ def test_het_group_index_heterogeneous(self, mock_env_vars, mock_get_exp): pipeline.run(dry_run=True) # Commands should have het_group_index 0 and 1 - assert cmd1.het_group_index == 0 - assert cmd2.het_group_index == 1 - assert "$SLURM_JOB_NODELIST_HET_GROUP_0" in cmd1.hostname_ref() - assert "$SLURM_JOB_NODELIST_HET_GROUP_1" in cmd2.hostname_ref() + assert cmd1.script.het_group_index == 0 + assert cmd2.script.het_group_index == 1 + assert "SLURM_MASTER_NODE_HET_GROUP_0" in cmd1.script.hostname_ref() + assert "SLURM_MASTER_NODE_HET_GROUP_1" in cmd2.script.hostname_ref() @patch("nemo_skills.pipeline.utils.declarative.get_exp") @patch("nemo_skills.pipeline.utils.declarative.get_env_variables") @@ -493,16 +450,16 @@ def test_het_group_index_per_job_not_global(self, mock_env_vars, mock_get_exp): mock_get_exp.return_value.__enter__.return_value = mock_exp # Create two separate heterogeneous jobs - cmd1 = Command(command="echo 1", name="cmd1") + cmd1 = make_command(inline="echo 1", name="cmd1") group1 = CommandGroup(commands=[cmd1], name="group1", log_dir="/logs") - cmd2 = Command(command="echo 2", name="cmd2") + cmd2 = make_command(inline="echo 2", name="cmd2") group2 = CommandGroup(commands=[cmd2], name="group2", log_dir="/logs") - cmd3 = Command(command="echo 3", name="cmd3") + cmd3 = make_command(inline="echo 3", name="cmd3") group3 = CommandGroup(commands=[cmd3], name="group3", log_dir="/logs") - cmd4 = Command(command="echo 4", name="cmd4") + cmd4 = make_command(inline="echo 4", name="cmd4") group4 = CommandGroup(commands=[cmd4], name="group4", log_dir="/logs") jobs = [ @@ -513,10 +470,10 @@ def test_het_group_index_per_job_not_global(self, mock_env_vars, mock_get_exp): pipeline.run(dry_run=True) # Both jobs should have het_group_index starting from 0 - assert cmd1.het_group_index == 0 - assert cmd2.het_group_index == 1 - assert cmd3.het_group_index == 0 # Starts from 0 again! - assert cmd4.het_group_index == 1 + assert cmd1.script.het_group_index == 0 + assert cmd2.script.het_group_index == 1 + assert cmd3.script.het_group_index == 0 # Starts from 0 again! + assert cmd4.script.het_group_index == 1 class TestDependencyResolution: @@ -536,7 +493,7 @@ def test_dependency_none_handling(self, mock_env_vars, mock_get_exp): mock_exp.add.return_value = "handle" mock_get_exp.return_value.__enter__.return_value = mock_exp - cmd = Command(command="echo test", name="cmd") + cmd = make_command(inline="echo test", name="cmd") group = CommandGroup(commands=[cmd], name="group", log_dir="/logs") jobs = [{"name": "job", "group": group, "dependencies": None}] @@ -559,7 +516,7 @@ def test_pipeline_run_after_applies_to_jobs(self, mock_env_vars, mock_get_exp): mock_exp.add.return_value = "handle" mock_get_exp.return_value.__enter__.return_value = mock_exp - cmd = Command(command="echo test", name="cmd") + cmd = make_command(inline="echo test", name="cmd") group = CommandGroup(commands=[cmd], name="group", log_dir="/logs") pipeline = Pipeline( @@ -589,7 +546,7 @@ def test_pipeline_job_missing_group_or_groups(self): def test_commandgroup_missing_log_dir(self): """Test that CommandGroup without log_dir raises error during execution.""" mock_config = {"executor": "none", "containers": {}} - cmd = Command(command="echo test", name="cmd") + cmd = make_command(inline="echo test", name="cmd") group = CommandGroup(commands=[cmd], name="group") # No log_dir pipeline = Pipeline(name="test", cluster_config=mock_config, jobs=[{"name": "job1", "group": group}]) @@ -626,14 +583,14 @@ def test_multiple_internal_dependencies(self): } # Job 1 and Job 2: independent - cmd1 = Command(command="echo job1", name="job1") + cmd1 = make_command(inline="echo job1", name="job1") group1 = CommandGroup(commands=[cmd1], name="group1", log_dir="/tmp/logs") - cmd2 = Command(command="echo job2", name="job2") + cmd2 = make_command(inline="echo job2", name="job2") group2 = CommandGroup(commands=[cmd2], name="group2", log_dir="/tmp/logs") # Job 3: depends on both job1 and job2 - cmd3 = Command(command="echo job3", name="job3") + cmd3 = make_command(inline="echo job3", name="job3") group3 = CommandGroup(commands=[cmd3], name="group3", log_dir="/tmp/logs") job1_spec = {"name": "job1", "group": group1} @@ -715,11 +672,11 @@ def mock_get_executor(**kwargs): } # Job 1: depends on external experiment - cmd1 = Command(command="echo job1", name="job1") + cmd1 = make_command(inline="echo job1", name="job1") group1 = CommandGroup(commands=[cmd1], name="group1", log_dir="/tmp/logs") # Job 2: depends on job1 (internal) AND external experiment - cmd2 = Command(command="echo job2", name="job2") + cmd2 = make_command(inline="echo job2", name="job2") group2 = CommandGroup(commands=[cmd2], name="group2", log_dir="/tmp/logs") job1_spec = { @@ -931,35 +888,38 @@ def capture_env_update(cluster_config, updates): # Debug: print what we captured print(f"Captured env updates: {env_updates_captured}") - # Find the client and sandbox environment updates - client_env = None - sandbox_env = None + # Verify both sandbox and client environment variables are captured + assert len(env_updates_captured) >= 2, ( + f"Expected at least 2 environment updates (sandbox + client), got {len(env_updates_captured)}: {env_updates_captured}" + ) + # Find the sandbox and client environment updates + sandbox_env = None + client_env = None for env_update in env_updates_captured: + if "LISTEN_PORT" in env_update and "NGINX_PORT" in env_update: + sandbox_env = env_update if "NEMO_SKILLS_SANDBOX_PORT" in env_update: client_env = env_update - elif "LISTEN_PORT" in env_update and "NGINX_PORT" in env_update: - sandbox_env = env_update - # Verify client got NEMO_SKILLS_SANDBOX_PORT (old behavior: exp.py line 493) - # This is the key fix - ensuring sandbox port is passed to client - assert client_env is not None, ( - f"Client environment update not found. Captured updates: {env_updates_captured}\n" - f"This means NEMO_SKILLS_SANDBOX_PORT was not set for the client command, " - f"so the Sandbox class cannot connect to the sandbox server." + # Verify sandbox got LISTEN_PORT and NGINX_PORT + assert sandbox_env is not None, ( + f"LISTEN_PORT/NGINX_PORT not set for sandbox command: {env_updates_captured}" ) - assert "NEMO_SKILLS_SANDBOX_PORT" in client_env, ( - "NEMO_SKILLS_SANDBOX_PORT not set for client command" + assert sandbox_env["LISTEN_PORT"] == sandbox_env["NGINX_PORT"], ( + f"LISTEN_PORT and NGINX_PORT should match: {sandbox_env}" ) - # Verify sandbox got its environment vars (old behavior: exp.py lines 525-538) - assert sandbox_env is not None, ( - f"Sandbox environment update not found. Captured: {env_updates_captured}" + # Verify client got NEMO_SKILLS_SANDBOX_PORT + assert client_env is not None, ( + f"NEMO_SKILLS_SANDBOX_PORT not set for client command: {env_updates_captured}" ) - assert "LISTEN_PORT" in sandbox_env, "LISTEN_PORT not set for sandbox" - assert "NGINX_PORT" in sandbox_env, "NGINX_PORT not set for sandbox" - # This test verifies the fix works end-to-end through the actual generate() function + # Verify the ports match between sandbox and client + assert client_env["NEMO_SKILLS_SANDBOX_PORT"] == sandbox_env["LISTEN_PORT"], ( + f"Sandbox port mismatch: client has {client_env['NEMO_SKILLS_SANDBOX_PORT']}, " + f"sandbox has {sandbox_env['LISTEN_PORT']}" + ) if __name__ == "__main__": diff --git a/tests/test_generation.py b/tests/test_generation.py index b69b526a0e..2693d62241 100644 --- a/tests/test_generation.py +++ b/tests/test_generation.py @@ -16,12 +16,12 @@ # running most things through subprocess since that's how it's usually used import subprocess -from unittest.mock import MagicMock import pytest from nemo_skills.evaluation.metrics import ComputeMetrics -from nemo_skills.pipeline.generate import _create_commandgroup_from_config +from nemo_skills.pipeline.generate import _create_job_unified +from nemo_skills.pipeline.utils.scripts import ServerScript def test_eval_gsm8k_api(tmp_path): @@ -153,36 +153,42 @@ def test_generate_openai_format(tmp_path, format): assert len(data[1]["generation"]) > 0 -def test_server_metadata_from_num_tasks(): +def test_server_metadata_from_num_tasks(tmp_path): """Test that metadata dict is properly created from server command returning (cmd, num_tasks).""" - mock_server_fn = MagicMock(return_value=("python server.py", 4)) cluster_config = { - "containers": {"vllm": "nvcr.io/nvidia/nemo:vllm", "nemo-skills": "nvcr.io/nvidia/nemo:skills"}, - "executor": "slurm", + "containers": { + "vllm": "apitest/vllm", + "nemo-skills": "apitest/nemo-skills", + "sandbox": "apitest/sandbox", + }, + "executor": "none", } server_config = { "server_type": "vllm", "num_gpus": 8, "num_nodes": 1, - "model_path": "/models/test", + "model_path": str(tmp_path / "model"), "server_port": 5000, + "server_args": "", } + generation_params = {"output_dir": "/tmp/out"} - cmd_group = _create_commandgroup_from_config( - generation_cmd="python generate.py", - server_config=server_config, - with_sandbox=False, - sandbox_port=None, + groups = _create_job_unified( + models=[server_config["model_path"]], + server_configs=[server_config], + generation_params=generation_params, cluster_config=cluster_config, installation_command=None, - get_server_command_fn=mock_server_fn, + with_sandbox=False, partition=None, keep_mounts_for_sandbox=False, task_name="test-task", log_dir="/tmp/logs", ) - server_cmd = cmd_group.commands[0] - assert isinstance(server_cmd.metadata, dict) - assert server_cmd.metadata["num_tasks"] == 4 - assert server_cmd.metadata["gpus"] == 8 + server_cmd = groups[0].commands[0] + assert isinstance(server_cmd.script, ServerScript) + assert server_cmd.script.num_tasks >= 1 + assert server_cmd.script.num_gpus == server_config["num_gpus"] + assert groups[0].hardware.num_gpus == server_config["num_gpus"] + assert groups[0].hardware.num_tasks == server_cmd.script.num_tasks diff --git a/tests/test_nemo_evaluator_pipeline.py b/tests/test_nemo_evaluator_pipeline.py index 22ac250882..0f333ab748 100644 --- a/tests/test_nemo_evaluator_pipeline.py +++ b/tests/test_nemo_evaluator_pipeline.py @@ -17,8 +17,14 @@ import pytest -from nemo_skills.pipeline.nemo_evaluator import nemo_evaluator as nemo_evaluator_fn +from nemo_skills.pipeline.nemo_evaluator import ( + EvaluatorClientScript, +) +from nemo_skills.pipeline.nemo_evaluator import ( + nemo_evaluator as nemo_evaluator_fn, +) from nemo_skills.pipeline.utils.declarative import Command, CommandGroup +from nemo_skills.pipeline.utils.scripts import ServerScript @pytest.fixture @@ -131,9 +137,8 @@ def test_no_servers_external_urls( # Verify client command client_cmd = group.commands[0] assert isinstance(client_cmd, Command) - assert "evaluator-test-0" in client_cmd.name - assert client_cmd.gpus is None # No GPUs when no hosted servers - assert client_cmd.nodes == 1 + assert client_cmd.name.startswith("evaluator-test-client-0") + assert isinstance(client_cmd.script, EvaluatorClientScript) # Verify hardware config assert group.hardware is not None @@ -181,16 +186,17 @@ def test_main_server_hosted( server_cmd = group.commands[0] assert isinstance(server_cmd, Command) assert "server" in server_cmd.name - assert server_cmd.gpus == 8 - assert server_cmd.nodes == 1 - assert "port" in server_cmd.metadata - assert server_cmd.metadata["log_prefix"] == "server" + assert isinstance(server_cmd.script, ServerScript) + assert server_cmd.script.num_gpus == 8 + assert server_cmd.script.log_prefix == "server" + assert server_cmd.script.port is not None # Verify client command client_cmd = group.commands[1] assert isinstance(client_cmd, Command) assert "client" in client_cmd.name - assert callable(client_cmd.command) # Should be lambda for cross-component refs + assert isinstance(client_cmd.script, EvaluatorClientScript) + assert callable(client_cmd.script.inline) # Should be lambda for cross-component refs # Verify hardware config (should use server GPUs) assert group.hardware.num_gpus == 8 @@ -235,14 +241,16 @@ def test_judge_server_hosted( judge_cmd = group.commands[0] assert isinstance(judge_cmd, Command) assert "judge-server" in judge_cmd.name - assert judge_cmd.gpus == 32 - assert judge_cmd.metadata["log_prefix"] == "judge-server" + assert isinstance(judge_cmd.script, ServerScript) + assert judge_cmd.script.num_gpus == 32 + assert judge_cmd.script.log_prefix == "judge-server" # Verify client command client_cmd = group.commands[1] assert isinstance(client_cmd, Command) assert "client" in client_cmd.name - assert callable(client_cmd.command) # Should be lambda for cross-component refs + assert isinstance(client_cmd.script, EvaluatorClientScript) + assert callable(client_cmd.script.inline) # Should be lambda for cross-component refs # Verify hardware config (should use judge server GPUs) assert group.hardware.num_gpus == 32 @@ -300,19 +308,22 @@ def test_both_servers_hosted_separate_groups( server_cmd = server_group.commands[0] assert isinstance(server_cmd, Command) assert "server" in server_cmd.name - assert server_cmd.gpus == 8 + assert isinstance(server_cmd.script, ServerScript) + assert server_cmd.script.num_gpus == 8 # Verify client command in first group client_cmd = server_group.commands[1] assert isinstance(client_cmd, Command) assert "client" in client_cmd.name - assert callable(client_cmd.command) # Lambda for cross-component refs + assert isinstance(client_cmd.script, EvaluatorClientScript) + assert callable(client_cmd.script.inline) # Lambda for cross-component refs # Verify judge server command in second group judge_cmd = judge_group.commands[0] assert isinstance(judge_cmd, Command) assert "judge-server" in judge_cmd.name - assert judge_cmd.gpus == 32 + assert isinstance(judge_cmd.script, ServerScript) + assert judge_cmd.script.num_gpus == 32 @patch("nemo_skills.pipeline.nemo_evaluator.Pipeline") From 2dcfe41bfc32dd7cd135bdf7f40c048c2e5186f9 Mon Sep 17 00:00:00 2001 From: Jocelyn Date: Wed, 7 Jan 2026 09:11:44 -0800 Subject: [PATCH 74/88] BIRD Benchmark (Text-to-SQL) (#1132) Signed-off-by: Jocelyn Huang Signed-off-by: Cheng-Ping Hsieh --- dockerfiles/Dockerfile.nemo-skills | 2 +- docs/evaluation/code.md | 40 +++++ docs/index.md | 2 +- nemo_skills/dataset/birdbench/__init__.py | 27 ++++ nemo_skills/dataset/birdbench/prepare.py | 126 +++++++++++++++ nemo_skills/evaluation/evaluator/__init__.py | 2 + nemo_skills/evaluation/evaluator/bird.py | 149 ++++++++++++++++++ .../evaluation/metrics/bird_metrics.py | 77 +++++++++ nemo_skills/evaluation/metrics/map_metrics.py | 2 + .../prompt/config/generic/text_to_sql.yaml | 15 ++ requirements/main.txt | 1 + 11 files changed, 441 insertions(+), 2 deletions(-) create mode 100644 nemo_skills/dataset/birdbench/__init__.py create mode 100644 nemo_skills/dataset/birdbench/prepare.py create mode 100644 nemo_skills/evaluation/evaluator/bird.py create mode 100644 nemo_skills/evaluation/metrics/bird_metrics.py create mode 100644 nemo_skills/prompt/config/generic/text_to_sql.yaml diff --git a/dockerfiles/Dockerfile.nemo-skills b/dockerfiles/Dockerfile.nemo-skills index 2474d850b3..3d35f10dc8 100644 --- a/dockerfiles/Dockerfile.nemo-skills +++ b/dockerfiles/Dockerfile.nemo-skills @@ -59,7 +59,7 @@ COPY pyproject.toml README.md /opt/NeMo-Skills/ COPY requirements /opt/NeMo-Skills/requirements/ # installing sdp in container only RUN pip install git+https://github.com/NVIDIA/NeMo-speech-data-processor@29b9b1ec0ceaf3ffa441c1d01297371b3f8e11d2 -ARG CACHEBUST=1 +ARG CACHEBUST=2 RUN pip install --no-cache-dir -r /opt/NeMo-Skills/requirements/main.txt # Fix http mismatch between lepton and dggs by manually downloading dggs here RUN pip install ddgs diff --git a/docs/evaluation/code.md b/docs/evaluation/code.md index 68decc1645..5be1b81ea1 100644 --- a/docs/evaluation/code.md +++ b/docs/evaluation/code.md @@ -316,6 +316,46 @@ Due to variance between runs, you can automatically repeat the evaluation and av --benchmarks=livecodebench:3 ``` +### BIRD + +The [BIRD benchmark](https://bird-bench.github.io/) is currently the only text-to-SQL benchmark that is supported. Evaluation is based on the SQL evaluation accuracy calculated in [this file](https://github.com/AlibabaResearch/DAMO-ConvAI/blob/main/bird/llm/src/evaluation.py) provided in the BIRD GitHub repository. + +#### Data Preparation + + +First, the data must be downloaded and prepared, which you can do by running: +```bash +ns prepare_data birdbench --cluster= --data_dir= +``` + +This will download and unpack a file into `/birdbench/dev_20240627`, which contains the BIRD dev manifest, table information, and database schemas. +The script will also process the original manifest into `/birdbench/dev.jsonl`, which will be the input for evaluation. +`` should be a path to the mount point where you want this data to be stored. + +See [the "Using data on cluster" documentation](./index.md#Using-data-on-cluster) for more information. + +#### Running the Evaluation + +The following command runs an evaluation of [Qwen3-8B](https://huggingface.co/Qwen/Qwen3-8B) on a Slurm cluster. + +```bash +ns eval \ + --cluster= \ + --server_type='sglang' \ + --server_gpus=8 \ + --model=Qwen/Qwen3-8B \ + --benchmarks=birdbench \ + --data_dir= \ + --output_dir= \ + ++inference.tokens_to_generate=10000 \ + ++inference.temperature=0.6 \ + ++inference.top_p=0.95 \ + ++inference.top_k=20 \ + ++max_concurrent_requests=1024 \ +``` +You should specify: ``, which should match your cluster config name; ``, which should be the location where your dataset is mounted from the cluster; and ``. +The former two arguments should match what you used in `prepare_data`. + ### livecodebench-cpp - Benchmark is defined in [`nemo_skills/dataset/livecodebench-cpp/__init__.py`](https://github.com/NVIDIA-NeMo/Skills/blob/main/nemo_skills/dataset/livecodebench-cpp/__init__.py) diff --git a/docs/index.md b/docs/index.md index a17dce8f81..32f216415b 100644 --- a/docs/index.md +++ b/docs/index.md @@ -16,7 +16,7 @@ Here are some of the features we support: - Evaluate your models on many popular benchmarks. - [**Math (natural language**)](./evaluation/natural-math.md): e.g. [aime24](./evaluation/natural-math.md#aime24), [aime25](./evaluation/natural-math.md#aime25), [hmmt_feb25](./evaluation/natural-math.md#hmmt_feb25) - [**Math (formal language)**](./evaluation/formal-math.md): e.g. [minif2f](./evaluation/formal-math.md#minif2f), [proofnet](./evaluation/formal-math.md#proofnet), [putnam-bench](./evaluation/formal-math.md#putnam-bench) - - [**Code**](./evaluation/code.md): e.g. [swe-bench](./evaluation/code.md#swe-bench), [livecodebench](./evaluation/code.md#livecodebench) + - [**Code**](./evaluation/code.md): e.g. [swe-bench](./evaluation/code.md#swe-bench), [livecodebench](./evaluation/code.md#livecodebench), [bird](./evaluation/code.md#BIRD) - [**Scientific knowledge**](./evaluation/scientific-knowledge.md): e.g., [hle](./evaluation/scientific-knowledge.md#hle), [scicode](./evaluation/scientific-knowledge.md#scicode), [gpqa](./evaluation/scientific-knowledge.md#gpqa) - [**Instruction following**](./evaluation/instruction-following.md): e.g. [ifbench](./evaluation/instruction-following.md#ifbench), [ifeval](./evaluation/instruction-following.md#ifeval) - [**Long-context**](./evaluation/long-context.md): e.g. [ruler](./evaluation/long-context.md#ruler), [mrcr](./evaluation/long-context.md#mrcr) diff --git a/nemo_skills/dataset/birdbench/__init__.py b/nemo_skills/dataset/birdbench/__init__.py new file mode 100644 index 0000000000..55505365cc --- /dev/null +++ b/nemo_skills/dataset/birdbench/__init__.py @@ -0,0 +1,27 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# settings that define how evaluation should be done by default (all can be changed from cmdline) +DATASET_GROUP = "code" +METRICS_TYPE = "bird" +EVAL_SPLIT = "dev" +GENERATION_ARGS = ( + "++prompt_config=generic/text_to_sql " + "++eval_type=bird " + "++inference.tokens_to_generate=10000 " + "++inference.temperature=0.6 " + "++inference.top_p=0.95 " + "++inference.top_k=20 " + "++max_concurrent_requests=1024" +) diff --git a/nemo_skills/dataset/birdbench/prepare.py b/nemo_skills/dataset/birdbench/prepare.py new file mode 100644 index 0000000000..07c195553b --- /dev/null +++ b/nemo_skills/dataset/birdbench/prepare.py @@ -0,0 +1,126 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import glob +import json +import os +import re +import sqlite3 +import zipfile +from pathlib import Path + +import wget + + +def download_data(data_dir): + # Download zip directly (HF Dataset is missing SQL files and table info) + print("Downloading and extracting data file...") + url = "https://bird-bench.oss-cn-beijing.aliyuncs.com/dev.zip" + filename = wget.download(url, out=data_dir) + with zipfile.ZipFile(Path(data_dir, filename), "r") as f_in: + f_in.extractall(data_dir) + + # Expand tables zipfiles + print("Extracting databases...") + dev_dir = Path(data_dir, "dev_20240627/") + dbs_zipfile = Path(dev_dir, "dev_databases.zip") + with zipfile.ZipFile(dbs_zipfile, "r") as f_dbs: + f_dbs.extractall(dev_dir) + + print("Extracted all data!") + return dev_dir + + +def read_tables_file(base_dir): + """ + Gets each db's information by using sqlite3 to get a table dump. + """ + tables_info = {} + all_db_dirs = glob.glob("*", root_dir=os.path.join(base_dir, "dev_databases")) + + for db_dir in all_db_dirs: + print(f"Reading database info from: {db_dir}") + table_info = "" + + # Grab the db's sqlite file & read the dump + full_db_dir = os.path.join(base_dir, "dev_databases", db_dir) + sqlite_file = os.path.join(full_db_dir, db_dir + ".sqlite") + assert os.path.exists(sqlite_file) + + with sqlite3.connect(os.path.join(full_db_dir, db_dir + ".sqlite")) as con: + con.text_factory = lambda b: b.decode(errors="ignore") + for line in con.iterdump(): + if line[:6] == "INSERT": + line = line.replace("\n", " ") + line = re.sub(" +", " ", line) + table_info += line + "\n" + + # Time to truncate any long INSERT chains (allow 10 max at once) + insert_chain = r"((INSERT.*$\n){10})((INSERT.*\n)*)" + table_info = re.sub(insert_chain, r"\1\n...\n", table_info, flags=re.MULTILINE) + + # Also get rid of any INSERT INTO * VALUES (...) <- lots of entries (>10) + many_values = r"(?:VALUES )(((\([^)]*)\)[,;]\s*)){10}(.*)(?:;)" + table_info = re.sub(many_values, r"...", table_info, flags=re.MULTILINE) + + tables_info[db_dir] = table_info + + return tables_info + + +def format_entries(file_path, tables_info, out_file): + """ + Combines the raw BIRD data entries with corresponding table info and + ground truth solution to form dev manifest + """ + with open(out_file, "w") as f_out: + with open(file_path, "r") as f_in: + entries = json.load(f_in) + + for i, entry in enumerate(entries): + new_entry = { + "question": entry["question"], + "gt_sql": entry["SQL"], + "sql_context": tables_info[entry["db_id"]], + "difficulty": entry["difficulty"], + "db_id": entry["db_id"], + "id": i, + } + + f_out.write(json.dumps(new_entry)) + f_out.write("\n") + + +def main(): + data_dir = str(Path(__file__).absolute().parent) + + dev_dir = download_data(data_dir) + # If already downloaded: dev_dir = Path(data_dir, "dev_20240627/") + print(f"\nData downloaded to: {dev_dir}") + + print("Starting processing...") + + # First read tables data + tables_info = read_tables_file(dev_dir) + print("Finished reading tables.") + + # Naming the input and output files the nearly same thing is likely + # confusing, but .jsonl is the expected format so we'll just + # keep the result in the upper-level directory, outside of dev_dir. + format_entries(Path(dev_dir, "dev.json"), tables_info, Path(data_dir, "dev.jsonl")) + print("Finished formatting entries. All done!") + + +if __name__ == "__main__": + main() diff --git a/nemo_skills/evaluation/evaluator/__init__.py b/nemo_skills/evaluation/evaluator/__init__.py index 0c2aa1e3b0..03b38ec3c1 100644 --- a/nemo_skills/evaluation/evaluator/__init__.py +++ b/nemo_skills/evaluation/evaluator/__init__.py @@ -18,6 +18,7 @@ from nemo_skills.evaluation.evaluator.audio import AudioEvaluator from nemo_skills.evaluation.evaluator.base import BaseEvaluator from nemo_skills.evaluation.evaluator.bfcl import eval_bfcl +from nemo_skills.evaluation.evaluator.bird import BirdEvaluator from nemo_skills.evaluation.evaluator.code import ( CodeExecEvaluator, eval_bigcodebench, @@ -68,6 +69,7 @@ "ioi": IOIEvaluator, "icpc": ICPCEvaluator, "audio": AudioEvaluator, + "bird": BirdEvaluator, } # Validation: Ensure no overlap between class and function maps diff --git a/nemo_skills/evaluation/evaluator/bird.py b/nemo_skills/evaluation/evaluator/bird.py new file mode 100644 index 0000000000..df51c4d709 --- /dev/null +++ b/nemo_skills/evaluation/evaluator/bird.py @@ -0,0 +1,149 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import re +import sqlite3 +from pathlib import Path + +from func_timeout import FunctionTimedOut, func_timeout + +from nemo_skills.evaluation.evaluator.base import BaseEvaluator, BaseEvaluatorConfig +from nemo_skills.utils import nested_dataclass + +# The following code was modified from: +# https://github.com/AlibabaResearch/DAMO-ConvAI/blob/main/bird/llm/src/evaluation.py + +# Original license as follows: + +# MIT License +# +# Copyright (c) 2022 Alibaba Research +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + + +def execute_sql(predicted_sql, ground_truth, db_path): + # Connect to the database + with sqlite3.connect(db_path) as conn: + cursor = conn.cursor() + cursor.execute(predicted_sql) + predicted_res = cursor.fetchall() + cursor.execute(ground_truth) + ground_truth_res = cursor.fetchall() + res = 0 + if set(predicted_res) == set(ground_truth_res): + res = 1 + return res + + +# ===== End of copied and modified code. ===== + + +@nested_dataclass(kw_only=True) +class BirdEvaluatorConfig(BaseEvaluatorConfig): + timeout: int = 30 + + # Answer format can be "BOXED", "CODEBLOCK", or "USE_REGEX", the last of + # which uses the given regex in the extraction_regex arg. + answer_format: str = "CODEBLOCK" + extraction_regex: str | None = None + regex_dotall: bool = False + + +class BirdEvaluator(BaseEvaluator): + def __init__(self, config: dict, num_parallel_requests=10): + super().__init__(config, num_parallel_requests) + self.eval_config = BirdEvaluatorConfig(**self.config) + + self.db_path = Path(self.eval_config.data_dir, "birdbench", "dev_20240627", "dev_databases") + + def _extract_answer(self, text): + """Uses the specified format/regex to get the answer from the output text.""" + regex = "" + dotall = False + answer_format = self.eval_config.answer_format + + if answer_format == "CODEBLOCK": + regex = r"(?:```sql)(.*?[a-zA-Z].*?)(?:```)" + dotall = True + elif answer_format == "BOXED": + regex = r"(?:boxed\{\{)(.*?[a-zA-Z].*?)(?:\}\})" + dotall = True + elif answer_format == "USE_REGEX": + regex = self.eval_config.extraction_regex + dotall = self.eval_config.regex_dotall + + if not regex: + logging.error( + "Answer format underspecified for BIRD evaluation; should be one of " + + "{CODEBLOCK, BOXED, USE_REGEX (provide extraction_regex)}.\n" + + f"Got {answer_format} instead." + ) + + # Use regex to extract answer from text + if dotall: + code_matches = re.findall(regex, text, flags=re.DOTALL) + else: + code_matches = re.findall(regex, text) + + if not code_matches: + return "SELECT 1" # No-op filler + + # Remove comments first + ans = re.sub(r"--.*?$|/\*.*?\*/", "", code_matches[-1], flags=re.DOTALL) # Use last match + # Collapse whitespace + ans = re.sub(r"\s+", " ", ans) + # Remove miscellaneous headers that snuck in + ans = re.sub(r"^\*\*.*\*\*", "", ans).strip() + + return ans + + async def eval_single(self, data_point: dict): + i = data_point["id"] + db_id = data_point["db_id"] + + # Retrieve pred and gt + predicted_sql = self._extract_answer(data_point["generation"]) + ground_truth = data_point["gt_sql"] + db_place = str(Path(self.db_path, db_id, db_id + ".sqlite")) + + try: + # Wait for result with timeout as set + res = func_timeout(self.eval_config.timeout, execute_sql, args=(predicted_sql, ground_truth, db_place)) + except FunctionTimedOut: + logging.info(f"SQL execution timed out for entry {i}") + res = 0 + except Exception as e: + logging.info(f"SQL execution failed for entry {i}:\n{e}") + res = 0 + + data_point["res"] = res + return data_point diff --git a/nemo_skills/evaluation/metrics/bird_metrics.py b/nemo_skills/evaluation/metrics/bird_metrics.py new file mode 100644 index 0000000000..24ed32a7c4 --- /dev/null +++ b/nemo_skills/evaluation/metrics/bird_metrics.py @@ -0,0 +1,77 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_skills.evaluation.metrics.base import BaseMetrics, as_float + + +class BirdMetrics(BaseMetrics): + """Metrics for BIRD text-to-SQL evaluation.""" + + def __init__(self): + super().__init__() + self.reset() + + def reset(self): + super().reset() + self.n = 0 + self.correct = 0 + self.simple_results = [] + self.moderate_results = [] + self.challenging_results = [] + + def update(self, predictions): + self.n += len(predictions) + + for pred in predictions: + # Each should be a 0 or 1 value + if pred["difficulty"] == "simple": + self.simple_results.append(pred["res"]) + elif pred["difficulty"] == "moderate": + self.moderate_results.append(pred["res"]) + elif pred["difficulty"] == "challenging": + self.challenging_results.append(pred["res"]) + + self.correct += pred["res"] + + def get_metrics(self): + sr = self.simple_results + mr = self.moderate_results + cr = self.challenging_results + + simple_acc = sum(sr) / len(sr) if sr else 0 + moderate_acc = sum(mr) / len(mr) if mr else 0 + challenging_acc = sum(cr) / len(cr) if cr else 0 + + acc = self.correct / self.n if self.n else 0 + + metrics_dict = {} + metrics_dict["total"] = { + "simple_acc": simple_acc * 100, + "moderate_acc": moderate_acc * 100, + "challenging_acc": challenging_acc * 100, + "total_acc": acc * 100, + } + return metrics_dict + + def evaluations_to_print(self): + return ["total"] + + def metrics_to_print(self): + metrics_to_print = { + "simple_acc": as_float, + "moderate_acc": as_float, + "challenging_acc": as_float, + "total_acc": as_float, + } + return metrics_to_print diff --git a/nemo_skills/evaluation/metrics/map_metrics.py b/nemo_skills/evaluation/metrics/map_metrics.py index 1c66a95bd7..a78ebba81c 100644 --- a/nemo_skills/evaluation/metrics/map_metrics.py +++ b/nemo_skills/evaluation/metrics/map_metrics.py @@ -21,6 +21,7 @@ from nemo_skills.evaluation.metrics.arena_metrics import ArenaMetrics from nemo_skills.evaluation.metrics.audio_metrics import AudioMetrics from nemo_skills.evaluation.metrics.bfcl_metrics import BFCLMetrics +from nemo_skills.evaluation.metrics.bird_metrics import BirdMetrics from nemo_skills.evaluation.metrics.code_metrics import ( BigCodeBenchMetrics, EvalPlusMetrics, @@ -50,6 +51,7 @@ "arena": ArenaMetrics, "audio": AudioMetrics, "bfcl": BFCLMetrics, + "bird": BirdMetrics, "evalplus": EvalPlusMetrics, "if": IFMetrics, "ioi": IOIMetrics, diff --git a/nemo_skills/prompt/config/generic/text_to_sql.yaml b/nemo_skills/prompt/config/generic/text_to_sql.yaml new file mode 100644 index 0000000000..5a383c5276 --- /dev/null +++ b/nemo_skills/prompt/config/generic/text_to_sql.yaml @@ -0,0 +1,15 @@ +# Prompt used for text-to-SQL + +system: Please reason step by step, and put your final answer within the tags "```sql" and "```". + +user: |- + ### Question + {question} + + The following is a SQL dump that describes the database and the tables in it. + {sql_context} + + Convert the question above to a SQL query for the database given. Use the answer tags given. + If there is more than one set of tags, the last one will be taken as your final answer. + + ### Answer: diff --git a/requirements/main.txt b/requirements/main.txt index f140a97b50..2ebd4d3b4b 100644 --- a/requirements/main.txt +++ b/requirements/main.txt @@ -22,6 +22,7 @@ fire # needed local code execution server for persistent sessions flask +func-timeout # Needed for BIRD benchmark gradio httpx huggingface_hub From 86195dfe32cd835ea4df091bc08eb443ffe79efc Mon Sep 17 00:00:00 2001 From: Minho Ryu Date: Wed, 7 Jan 2026 09:59:37 -0800 Subject: [PATCH 75/88] o3-mini-20250131 -> o3-mini-2025-01-31 (#1149) Signed-off-by: bzantium Co-authored-by: Igor Gitman Signed-off-by: Cheng-Ping Hsieh --- docs/evaluation/natural-math.md | 2 +- docs/tutorials/posts/llama-nemotron-super-v1.5-evals.md | 6 +++--- docs/tutorials/posts/nemotron-nano-v2-evals.md | 4 ++-- nemo_skills/dataset/hle/__init__.py | 4 ++-- nemo_skills/dataset/simpleqa/__init__.py | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/evaluation/natural-math.md b/docs/evaluation/natural-math.md index 9d0aac2d4e..c7a7cf5f58 100644 --- a/docs/evaluation/natural-math.md +++ b/docs/evaluation/natural-math.md @@ -104,7 +104,7 @@ For example, in a benchmark's `__init__.py` file, you can add default LLM-as-jud ```bash JUDGE_PIPELINE_ARGS = { - "model": "o3-mini-20250131", + "model": "o3-mini-2025-01-31", "server_type": "openai", "server_address": "https://api.openai.com/v1", } diff --git a/docs/tutorials/posts/llama-nemotron-super-v1.5-evals.md b/docs/tutorials/posts/llama-nemotron-super-v1.5-evals.md index 536b97a251..b447c61f83 100644 --- a/docs/tutorials/posts/llama-nemotron-super-v1.5-evals.md +++ b/docs/tutorials/posts/llama-nemotron-super-v1.5-evals.md @@ -158,7 +158,7 @@ ns eval \ #### Command for HLE Eval (Reasoning on) -For HLE, because symbolic comparison is not sufficient to determine the correctness of the output, we use the recommended `o3-mini-20250131` model as the judge. Note that this model is the default in Nemo-Skills, and we have just added this argument for illustration purposes. To evaluate for the [Artificial Analysis Index (AAI) setting, please use the gpt-4o-20240806 model as the judge](https://artificialanalysis.ai/methodology/intelligence-benchmarking#intelligence-index-evaluation-suite-overview){target="_blank"}. +For HLE, because symbolic comparison is not sufficient to determine the correctness of the output, we use the recommended `o3-mini-2025-01-31` model as the judge. Note that this model is the default in Nemo-Skills, and we have just added this argument for illustration purposes. To evaluate for the [Artificial Analysis Index (AAI) setting, please use the gpt-4o-20240806 model as the judge](https://artificialanalysis.ai/methodology/intelligence-benchmarking#intelligence-index-evaluation-suite-overview){target="_blank"}. Note that using any of the OpenAI hosted models requires `OPENAI_API_KEY`. Alternatively, a self-hosted judge model can also be used for judgement. For example, `--judge_model="/workspace/Llama-3_3-Nemotron-Super-49B-v1_5"` in tandem with `--judge_server_type="vllm" --judge_server_gpus 2` will use the `Llama-3_3-Nemotron-Super-49B-v1_5` itself as a judge. @@ -171,7 +171,7 @@ ns eval \ --output_dir=/workspace/llama_nemotron_49b_1_5/ \ --benchmarks=hle:16 \ --server_gpus=2 \ - --judge_model="o3-mini-20250131" \ + --judge_model="o3-mini-2025-01-31" \ --extra_judge_args="++inference.tokens_to_generate=4096 ++max_concurrent_requests=8" \ ++parse_reasoning=True \ ++inference.tokens_to_generate=65536 \ @@ -434,7 +434,7 @@ ns eval \ --output_dir=/workspace/llama_nemotron_49b_1_5_reasoning_off/ \ --benchmarks=hle:16 \ --server_gpus=2 \ - --judge_model="o3-mini-20250131" \ + --judge_model="o3-mini-2025-01-31" \ --extra_judge_args="++inference.tokens_to_generate=4096 ++max_concurrent_requests=8" \ ++inference.tokens_to_generate=65536 \ ++inference.temperature=0.0 \ diff --git a/docs/tutorials/posts/nemotron-nano-v2-evals.md b/docs/tutorials/posts/nemotron-nano-v2-evals.md index 45fa9bb1c7..0e64342a8e 100644 --- a/docs/tutorials/posts/nemotron-nano-v2-evals.md +++ b/docs/tutorials/posts/nemotron-nano-v2-evals.md @@ -188,7 +188,7 @@ ns eval \ #### Command for HLE Eval -For HLE, because symbolic comparison is not sufficient to determine the correctness of the output, we use the recommended `o3-mini-20250131` model as the judge. Note that this model is the default in Nemo-Skills, and we have just added this argument for illustration purposes. To evaluate for the [Artificial Analysis Index (AAI) setting, please use the gpt-4o-20240806 model as the judge](https://artificialanalysis.ai/methodology/intelligence-benchmarking#intelligence-index-evaluation-suite-overview){target="_blank"}. +For HLE, because symbolic comparison is not sufficient to determine the correctness of the output, we use the recommended `o3-mini-2025-01-31` model as the judge. Note that this model is the default in Nemo-Skills, and we have just added this argument for illustration purposes. To evaluate for the [Artificial Analysis Index (AAI) setting, please use the gpt-4o-20240806 model as the judge](https://artificialanalysis.ai/methodology/intelligence-benchmarking#intelligence-index-evaluation-suite-overview){target="_blank"}. Note that using any of the OpenAI hosted models requires `OPENAI_API_KEY`. Alternatively, a self-hosted judge model can also be used for judgement. For example, `--judge_model="/workspace/NVIDIA-Nemotron-Nano-9B-v2"` in tandem with `--judge_server_type="vllm" --judge_server_gpus 1` will use the `NVIDIA-Nemotron-Nano-9B-v2` itself as a judge. @@ -202,7 +202,7 @@ ns eval \ --server_type=vllm \ --server_gpus=1 \ --server_args="--mamba_ssm_cache_dtype float32 " \ - --judge_model="o3-mini-20250131" \ + --judge_model="o3-mini-2025-01-31" \ --extra_judge_args="++inference.tokens_to_generate=4096 ++max_concurrent_requests=8" \ ++parse_reasoning=True \ ++inference.tokens_to_generate=32768 \ diff --git a/nemo_skills/dataset/hle/__init__.py b/nemo_skills/dataset/hle/__init__.py index 51db80829d..d805fbf3e7 100644 --- a/nemo_skills/dataset/hle/__init__.py +++ b/nemo_skills/dataset/hle/__init__.py @@ -20,10 +20,10 @@ # Some answers are not possible to compare symbolically, so have to use a judge model # Setting openai judge by default, but can be overriden from command line for a locally hosted model -# Currently using o3-mini-20250131 which is used by the official leaderboard - https://agi.safe.ai/ +# Currently using o3-mini-2025-01-31 which is used by the official leaderboard - https://agi.safe.ai/ # To approximate the Artificial Analysis Index results, we suggest using gpt-4o - https://artificialanalysis.ai/methodology/intelligence-benchmarking#evaluation-suite-details JUDGE_PIPELINE_ARGS = { - "model": "o3-mini-20250131", + "model": "o3-mini-2025-01-31", "server_type": "openai", "server_address": "https://api.openai.com/v1", } diff --git a/nemo_skills/dataset/simpleqa/__init__.py b/nemo_skills/dataset/simpleqa/__init__.py index d3829e4281..c1e7d82158 100644 --- a/nemo_skills/dataset/simpleqa/__init__.py +++ b/nemo_skills/dataset/simpleqa/__init__.py @@ -20,10 +20,10 @@ # SimpleQA requires judge model for evaluating factual accuracy # Setting openai judge by default, but can be overridden from command line for a locally hosted model -# Using o3-mini-20250131 as recommended for factual evaluation tasks +# Using o3-mini-2025-01-31 as recommended for factual evaluation tasks JUDGE_PIPELINE_ARGS = { - "model": "o3-mini-20250131", + "model": "o3-mini-2025-01-31", "server_type": "openai", "server_address": "https://api.openai.com/v1", } From 0c57d243e91ec023e0c4e25d2b76310bab945edd Mon Sep 17 00:00:00 2001 From: Igor Gitman Date: Wed, 7 Jan 2026 12:21:31 -0800 Subject: [PATCH 76/88] Unify local sandbox with slurm setup (#1153) Signed-off-by: Igor Gitman Signed-off-by: Cheng-Ping Hsieh --- .../code_execution/local_sandbox/start_local_sandbox.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nemo_skills/code_execution/local_sandbox/start_local_sandbox.sh b/nemo_skills/code_execution/local_sandbox/start_local_sandbox.sh index db40d3c5ce..03da9b7546 100755 --- a/nemo_skills/code_execution/local_sandbox/start_local_sandbox.sh +++ b/nemo_skills/code_execution/local_sandbox/start_local_sandbox.sh @@ -20,9 +20,9 @@ SANDBOX_NAME=${1:-'local-sandbox'} docker build --tag=${SANDBOX_NAME} --build-arg="NUM_WORKERS=$((`nproc --all`))" -f dockerfiles/Dockerfile.sandbox . echo "Multi-worker mode: Starting $((`nproc --all`)) workers with session affinity" -docker run --network=host \ +docker run --network=host --rm \ --memory=${NEMO_SKILLS_SANDBOX_MEM_LIMIT:-"16g"} \ ${UWSGI_CPU_AFFINITY:+-e UWSGI_CPU_AFFINITY=${UWSGI_CPU_AFFINITY}} \ ${UWSGI_PROCESSES:+-e UWSGI_PROCESSES=${UWSGI_PROCESSES}} \ - --restart unless-stopped \ + -v /nemo_run:/nemo_run \ --name=local-sandbox ${SANDBOX_NAME} From 257480a173be19303b0501ce0e487ae014377a3e Mon Sep 17 00:00:00 2001 From: Mateusz Winiarek <72758259+Froxyy-dev@users.noreply.github.com> Date: Fri, 9 Jan 2026 00:54:18 +0100 Subject: [PATCH 77/88] fix: robust judgement handling (#1134) Signed-off-by: Mateusz Winiarek Co-authored-by: Igor Gitman Signed-off-by: Cheng-Ping Hsieh --- nemo_skills/evaluation/metrics/utils.py | 15 ++++++++------- .../summarize_results_output-ms8192.txt | 2 +- .../eval_outputs/summarize_results_output.txt | 2 +- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/nemo_skills/evaluation/metrics/utils.py b/nemo_skills/evaluation/metrics/utils.py index f5f804d5b7..6b25098407 100644 --- a/nemo_skills/evaluation/metrics/utils.py +++ b/nemo_skills/evaluation/metrics/utils.py @@ -36,13 +36,14 @@ def read_predictions(predictions, line_idx, file_handles): def is_correct_judgement(judgement, return_none=False) -> Union[bool, None]: # Match both plain "Judgement:" and markdown bold "**Judgement**:" formats, this happens for gpt-4o which is AA Judge model. - match = re.search(r"\*{0,2}Judgement\*{0,2}\s*:", judgement, re.IGNORECASE) - if match: - verdict = judgement[match.end() :].strip() - if verdict.lower().startswith("yes"): - return True - elif verdict.lower().startswith("no"): - return False + if judgement: + match = re.search(r"\*{0,2}Judgement\*{0,2}\s*:", judgement, re.IGNORECASE) + if match: + verdict = judgement[match.end() :].strip().lstrip("*").strip() + if verdict.lower().startswith("yes"): + return True + elif verdict.lower().startswith("no"): + return False if return_none: return None diff --git a/tests/data/eval_outputs/summarize_results_output-ms8192.txt b/tests/data/eval_outputs/summarize_results_output-ms8192.txt index 4035bc4193..1704d97441 100644 --- a/tests/data/eval_outputs/summarize_results_output-ms8192.txt +++ b/tests/data/eval_outputs/summarize_results_output-ms8192.txt @@ -1,7 +1,7 @@ --------------------------------------------------------- Metrics for Max Sequence Length 8192 -------------------------------------------------------- --------------------------------------------------------------------- answer-judge -------------------------------------------------------------------- evaluation_mode | num_entries | avg_tokens | correct_judgements | false_positives | false_negatives | invalid_judgements | precision | recall | f1 -pass@1[avg-of-4] | 12 | 189 | 43.75% ± 7.98% | 18.75% | 22.92% | 14.58% | 58.45% | 42.86% | 48.32% +pass@1[avg-of-4] | 12 | 189 | 45.83% ± 4.81% | 18.75% | 22.92% | 12.50% | 60.24% | 46.43% | 51.07% majority@4 | 12 | 189 | 54.17% | 16.67% | 25.00% | 0.00% | 66.67% | 57.14% | 61.54% pass@4 | 12 | 189 | 66.67% | 8.33% | 25.00% | 0.00% | 80.00% | 57.14% | 66.67% diff --git a/tests/data/eval_outputs/summarize_results_output.txt b/tests/data/eval_outputs/summarize_results_output.txt index b85e7b4ef8..f80d15d613 100644 --- a/tests/data/eval_outputs/summarize_results_output.txt +++ b/tests/data/eval_outputs/summarize_results_output.txt @@ -1,6 +1,6 @@ --------------------------------------------------------------------- answer-judge -------------------------------------------------------------------- evaluation_mode | num_entries | avg_tokens | correct_judgements | false_positives | false_negatives | invalid_judgements | precision | recall | f1 -pass@1[avg-of-4] | 12 | 189 | 43.75% ± 7.98% | 18.75% | 22.92% | 14.58% | 58.45% | 42.86% | 48.32% +pass@1[avg-of-4] | 12 | 189 | 45.83% ± 4.81% | 18.75% | 22.92% | 12.50% | 60.24% | 46.43% | 51.07% majority@4 | 12 | 189 | 54.17% | 16.67% | 25.00% | 0.00% | 66.67% | 57.14% | 61.54% pass@4 | 12 | 189 | 66.67% | 8.33% | 25.00% | 0.00% | 80.00% | 57.14% | 66.67% From fcb671cd0d82d48b2fbb019cd82bcac69e9d7cfa Mon Sep 17 00:00:00 2001 From: Valentin Mendelev Date: Fri, 9 Jan 2026 14:37:14 +0100 Subject: [PATCH 78/88] generation.py to respect separate server type for the client (#1135) Signed-off-by: Valentin Mendelev Signed-off-by: Nikolay Karpov Co-authored-by: Nikolay Karpov Signed-off-by: Cheng-Ping Hsieh --- nemo_skills/pipeline/utils/generation.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/nemo_skills/pipeline/utils/generation.py b/nemo_skills/pipeline/utils/generation.py index 863b196e20..cc1238c168 100644 --- a/nemo_skills/pipeline/utils/generation.py +++ b/nemo_skills/pipeline/utils/generation.py @@ -550,6 +550,11 @@ def configure_client( - server_address: Address of the server. - extra_arguments: Updated extra arguments for the command. """ + # Check if user already specified server.server_type in extra_arguments + server_type_arg = "" if "++server.server_type=" in extra_arguments else f"++server.server_type={server_type} " + # Only add server_type if user didn't specify it (allows vllm_multimodal override) + extra_arguments = server_type_arg + extra_arguments + if server_gpus: # we need to host the model server_port = get_free_port(strategy="random") if get_random_port else 5000 assert server_gpus is not None, "Need to specify server_gpus if hosting the model" @@ -567,13 +572,9 @@ def configure_client( if server_container: server_config["container"] = server_container extra_arguments = ( - f"{extra_arguments} ++server.server_type={server_type} ++server.host=127.0.0.1 " - f"++server.port={server_port} ++server.model={model} " + f"++server.host=127.0.0.1 ++server.port={server_port} ++server.model={model} {extra_arguments}" ) else: # model is hosted elsewhere server_config = None - extra_arguments = ( - f"{extra_arguments} ++server.server_type={server_type} " - f"++server.base_url={server_address} ++server.model={model} " - ) + extra_arguments = f"++server.base_url={server_address} ++server.model={model} {extra_arguments}" return server_config, server_address, extra_arguments From e36619ba507ace8f253dc71b7f3a4a757f17596b Mon Sep 17 00:00:00 2001 From: Dan Lord Date: Fri, 9 Jan 2026 17:51:12 -0800 Subject: [PATCH 79/88] Add compute eval (#1158) Signed-off-by: George Armstrong Signed-off-by: Dan Lord Co-authored-by: George Armstrong Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> Signed-off-by: Cheng-Ping Hsieh --- dockerfiles/Dockerfile.nemo-skills | 2 +- docs/evaluation/code.md | 59 +++++++++++++ nemo_skills/dataset/compute-eval/__init__.py | 19 ++++ nemo_skills/dataset/compute-eval/prepare.py | 87 +++++++++++++++++++ nemo_skills/evaluation/evaluator/__init__.py | 2 + .../evaluation/evaluator/compute_eval.py | 85 ++++++++++++++++++ .../evaluation/metrics/code_metrics.py | 12 +++ nemo_skills/evaluation/metrics/map_metrics.py | 2 + nemo_skills/inference/eval/compute_eval.py | 81 +++++++++++++++++ .../prompt/config/compute-eval/baseline.yaml | 65 ++++++++++++++ requirements/main.txt | 1 + tests/gpu-tests/test_eval.py | 2 + tests/test_datasets.py | 1 + 13 files changed, 417 insertions(+), 1 deletion(-) create mode 100644 nemo_skills/dataset/compute-eval/__init__.py create mode 100644 nemo_skills/dataset/compute-eval/prepare.py create mode 100644 nemo_skills/evaluation/evaluator/compute_eval.py create mode 100644 nemo_skills/inference/eval/compute_eval.py create mode 100644 nemo_skills/prompt/config/compute-eval/baseline.yaml diff --git a/dockerfiles/Dockerfile.nemo-skills b/dockerfiles/Dockerfile.nemo-skills index 3d35f10dc8..631d9a706d 100644 --- a/dockerfiles/Dockerfile.nemo-skills +++ b/dockerfiles/Dockerfile.nemo-skills @@ -59,7 +59,7 @@ COPY pyproject.toml README.md /opt/NeMo-Skills/ COPY requirements /opt/NeMo-Skills/requirements/ # installing sdp in container only RUN pip install git+https://github.com/NVIDIA/NeMo-speech-data-processor@29b9b1ec0ceaf3ffa441c1d01297371b3f8e11d2 -ARG CACHEBUST=2 +ARG CACHEBUST=3 RUN pip install --no-cache-dir -r /opt/NeMo-Skills/requirements/main.txt # Fix http mismatch between lepton and dggs by manually downloading dggs here RUN pip install ddgs diff --git a/docs/evaluation/code.md b/docs/evaluation/code.md index 5be1b81ea1..0361fc4db6 100644 --- a/docs/evaluation/code.md +++ b/docs/evaluation/code.md @@ -178,6 +178,65 @@ all you need to do is replace `openhands` with `swe_agent` in the command above. !!! note For evaluation, we use a [custom fork](https://github.com/Kipok/SWE-bench) of the SWE-bench repository that supports running evaluation inside of an existing container. It may not always have the latest updates from the upstream repo. +### compute-eval + +- Benchmark is defined in [`nemo_skills/dataset/compute-eval/__init__.py`](https://github.com/NVIDIA-NeMo/Skills/blob/main/nemo_skills/dataset/compute-eval/__init__.py) +- Original benchmark source is [here](https://github.com/NVIDIA/compute-eval). + +ComputeEval is a benchmark for evaluating Large Language Models on CUDA code generation tasks. It features handcrafted CUDA programming challenges that test an LLM's capability at writing reliable CUDA code. The benchmark includes functional correctness evaluation through compilation and execution against held-out test suites. + +**Prerequisites:** NVIDIA GPU with CUDA Toolkit 12 or greater must be installed, and `nvcc` must be available in your PATH. + +#### Data Preparation + +First, prepare the dataset by running the `ns prepare_data` command. You can optionally specify a release version: + +```bash +ns prepare_data compute-eval --release 2025-1 +``` + +If no release is specified, the default release will be downloaded. This will generate an `eval.jsonl` file in the `nemo_skills/dataset/compute-eval/` directory. + +**Note:** You need to set the `HF_TOKEN` environment variable because the dataset requires authentication. + +#### Running the Evaluation + +Once the data is prepared, you can run the evaluation. Replace `<...>` placeholders with your cluster and directory paths. + +This command runs an evaluation of [OpenReasoning-Nemotron-32B](https://huggingface.co/nvidia/OpenReasoning-Nemotron-32B) on a Slurm cluster: + +```bash +ns eval \ + --cluster= \ + --model=nvidia/OpenReasoning-Nemotron-32B \ + --server_type=vllm \ + --server_args="--async-scheduling" \ + --server_nodes=1 \ + --server_gpus=8 \ + --benchmarks=compute-eval \ + --data_dir= \ + --output_dir= \ + ++inference.temperature=0.6 \ + ++inference.top_p=0.95 \ + ++inference.tokens_to_generate=16384 +``` + +**Security Note:** ComputeEval executes machine-generated CUDA code. While the benchmark is designed for evaluation purposes, we strongly recommend running evaluations in a sandboxed environment (e.g., a Docker container or virtual machine) to minimize security risks. + +#### Verifying Results + +After all jobs are complete, you can check the results in `/eval-results/compute-eval/metrics.json`. You can also review `/eval-results/compute-eval/summarized-results/main_*`. They should look something like this: + +``` +---------------------------- compute-eval ----------------------------- +evaluation_mode | num_entries | avg_tokens | gen_seconds | accuracy +pass@1 | 50 | 8432 | 1245 | 64.00% +``` + +The benchmark reports: +- **accuracy**: Percentage of problems where generated code compiled and passed all tests +- **pass@1**: Same as accuracy for single-solution generation +- **pass@k**: Success rate when generating k solutions per problem (if configured) ### IOI diff --git a/nemo_skills/dataset/compute-eval/__init__.py b/nemo_skills/dataset/compute-eval/__init__.py new file mode 100644 index 0000000000..cf1eed5e3b --- /dev/null +++ b/nemo_skills/dataset/compute-eval/__init__.py @@ -0,0 +1,19 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +EVAL_SPLIT = "eval" +DATASET_GROUP = "code" +METRICS_TYPE = "compute-eval" +GENERATION_MODULE = "nemo_skills.inference.eval.compute_eval" +GENERATION_ARGS = "++prompt_config=compute-eval/baseline ++eval_type=compute-eval" diff --git a/nemo_skills/dataset/compute-eval/prepare.py b/nemo_skills/dataset/compute-eval/prepare.py new file mode 100644 index 0000000000..f291f84c84 --- /dev/null +++ b/nemo_skills/dataset/compute-eval/prepare.py @@ -0,0 +1,87 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import argparse +import json +import os +from pathlib import Path + +from datasets import load_dataset + +_CONTEXT_FILES_BLOCK_TEMPLATE = """ +--- file: {path} +```{fence} +{content} +``` +""" + + +def _fence_for_path(path: str) -> str: + p = path.lower() + if p.endswith((".cu", ".cuh")): + return "cuda" + if p.endswith((".cc", ".cpp", ".cxx")): + return "cpp" + if p.endswith(".c"): + return "c" + if p.endswith(".h") or p.endswith(".hpp"): + return "h" + # Default to plaintext if unknown + return "" + + +def _format_context_files_block(context_files: list[dict[str, str]]) -> str: + blocks: list[str] = [] + for source in context_files: + if "path" not in source or "content" not in source: + continue + + fence = _fence_for_path(source["path"]) + blocks.append( + _CONTEXT_FILES_BLOCK_TEMPLATE.format(path=source["path"], fence=fence, content=source["content"]) + ) + return "".join(blocks) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Download and prepare nvidia/compute-eval dataset") + parser.add_argument( + "--release", + type=str, + default=None, + help="Release to download (e.g., '2025-1', '2025-2'). If not specified, downloads default release.", + ) + + args = parser.parse_args() + + token = os.getenv("HF_TOKEN", None) + if not token: + print("Error: HF_TOKEN environment variable not set. Please set it to access the dataset.") + exit(1) + + dataset = load_dataset("nvidia/compute-eval", args.release, token=token) + data_dir = Path(__file__).absolute().parent + data_dir.mkdir(exist_ok=True) + + with open(data_dir / "eval.jsonl", "wt", encoding="utf-8") as f: + for item in dataset["eval"]: + record = { + "problem": item, + "task_id": item["task_id"], + "problem_prompt": item["prompt"], + "build_command": item["build_command"], + "context_files_block": _format_context_files_block(item["context_files"]), + } + + # Dumping using default=str to handle datetime serialization from the problem records + f.write(json.dumps(record, default=str) + "\n") diff --git a/nemo_skills/evaluation/evaluator/__init__.py b/nemo_skills/evaluation/evaluator/__init__.py index 03b38ec3c1..269bff939c 100644 --- a/nemo_skills/evaluation/evaluator/__init__.py +++ b/nemo_skills/evaluation/evaluator/__init__.py @@ -27,6 +27,7 @@ eval_livebench_coding, eval_livecodebench_pro, ) +from nemo_skills.evaluation.evaluator.compute_eval import ComputeEvalEvaluator from nemo_skills.evaluation.evaluator.icpc import ICPCEvaluator from nemo_skills.evaluation.evaluator.ifbench import eval_ifbench from nemo_skills.evaluation.evaluator.ifeval import eval_if @@ -70,6 +71,7 @@ "icpc": ICPCEvaluator, "audio": AudioEvaluator, "bird": BirdEvaluator, + "compute-eval": ComputeEvalEvaluator, } # Validation: Ensure no overlap between class and function maps diff --git a/nemo_skills/evaluation/evaluator/compute_eval.py b/nemo_skills/evaluation/evaluator/compute_eval.py new file mode 100644 index 0000000000..eab2b2d6e0 --- /dev/null +++ b/nemo_skills/evaluation/evaluator/compute_eval.py @@ -0,0 +1,85 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import asyncio +import logging +from typing import Annotated, Any + +from compute_eval.data.data_model import CudaCppProblem, CudaPythonProblem, FileSolution, PatchSolution +from compute_eval.execution import evaluate_solution +from compute_eval.utils.eval_utils import get_nvcc_version, parse_semver +from pydantic import Field, TypeAdapter + +from nemo_skills.evaluation.evaluator import BaseEvaluator +from nemo_skills.utils import get_logger_name + +_LOG = logging.getLogger(get_logger_name(__file__)) +_PROBLEM_ADAPTER = TypeAdapter(Annotated[CudaCppProblem | CudaPythonProblem, Field(discriminator="type")]) +_SOLUTION_ADAPTER = TypeAdapter(Annotated[FileSolution | PatchSolution, Field(discriminator="type")]) + + +class ComputeEvalEvaluator(BaseEvaluator): + _installed_ctk_major: int + _installed_ctk_minor: int + + def __init__(self, config: dict, num_parallel_requests=10): + super().__init__(config, num_parallel_requests) + nvcc_version = get_nvcc_version() + if not nvcc_version: + raise RuntimeError( + "NVCC not found. Please ensure that the CUDA Toolkit is installed and nvcc is in your PATH." + ) + + self._installed_ctk_major, self._installed_ctk_minor, _ = parse_semver(nvcc_version) + + async def eval_single(self, data_point: dict[str, Any]) -> dict[str, Any]: + # noinspection PyBroadException + try: + problem = _PROBLEM_ADAPTER.validate_python(data_point["problem"]) + solution = _SOLUTION_ADAPTER.validate_python(data_point["solution"]) + + graded = await asyncio.to_thread( + evaluate_solution, + installed_ctk_major=self._installed_ctk_major, + installed_ctk_minor=self._installed_ctk_minor, + problem=problem, + solution=solution, + ) + + return { + "passed": graded.passed, + "skipped": graded.skipped, + "elapsed_time": graded.elapsed_time, + "build_output": graded.build_output, + "test_output": graded.test_output, + } + except KeyError as e: + _LOG.error(f"Missing required field in data_point: {e}") + return { + "passed": False, + "skipped": False, + "elapsed_time": 0.0, + "build_output": "", + "test_output": "", + "error": f"Missing required field: {e}", + } + except Exception as e: + _LOG.error(f"Error during evaluation: {e}") + return { + "passed": False, + "skipped": False, + "elapsed_time": 0.0, + "build_output": "", + "test_output": "", + "error": str(e), + } diff --git a/nemo_skills/evaluation/metrics/code_metrics.py b/nemo_skills/evaluation/metrics/code_metrics.py index 8274efd77b..e537e0abca 100644 --- a/nemo_skills/evaluation/metrics/code_metrics.py +++ b/nemo_skills/evaluation/metrics/code_metrics.py @@ -121,3 +121,15 @@ def get_incorrect_sample(self, prediction: dict) -> dict: def update(self, predictions): super().update(predictions) self._compute_pass_at_k(predictions=predictions) + + +class ComputeEvalMetrics(BaseMetrics): + def _get_score_dict(self, prediction: dict) -> dict[str, bool | int | float]: + return {"accuracy": prediction["passed"]} + + def get_incorrect_sample(self, prediction: dict) -> dict: + return {"passed": False} + + def update(self, predictions): + super().update(predictions) + self._compute_pass_at_k(predictions=predictions) diff --git a/nemo_skills/evaluation/metrics/map_metrics.py b/nemo_skills/evaluation/metrics/map_metrics.py index a78ebba81c..bcdece2e44 100644 --- a/nemo_skills/evaluation/metrics/map_metrics.py +++ b/nemo_skills/evaluation/metrics/map_metrics.py @@ -24,6 +24,7 @@ from nemo_skills.evaluation.metrics.bird_metrics import BirdMetrics from nemo_skills.evaluation.metrics.code_metrics import ( BigCodeBenchMetrics, + ComputeEvalMetrics, EvalPlusMetrics, HumanEvalInfillingMetrics, LiveCodeBenchMetrics, @@ -72,6 +73,7 @@ "mmau_pro_closed_form": MMAUProMetrics, "mmau_pro_open_ended": MMAUProMetrics, "mmau_pro_instruction_following": MMAUProMetrics, + "compute-eval": ComputeEvalMetrics, } diff --git a/nemo_skills/inference/eval/compute_eval.py b/nemo_skills/inference/eval/compute_eval.py new file mode 100644 index 0000000000..4e54270122 --- /dev/null +++ b/nemo_skills/inference/eval/compute_eval.py @@ -0,0 +1,81 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging +import sys + +import hydra +from compute_eval.data.data_model import FileSolution + +# noinspection PyProtectedMember +from compute_eval.generate_completions import _parse_solution + +from nemo_skills.inference.generate import GenerateSolutionsConfig, GenerationTask +from nemo_skills.inference.model import server_params +from nemo_skills.utils import ( + get_help_message, + get_logger_name, + setup_logging, +) + +_LOG = logging.getLogger(get_logger_name(__file__)) + + +class ComputeEvalGenerationTask(GenerationTask): + def __init__(self, cfg: GenerateSolutionsConfig): + super().__init__(cfg) + + async def process_single_datapoint(self, data_point, data): + res = await super().process_single_datapoint(data_point, data) + try: + solution = FileSolution( + task_id=data_point["task_id"], + files=_parse_solution(res["generation"]), + ) + return { + "solution": solution.model_dump(), + "generation": res["generation"], + } + except KeyError as e: + _LOG.error(f"Missing required field: {e}") + return { + "solution": None, + "generation": res.get("generation", ""), + "error": f"Missing required field: {e}", + } + except Exception as e: + _LOG.error(f"Failed to parse solution: {e}") + return { + "solution": None, + "generation": res.get("generation", ""), + "error": f"Failed to parse solution: {e}", + } + + +GENERATION_TASK_CLASS = ComputeEvalGenerationTask + + +@hydra.main(version_base=None, config_name="base_generation_config") +def run_compute_eval(cfg: GenerateSolutionsConfig): + _LOG.info("Config used: %s", cfg) + + task = ComputeEvalGenerationTask(cfg) + task.generate() + + +if __name__ == "__main__": + if "--help" in sys.argv or "-h" in sys.argv: + print(get_help_message(GenerateSolutionsConfig, server_params=server_params())) + else: + setup_logging() + run_compute_eval() diff --git a/nemo_skills/prompt/config/compute-eval/baseline.yaml b/nemo_skills/prompt/config/compute-eval/baseline.yaml new file mode 100644 index 0000000000..54126a6251 --- /dev/null +++ b/nemo_skills/prompt/config/compute-eval/baseline.yaml @@ -0,0 +1,65 @@ +system: |- + You are a senior CUDA/C/C++ engineer. Produce complete, compilable solutions from a structured problem specification. Follow these rules: + + General + - You will be given: a problem description, context files (editable), and build environment details (e.g., build command). + - Hidden tests exist but are not shown. Do not mention tests, do not write test code, and do not add I/O used only for testing. + - Use only the APIs and contracts specified in the problem and context files. Preserve all provided function signatures exactly. + - Prefer using only headers already present in the provided codebase. Avoid adding new headers unless strictly necessary and supported by the build command. Do not introduce third-party dependencies. + + Context files policy + - You may modify provided context files when necessary. If you include any file in your solution output (new or modified), emit its full and final contents; your output will overwrite the provided version. + - Only emit files you add or modify. Do not output files that are unchanged, and do not include placeholder blocks saying "no changes" or similar. + + Build command + - You should pay careful attention to the build command or any context files about the build process. + - The build command and/or context build files may include important hints about required files or expected project structure. This likely includes the name of the expected solution file, important macros, standards, or linked libraries. + - Pay special attention to -I or -isystem flags -- they indicate important include paths. Remember, if a -I or -isystem flag is present you do not need to include the relative path in your #include statements. + + + Output format + - Output only source files needed for the solution. No explanations or commentary. + - Each file must be in its own fenced code block, with the first line indicating its path as a comment. + Example: + ``` + // file: geodistance.cu + #include "geodistance.h" + ... + ``` + + Code quality and constraints + + The solution must compile cleanly with the provided build command and target architectures. + Avoid unnecessary heap allocations, environment access, and global mutable state. Keep deterministic behavior. + Honor all contracts, constants, and macros defined in provided headers. + + For CUDA: + Implement kernels with correct global signatures and parameter types. + Bounds-check all memory accesses; consider grid-stride loops when appropriate for scalability. + Favor coalesced memory access and avoid undefined behavior. + Apply appropriate numerical stability practices when needed (e.g., clamp arguments before acos/asin). + + Reasoning discipline + + Think through edge cases and performance internally, but output only the final code files, no analysis or explanations. + +user: |- + Produce the complete solution as one or more source files that compile with the provided build command. Do not output anything except the code files. + + Problem + Description: + $problem_prompt + + Build command: + $build_command + + Context files: + $context_files_block + + Output requirements + + Emit only the source files necessary to satisfy the problem (new or modified). + Only emit files you add or modify. Do not output files that are unchanged, and do not include placeholder blocks saying "no changes" or similar. + Do not include any test code or references to tests. + If an interface header is provided (e.g., declares functions to implement), place implementations in a corresponding .cu/.cc source file and include that header. + Begin your response with the first code block. diff --git a/requirements/main.txt b/requirements/main.txt index 2ebd4d3b4b..2c2fa36696 100644 --- a/requirements/main.txt +++ b/requirements/main.txt @@ -14,6 +14,7 @@ bs4 click < 8.2.0 # https://github.com/ai-dynamo/dynamo/issues/1039 +compute-eval @ git+https://github.com/NVIDIA/compute-eval.git@2d14770 datasets<4 # lcb problem with datasets 4.0.0 # ddgs # Needed for BFCLv4 - currently cannot be installed directly due to lepton obsolete httpx version in use evalplus @ git+https://github.com/evalplus/evalplus@c91370f diff --git a/tests/gpu-tests/test_eval.py b/tests/gpu-tests/test_eval.py index a753cb7f53..5724c422a4 100644 --- a/tests/gpu-tests/test_eval.py +++ b/tests/gpu-tests/test_eval.py @@ -47,6 +47,8 @@ "mrcr", "audiobench", "librispeech-pc", + # Excluded for the time being as compute eval requires either a CTK or local docker engine to run + "compute-eval", } diff --git a/tests/test_datasets.py b/tests/test_datasets.py index 86fd152df2..b788c00beb 100644 --- a/tests/test_datasets.py +++ b/tests/test_datasets.py @@ -59,6 +59,7 @@ ("mmau-pro", ["test"]), ("audiobench", ["test"]), ("librispeech-pc", ["test"]), + ("compute-eval", ["eval"]), ] From e256fb6b3b22053856e81277b0bc072e93b65341 Mon Sep 17 00:00:00 2001 From: George <37293288+Jorjeous@users.noreply.github.com> Date: Mon, 12 Jan 2026 20:51:52 +0400 Subject: [PATCH 80/88] add musan dataset (#1139) Signed-off-by: George Zelenfroind Signed-off-by: George <37293288+Jorjeous@users.noreply.github.com> Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> Signed-off-by: Cheng-Ping Hsieh --- nemo_skills/dataset/musan/__init__.py | 57 ++ nemo_skills/dataset/musan/prepare.py | 492 ++++++++++++++++++ nemo_skills/evaluation/evaluator/audio.py | 14 +- .../evaluation/metrics/audio_metrics.py | 23 +- nemo_skills/pipeline/prepare_data.py | 2 +- tests/gpu-tests/test_eval.py | 1 + tests/test_datasets.py | 1 + 7 files changed, 575 insertions(+), 15 deletions(-) create mode 100644 nemo_skills/dataset/musan/__init__.py create mode 100644 nemo_skills/dataset/musan/prepare.py diff --git a/nemo_skills/dataset/musan/__init__.py b/nemo_skills/dataset/musan/__init__.py new file mode 100644 index 0000000000..2962ad75bf --- /dev/null +++ b/nemo_skills/dataset/musan/__init__.py @@ -0,0 +1,57 @@ +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""MUSAN: A Music, Speech, and Noise Corpus + +MUSAN is a corpus of music, speech, and noise recordings designed for training +models for voice activity detection and music/speech discrimination. + +DOWNLOAD OPTIONS: + +1. HuggingFace (default - INCOMPLETE): + - 774 samples, 5h 4m total + - Noise: 728 samples (78% complete) + - Fast, no API key needed + +2. Kaggle (RECOMMENDED - COMPLETE): ✓ + - 10.3 GB, 2,016 WAV files + - Noise: 930 files (99.8% complete!) + - Music: 660 files, Speech: 426 files + - Requires Kaggle API key (one-time setup) + +3. OpenSLR (official - COMPLETE): + - 11 GB, full dataset + - No API key needed + +Reference: + David Snyder, Guoguo Chen, and Daniel Povey + "MUSAN: A Music, Speech, and Noise Corpus" + arXiv:1510.08484, 2015 +""" + +DATASET_GROUP = "speechlm" +IS_BENCHMARK_GROUP = True +SCORE_MODULE = "nemo_skills.evaluation.metrics.audio_metrics" +METRICS_TYPE = "audio" + +# Evaluation settings +EVAL_ARGS = "++eval_type=audio " + +# Generation settings - OpenAI format for audio-language models +GENERATION_ARGS = "++prompt_format=openai " + +# Benchmark - single test.jsonl contains all noise samples at top level +BENCHMARKS = { + "musan": {}, +} diff --git a/nemo_skills/dataset/musan/prepare.py b/nemo_skills/dataset/musan/prepare.py new file mode 100644 index 0000000000..8a735896ba --- /dev/null +++ b/nemo_skills/dataset/musan/prepare.py @@ -0,0 +1,492 @@ +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""MUSAN Dataset Preparation for nemo-skills + +Prepares the MUSAN dataset (Music, Speech, and Noise Corpus) for use with nemo-skills. + +Dataset sources: + - HuggingFace: 774 samples (~5h), incomplete, fast download + - Kaggle: 2016 files (10.3GB), nearly complete, requires API key + - OpenSLR: Complete dataset (11GB), official source + +Usage: + python -m nemo_skills.dataset.musan.prepare --source kaggle --categories noise + python -m nemo_skills.dataset.musan.prepare --categories noise --max-samples 100 + python -m nemo_skills.dataset.musan.prepare --source openslr --categories noise +""" + +import argparse +import json +import os +import sys +import tarfile +import urllib.request +from pathlib import Path +from typing import Dict, List + +import numpy as np +import soundfile as sf +from tqdm import tqdm + +# HuggingFace dataset label mappings +CATEGORY_LABELS = { + "noise": 0, + "music": 1, +} + +LABEL_TO_CATEGORY = { + 0: "noise", + 1: "other", +} + + +def download_from_kaggle(output_dir: Path) -> Path: + """Download MUSAN dataset from Kaggle using kagglehub.""" + try: + import kagglehub + except ImportError: + raise ImportError("kagglehub not installed. Run: pip install kagglehub") + + print("Downloading from Kaggle (requires API key in ~/.kaggle/kaggle.json)") + + try: + path = kagglehub.dataset_download("dogrose/musan-dataset") + print(f"Downloaded to: {path}") + return Path(path) + except Exception as e: + raise Exception(f"Kaggle download failed: {e}") + + +def download_from_openslr(output_dir: Path) -> Path: + """Download MUSAN dataset from OpenSLR (11 GB).""" + url = "https://www.openslr.org/resources/17/musan.tar.gz" + download_path = output_dir / "musan.tar.gz" + extract_path = output_dir / "musan_openslr" + + print("Downloading from OpenSLR (~11 GB)") + print(f"URL: {url}") + + if not download_path.exists(): + + def reporthook(block_num, block_size, total_size): + downloaded = block_num * block_size + percent = min(downloaded / total_size * 100, 100) + mb_downloaded = downloaded / (1024 * 1024) + mb_total = total_size / (1024 * 1024) + print(f"\r{percent:.1f}% ({mb_downloaded:.1f}/{mb_total:.1f} MB)", end="") + + urllib.request.urlretrieve(url, download_path, reporthook) + print("\nDownload complete") + else: + print(f"Using cached archive: {download_path}") + + if not extract_path.exists(): + print(f"Extracting to {extract_path}...") + extract_path.mkdir(parents=True, exist_ok=True) + with tarfile.open(download_path, "r:gz") as tar: + if sys.version_info >= (3, 11, 4): + tar.extractall(extract_path, filter="data") + else: + tar.extractall(extract_path) + print("Extraction complete") + else: + print(f"Using extracted data: {extract_path}") + + return extract_path / "musan" + + +def load_dataset_from_source(source: str, output_dir: Path): + """Load MUSAN dataset from specified source.""" + if source == "huggingface": + from datasets import load_dataset + + print("Loading from HuggingFace...") + dataset = load_dataset("FluidInference/musan", split="train") + print(f"Loaded {len(dataset)} samples") + return dataset, "huggingface" + + elif source == "kaggle": + dataset_path = download_from_kaggle(output_dir) + musan_path = dataset_path / "musan" + if not musan_path.exists(): + raise ValueError(f"'musan' directory not found in {dataset_path}") + + print(f"Dataset path: {musan_path}") + for cat in ["music", "speech", "noise"]: + cat_path = musan_path / cat + if cat_path.exists(): + wav_count = len(list(cat_path.glob("**/*.wav"))) + print(f" {cat}: {wav_count} files") + + return musan_path, "kaggle" + + elif source == "openslr": + dataset_path = download_from_openslr(output_dir) + print(f"Dataset path: {dataset_path}") + for cat in ["music", "speech", "noise"]: + cat_path = dataset_path / cat + if cat_path.exists(): + wav_count = len(list(cat_path.glob("**/*.wav"))) + print(f" {cat}: {wav_count} files") + + return dataset_path, "openslr" + + else: + raise ValueError(f"Unknown source: {source}") + + +def get_audio_duration(audio_array: np.ndarray, sampling_rate: int) -> float: + """Compute audio duration in seconds.""" + if audio_array is None or len(audio_array) == 0: + return 0.0 + return float(len(audio_array) / sampling_rate) + + +def save_audio_file(audio_array: np.ndarray, sampling_rate: int, output_path: str): + """Save audio array to WAV file.""" + os.makedirs(os.path.dirname(output_path), exist_ok=True) + sf.write(output_path, audio_array, sampling_rate) + + +def create_manifest_entry( + audio_filename: str, + duration: float, + category: str, + sample_id: int, + label: str, +) -> Dict: + """Create nemo-skills manifest entry.""" + audio_root = os.getenv("NEMO_SKILLS_AUDIO_ROOT", "/data") + audio_rel_path = f"{audio_root}/musan/{category}/audio/{audio_filename}" + audio_metadata = {"path": audio_rel_path, "duration": duration} + + # Instruction for transcription (expects empty response for non-speech audio) + instruction = "Transcribe the speech in this audio. If there is no speech, do not output anything." + + entry = { + "audio_path": [audio_rel_path], + "messages": [ + {"role": "system", "content": "You are a helpful assistant. /no_think"}, + { + "role": "user", + "content": instruction, + "audio": audio_metadata, + "audios": [audio_metadata], + }, + ], + "expected_answer": "", + "dataset": "musan", + "subset_for_metrics": f"musan_{category}", + "sample_id": sample_id, + "category": category, + "original_label": label, + "task_type": "Hallucination", + "audio_duration": duration, + "question": instruction, + } + + return entry + + +def process_category_from_files( + category: str, + dataset_path: Path, + output_dir: Path, + save_audio: bool = True, + split: str = "train", + max_samples: int = -1, +) -> tuple[int, List[Dict]]: + """Process MUSAN category from WAV files (Kaggle/OpenSLR format).""" + category_path = dataset_path / category + if not category_path.exists(): + raise ValueError(f"Category directory not found: {category_path}") + + wav_files = sorted(list(category_path.glob("**/*.wav"))) + print(f"Found {len(wav_files)} WAV files") + + if len(wav_files) == 0: + return 0, [] + + if max_samples > 0 and len(wav_files) > max_samples: + wav_files = wav_files[:max_samples] + print(f"Limited to {max_samples} samples") + + audio_dir = output_dir / category / "audio" + dataset_dir = output_dir / category + os.makedirs(audio_dir, exist_ok=True) + os.makedirs(dataset_dir, exist_ok=True) + + manifest_entries = [] + successful = 0 + failed = 0 + + for idx, wav_path in enumerate(tqdm(wav_files, desc=f"Processing {category}")): + try: + audio_array, sampling_rate = sf.read(str(wav_path)) + duration = get_audio_duration(audio_array, sampling_rate) + audio_filename = f"musan_{category}_{idx:06d}.wav" + local_audio_path = audio_dir / audio_filename + + if save_audio: + try: + save_audio_file(audio_array, sampling_rate, str(local_audio_path)) + except Exception as e: + print(f"Failed to save sample {idx}: {e}") + failed += 1 + continue + + entry = create_manifest_entry( + audio_filename=audio_filename, + duration=duration, + category=category, + sample_id=idx, + label=wav_path.stem, + ) + + manifest_entries.append(entry) + successful += 1 + + except Exception as e: + print(f"Error processing {wav_path}: {e}") + failed += 1 + continue + + manifest_path = dataset_dir / "test.jsonl" + with open(manifest_path, "w", encoding="utf-8") as f: + for entry in manifest_entries: + f.write(json.dumps(entry, ensure_ascii=False) + "\n") + + print(f"Saved {successful} samples to {manifest_path}") + if failed > 0: + print(f"Failed: {failed} samples") + + return successful, manifest_entries + + +def process_category( + category: str, + output_dir: Path, + dataset, + source_type: str, + save_audio: bool = True, + split: str = "train", + max_samples: int = -1, +) -> tuple[int, List[Dict]]: + """Process a single MUSAN category.""" + print(f"\n{'=' * 60}") + print(f"Processing: {category}") + print(f"{'=' * 60}") + + if source_type in ["kaggle", "openslr"]: + return process_category_from_files( + category=category, + dataset_path=dataset, + output_dir=output_dir, + save_audio=save_audio, + split=split, + max_samples=max_samples, + ) + + elif source_type != "huggingface": + raise NotImplementedError(f"Source '{source_type}' not supported") + + filtered_samples = [] + target_label = CATEGORY_LABELS.get(category) + if target_label is None: + print(f"Unknown category '{category}'") + return 0, [] + + for sample in dataset: + label = sample.get("label") + if label == target_label: + filtered_samples.append(sample) + + print(f"Found {len(filtered_samples)} samples") + + if len(filtered_samples) == 0: + return 0, [] + + if max_samples > 0 and len(filtered_samples) > max_samples: + filtered_samples = filtered_samples[:max_samples] + print(f"Limited to {max_samples} samples") + + # Create output directories + audio_dir = output_dir / category / "audio" + dataset_dir = output_dir / category + os.makedirs(audio_dir, exist_ok=True) + os.makedirs(dataset_dir, exist_ok=True) + + manifest_entries = [] + successful = 0 + failed = 0 + + for idx, sample in enumerate(tqdm(filtered_samples, desc=f"Processing {category}")): + try: + audio_dict = sample.get("audio") + if audio_dict is None: + failed += 1 + continue + + if isinstance(audio_dict, dict): + audio_array = audio_dict.get("array") + sampling_rate = audio_dict.get("sampling_rate", 16000) + else: + failed += 1 + continue + + if audio_array is None or len(audio_array) == 0: + failed += 1 + continue + + if isinstance(audio_array, list): + audio_array = np.array(audio_array) + + duration = get_audio_duration(audio_array, sampling_rate) + audio_filename = f"musan_{category}_{idx:06d}.wav" + local_audio_path = audio_dir / audio_filename + + if save_audio: + try: + save_audio_file(audio_array, sampling_rate, str(local_audio_path)) + except Exception as e: + print(f"Failed to save sample {idx}: {e}") + failed += 1 + continue + + label = sample.get("label", -1) + entry = create_manifest_entry( + audio_filename=audio_filename, + duration=duration, + category=category, + sample_id=idx, + label=str(label), + ) + + manifest_entries.append(entry) + successful += 1 + + except Exception as e: + print(f"Error processing sample {idx}: {e}") + failed += 1 + continue + + manifest_path = dataset_dir / "test.jsonl" + with open(manifest_path, "w", encoding="utf-8") as f: + for entry in manifest_entries: + f.write(json.dumps(entry, ensure_ascii=False) + "\n") + + print(f"Saved {successful} samples to {manifest_path}") + if failed > 0: + print(f"Failed: {failed} samples") + + return successful, manifest_entries + + +def main(): + parser = argparse.ArgumentParser(description="Prepare MUSAN dataset for nemo-skills") + parser.add_argument( + "--source", + choices=["huggingface", "kaggle", "openslr"], + default="huggingface", + help="Download source: huggingface (fast, incomplete), kaggle (complete, API key), openslr (complete, 11GB)", + ) + parser.add_argument("--split", default="train", choices=["train", "validation", "test"]) + parser.add_argument("--output-dir", type=str, default=None) + parser.add_argument( + "--categories", + nargs="+", + choices=["music", "speech", "noise"], + default=["music", "speech", "noise"], + ) + parser.add_argument("--no-audio", dest="save_audio", action="store_false") + parser.add_argument("--max-samples", type=int, default=-1) + parser.set_defaults(save_audio=True) + + args = parser.parse_args() + + if args.output_dir: + output_dir = Path(args.output_dir) + else: + output_dir = Path(__file__).parent + + output_dir.mkdir(parents=True, exist_ok=True) + + print("\n" + "=" * 60) + print("MUSAN Dataset Preparation") + print("=" * 60) + print(f"Source: {args.source}") + print(f"Output: {output_dir}") + print(f"Categories: {', '.join(args.categories)}") + print("=" * 60 + "\n") + + try: + dataset, source_type = load_dataset_from_source(args.source, output_dir) + except Exception as e: + print(f"Failed to load dataset: {e}") + return + + total_samples = 0 + successful_categories = [] + failed_categories = [] + all_entries = [] + + for category in args.categories: + try: + num_samples, entries = process_category( + category=category, + output_dir=output_dir, + dataset=dataset, + source_type=source_type, + save_audio=args.save_audio, + split=args.split, + max_samples=args.max_samples, + ) + total_samples += num_samples + successful_categories.append(category) + all_entries.extend(entries) + + except Exception as e: + print(f"\nFailed: {category} - {e}\n") + failed_categories.append((category, str(e))) + + if all_entries: + combined_manifest_path = output_dir / "test.jsonl" + with open(combined_manifest_path, "w", encoding="utf-8") as f: + for entry in all_entries: + f.write(json.dumps(entry, ensure_ascii=False) + "\n") + print(f"\nCombined manifest: {combined_manifest_path}") + print(f"Total samples: {len(all_entries)}") + + print("\n" + "=" * 60) + print("SUMMARY") + print("=" * 60) + print( + f"Requested: {len(args.categories)}, Successful: {len(successful_categories)}, Failed: {len(failed_categories)}" + ) + print(f"Total samples: {total_samples}") + + if successful_categories: + for name in successful_categories: + print(f" ✓ {name}") + + if failed_categories: + for name, error in failed_categories: + print(f" ✗ {name}: {error}") + + print("=" * 60 + "\n") + + +if __name__ == "__main__": + main() diff --git a/nemo_skills/evaluation/evaluator/audio.py b/nemo_skills/evaluation/evaluator/audio.py index c212666311..ff97181bbb 100644 --- a/nemo_skills/evaluation/evaluator/audio.py +++ b/nemo_skills/evaluation/evaluator/audio.py @@ -224,7 +224,7 @@ def evaluate_cer(reference: str, hypothesis: str) -> dict[str, Any]: def evaluate_hallucination(reference: str, hypothesis: str, audio_context: dict = None) -> dict[str, Any]: """Detect potential hallucinations via speaking rate anomaly. - Normal speech: ~10-15 chars/second. Higher rates suggest repetition/hallucination. + Normal speech: ~600-900 chars/minute. Higher rates suggest repetition/hallucination. Requires audio_duration in audio_context. """ audio_duration = audio_context.get("audio_duration") if audio_context else None @@ -238,10 +238,11 @@ def evaluate_hallucination(reference: str, hypothesis: str, audio_context: dict } char_count = len(hypothesis) - char_rate = char_count / audio_duration + # Convert to chars/minute + char_rate = (char_count / audio_duration) * 60.0 - # Hallucination threshold: >25 chars/sec (too fast = likely repetition) - is_hallucinating = char_rate > 25.0 + # Hallucination threshold: >1500 chars/min (25 chars/second * 60) + is_hallucinating = char_rate > 1500.0 return { "hallucination_rate": 1.0 if is_hallucinating else 0.0, @@ -385,8 +386,9 @@ def evaluate_sample(sample: dict[str, Any], config: AudioEvaluatorConfig) -> dic audio_duration = sample.get("audio_duration", None) if audio_duration and audio_duration > 0 and expected_answer and generation: - updates["ref_char_rate"] = len(expected_answer) / audio_duration - updates["hyp_char_rate"] = len(generation) / audio_duration + # chars/minute (chars/second * 60) + updates["ref_char_rate"] = (len(expected_answer) / audio_duration) * 60.0 + updates["hyp_char_rate"] = (len(generation) / audio_duration) * 60.0 updates["char_rate_diff"] = abs(updates["hyp_char_rate"] - updates["ref_char_rate"]) return updates diff --git a/nemo_skills/evaluation/metrics/audio_metrics.py b/nemo_skills/evaluation/metrics/audio_metrics.py index 95a133833d..7142f634fe 100644 --- a/nemo_skills/evaluation/metrics/audio_metrics.py +++ b/nemo_skills/evaluation/metrics/audio_metrics.py @@ -34,7 +34,7 @@ import logging -from nemo_skills.evaluation.metrics.base import BaseMetrics, as_int, as_percentage +from nemo_skills.evaluation.metrics.base import BaseMetrics, as_float, as_int, as_percentage from nemo_skills.utils import get_logger_name LOG = logging.getLogger(get_logger_name(__file__)) @@ -72,7 +72,8 @@ def __init__(self, compute_no_answer: bool = True, max_k: int = 1): self.pc_rate_scores = [] self.punct_f1_scores = [] self.cap_accuracy_scores = [] - self.char_rate_scores = [] + self.total_hallucinated_chars = 0 + self.total_audio_seconds = 0.0 # Judge scores (AudioBench-style rating 0-5, or legacy binary Yes/No mapped to 1/0) self.judge_ratings = [] @@ -210,8 +211,13 @@ def update(self, predictions): self.punct_f1_scores.append(pred["punct_f1"]) if "cap_accuracy" in pred and pred["cap_accuracy"] is not None: self.cap_accuracy_scores.append(pred["cap_accuracy"]) - if "char_rate" in pred and pred["char_rate"] is not None: - self.char_rate_scores.append(pred["char_rate"]) + + if pred.get("task_type") == "Hallucination": + predicted_text = pred.get("predicted_answer") or pred.get("generation") or "" + audio_duration = pred.get("audio_duration", 0.0) + if audio_duration > 0: + self.total_hallucinated_chars += len(predicted_text.strip()) + self.total_audio_seconds += audio_duration # Collect judge ratings (0-5) from judge datasets if available score_dict = self._get_score_dict(pred) @@ -276,8 +282,9 @@ def get_metrics(self): agg_metrics["cap_accuracy"] = round( 100.0 * sum(self.cap_accuracy_scores) / len(self.cap_accuracy_scores), 2 ) - if self.char_rate_scores: - agg_metrics["char_rate"] = round(sum(self.char_rate_scores) / len(self.char_rate_scores), 2) + if self.total_audio_seconds > 0: + total_minutes = self.total_audio_seconds / 60.0 + agg_metrics["char_rate"] = round(self.total_hallucinated_chars / total_minutes, 2) return metrics_dict @@ -337,8 +344,8 @@ def metrics_to_print(self): base_metrics["punct_f1"] = as_percentage if self.cap_accuracy_scores: base_metrics["cap_accuracy"] = as_percentage - if self.char_rate_scores: - base_metrics["char_rate"] = as_int + if self.total_audio_seconds > 0: + base_metrics["char_rate"] = as_float base_metrics["num_entries"] = as_int # Add at end for better display order diff --git a/nemo_skills/pipeline/prepare_data.py b/nemo_skills/pipeline/prepare_data.py index 8c3a58a8ba..f4f8328d13 100644 --- a/nemo_skills/pipeline/prepare_data.py +++ b/nemo_skills/pipeline/prepare_data.py @@ -31,7 +31,7 @@ # TODO: read this from init.py -DATASETS_REQUIRE_DATA_DIR = ["ruler", "ioi24", "mmau-pro", "librispeech-pc", "audiobench", "asr-leaderboard"] +DATASETS_REQUIRE_DATA_DIR = ["ruler", "ioi24", "mmau-pro", "librispeech-pc", "audiobench", "asr-leaderboard", "musan"] @app.command(context_settings={"allow_extra_args": True, "ignore_unknown_options": True}) diff --git a/tests/gpu-tests/test_eval.py b/tests/gpu-tests/test_eval.py index 5724c422a4..ee908a9108 100644 --- a/tests/gpu-tests/test_eval.py +++ b/tests/gpu-tests/test_eval.py @@ -47,6 +47,7 @@ "mrcr", "audiobench", "librispeech-pc", + "musan", # Excluded for the time being as compute eval requires either a CTK or local docker engine to run "compute-eval", } diff --git a/tests/test_datasets.py b/tests/test_datasets.py index b788c00beb..efa1415c53 100644 --- a/tests/test_datasets.py +++ b/tests/test_datasets.py @@ -59,6 +59,7 @@ ("mmau-pro", ["test"]), ("audiobench", ["test"]), ("librispeech-pc", ["test"]), + ("musan", ["test"]), ("compute-eval", ["eval"]), ] From 5536f54ed15c13a880a7d09c05451e44c3172b68 Mon Sep 17 00:00:00 2001 From: Cheng-Ping Hsieh Date: Tue, 13 Jan 2026 14:58:17 +0800 Subject: [PATCH 81/88] fix Signed-off-by: Cheng-Ping Hsieh --- nemo_skills/dataset/ruler2/prepare_mmlu.py | 14 ++-- nemo_skills/dataset/ruler2/prepare_niah.py | 77 ++++++++-------------- nemo_skills/dataset/ruler2/prepare_qa.py | 13 ++-- nemo_skills/dataset/ruler2/tokenizer.py | 43 +----------- requirements/main.txt | 1 + 5 files changed, 48 insertions(+), 100 deletions(-) diff --git a/nemo_skills/dataset/ruler2/prepare_mmlu.py b/nemo_skills/dataset/ruler2/prepare_mmlu.py index 4f69dfaf58..963a54a641 100644 --- a/nemo_skills/dataset/ruler2/prepare_mmlu.py +++ b/nemo_skills/dataset/ruler2/prepare_mmlu.py @@ -245,7 +245,7 @@ def generate_input_output(index, num_qs): else: repeats = 1 - curr_context = random.sample([item for item in haystack for _ in range(repeats)], num_qs) + curr_context = [dict(item) for item in random.sample([item for item in haystack for _ in range(repeats)], num_qs)] if args.num_order > 0: random_numbers = [generate_random_number() for _ in range(math.ceil((num_qs + 1) / args.num_order))] @@ -261,7 +261,7 @@ def generate_input_output(index, num_qs): random.shuffle(curr_context) examples = random.sample(curr_context, args.fewshot) - true_context = needle[index] + true_context = dict(needle[index]) true_context["random_index"] = random_numbers[-1] if args.insert_position < 0: insert_position = random.randint(0, len(curr_context)) @@ -383,15 +383,17 @@ def generate_samples(max_seq_length: int, incremental: int = 10): # Generate samples for index in tqdm(needle_sample): used_qs = num_qs - while(True): + while True: try: input_text, save_dict = generate_input_output(index, used_qs) length = len(TOKENIZER.text_to_tokens(input_text)) assert length <= max_seq_length, f"{length} exceeds max_seq_length." break - except: - if used_qs > incremental: - used_qs -= incremental ++ except AssertionError: ++ if used_qs > incremental: ++ used_qs -= incremental ++ else: ++ raise save_dict["length"] = length formatted_output = save_dict diff --git a/nemo_skills/dataset/ruler2/prepare_niah.py b/nemo_skills/dataset/ruler2/prepare_niah.py index 5596582bf8..7dfa844260 100644 --- a/nemo_skills/dataset/ruler2/prepare_niah.py +++ b/nemo_skills/dataset/ruler2/prepare_niah.py @@ -49,7 +49,7 @@ parser.add_argument("--num_needle_k", type=int, default=1) parser.add_argument("--num_needle_v", type=int, default=1) parser.add_argument("--num_needle_q", type=int, default=1) -parser.add_argument("--type_haystack", type=str, default='essay', help='[Options] noise, essay, needle.') +parser.add_argument("--type_haystack", type=str, default='needle', help='[Options] needle.') parser.add_argument("--type_needle_k", type=str, default='words', help='[Options] numbers, words, uuids.') parser.add_argument("--type_needle_v", type=str, default='numbers', help='[Options] numbers, words, uuids.') parser.add_argument("--num_digits_k", type=int, default=7) @@ -94,15 +94,17 @@ def generate_random_word(): def generate_random_uuid(): return str(uuid.UUID(int=random.getrandbits(128), version=4)) -def generate_random(type_needle: str, digits: int = None): +def generate_random(type_needle: str, digits: int | None = None): if type_needle == 'numbers': + if digits is None: + raise ValueError("digits must be provided when type_needle='numbers'") return generate_random_number(digits) elif type_needle == 'words': return generate_random_word() elif type_needle == 'uuids': return generate_random_uuid() else: - raise NotImplementedError(f'{args.type_needle} is not implemented.') + raise NotImplementedError(f'{type_needle} is not implemented.') def generate_input_output(num_haystack): keys, values, needles = [], [], [] @@ -121,49 +123,26 @@ def generate_input_output(num_haystack): random.Random(args.random_seed).shuffle(needles) # Context - if args.type_haystack == 'essay': - if num_haystack <= len(haystack): - text = " ".join(haystack[:num_haystack]) - else: - repeats = (num_haystack + len(haystack) - 1) // len(haystack) # Ceiling division - text = " ".join((haystack * repeats)[:num_haystack]) - document_sents = sent_tokenize(text.strip()) - insertion_positions = [0] + \ - sorted([int(len(document_sents) * (depth / 100)) for depth in random.sample(DEPTHS, len(needles))]) + \ - [len(document_sents)] - document_sents_list = [] - for i in range(1,len(insertion_positions)): - last_pos = insertion_positions[i-1] - next_pos = insertion_positions[i] - document_sents_list.append(" ".join(document_sents[last_pos:next_pos])) - if i-1 < len(needles): - document_sents_list.append(needles[i-1]) - context = " ".join(document_sents_list) - + if args.num_needle_v == 1: + sentences = [haystack.format( + type_needle_v=args.type_needle_v, + key=generate_random(args.type_needle_k, args.num_digits_k), + value=generate_random(args.type_needle_v, args.num_digits_v), + ) for _ in range(num_haystack)] else: - if args.type_haystack == 'noise': - sentences = [haystack] * num_haystack - elif args.type_haystack == 'needle': - if args.num_needle_v == 1: - sentences = [haystack.format( - type_needle_v=args.type_needle_v, - key=generate_random(args.type_needle_k, args.num_digits_k), - value=generate_random(args.type_needle_v, args.num_digits_v), - ) for _ in range(num_haystack)] - else: - haystack_values = [generate_random(args.type_needle_v, args.num_digits_v) for _ in range(num_haystack)] - haystack_keys = ([generate_random(args.type_needle_k, args.num_digits_k) for _ in range(math.ceil(num_haystack / args.num_needle_v))] * args.num_needle_v)[:num_haystack] - sentences = [haystack.format( - type_needle_v=args.type_needle_v, - key=haystack_keys[i], - value=haystack_values[i], - ) for i in range(num_haystack)] - random.shuffle(sentences) - - indexes = sorted(random.sample(range(num_haystack), len(needles)), reverse=True) - for index, element in zip(indexes, needles): - sentences.insert(index, element) - context = "\n".join(sentences) + haystack_values = [generate_random(args.type_needle_v, args.num_digits_v) for _ in range(num_haystack)] + haystack_keys = ([generate_random(args.type_needle_k, args.num_digits_k) for _ in range(math.ceil(num_haystack / args.num_needle_v))] * args.num_needle_v)[:num_haystack] + sentences = [haystack.format( + type_needle_v=args.type_needle_v, + key=haystack_keys[i], + value=haystack_values[i], + ) for i in range(num_haystack)] + random.shuffle(sentences) + + indexes = sorted(random.sample(range(num_haystack), len(needles)), reverse=True) + for index, element in zip(indexes, needles): + sentences.insert(index, element) + context = "\n".join(sentences) ## Query and Answer @@ -216,7 +195,7 @@ def generate_samples(num_samples: int, max_seq_length: int, incremental: int = 5 logger.info(f"Starting binary search with bounds: {lower_bound} to {upper_bound}") while lower_bound <= upper_bound: mid = (lower_bound + upper_bound) // 2 - input_text, save_dict = generate_input_output(mid) + input_text, _answers = generate_input_output(mid) total_tokens = len(TOKENIZER.text_to_tokens(input_text)) logger.info(f"Testing haystack size: {mid}, resulting tokens: {total_tokens}/{max_seq_length}") @@ -235,15 +214,17 @@ def generate_samples(num_samples: int, max_seq_length: int, incremental: int = 5 # Generate samples for index in tqdm(range(num_samples)): used_haystack = num_haystack - while(True): + while True: try: input_text, answer = generate_input_output(used_haystack) length = len(TOKENIZER.text_to_tokens(input_text)) assert length <= max_seq_length, f"{length} exceeds max_seq_length." break - except: + except AssertionError: if used_haystack > incremental: used_haystack -= incremental + else: + raise formatted_output = { 'index': index, diff --git a/nemo_skills/dataset/ruler2/prepare_qa.py b/nemo_skills/dataset/ruler2/prepare_qa.py index a72f2ff0c2..d1096a615c 100644 --- a/nemo_skills/dataset/ruler2/prepare_qa.py +++ b/nemo_skills/dataset/ruler2/prepare_qa.py @@ -171,13 +171,13 @@ def generate_random_number(num_digits=7): def generate_input_output(index, num_docs): - curr_needle = needle[index] + curr_needle = dict(needle[index]) curr_needle["context"] = [{**c, "random_index": generate_random_number()} for c in curr_needle["context"]] curr_needle["distractor"] = [{**c, "random_index": generate_random_number()} for c in curr_needle["distractor"]] if args.fewshot > 0: fewshot_examples = random.sample([i for i in range(len(needle)) if i != index], args.fewshot) - fewshot_examples = [needle[i] for i in fewshot_examples] + fewshot_examples = [dict(needle[i]) for i in fewshot_examples] for e in fewshot_examples: e["context"] = [{**c, "random_index": generate_random_number()} for c in e['context']] e["distractor"] = [{**c, "random_index": generate_random_number()} for c in e['distractor']] @@ -185,6 +185,9 @@ def generate_input_output(index, num_docs): fewshot_examples = [] remaining_haystack_size = len(haystack) - len(set([c["text"] for c in (curr_needle["context"] + curr_needle["distractor"])] + [f["text"] for e in fewshot_examples for f in (e["context"] + e["distractor"])])) + if remaining_haystack_size <= 0: + raise ValueError("No remaining haystack documents available after exclusions.") + if num_docs > remaining_haystack_size: repeats = (num_docs + remaining_haystack_size - 1) // remaining_haystack_size # Ceiling division else: @@ -307,16 +310,18 @@ def generate_samples(num_samples: int, max_seq_length: int, incremental: int = 5 # Generate samples for index in tqdm(range(num_samples)): used_docs = num_docs - while(True): + while True: try: input_text, save_dict = generate_input_output(index, used_docs) length = len(TOKENIZER.text_to_tokens(input_text)) if max_seq_length > 0: assert length <= max_seq_length, f"{length} exceeds max_seq_length." break - except: + except AssertionError: if used_docs > incremental: used_docs -= incremental + else: + raise save_dict["length"] = length formatted_output = save_dict diff --git a/nemo_skills/dataset/ruler2/tokenizer.py b/nemo_skills/dataset/ruler2/tokenizer.py index 567ec8ef06..4917d2cd58 100644 --- a/nemo_skills/dataset/ruler2/tokenizer.py +++ b/nemo_skills/dataset/ruler2/tokenizer.py @@ -24,14 +24,7 @@ def select_tokenizer(tokenizer_type, tokenizer_path): - if tokenizer_type == 'nemo': - if '.model' in tokenizer_path: - return NeMoSentencePieceTokenizer(model_path=tokenizer_path) - elif '.json' in tokenizer_path: - return NeMoTikTokenTokenizer(vocab_file=tokenizer_path) - else: - raise ValueError(f"Unknown tokenizer file format {tokenizer_path}") - elif tokenizer_type == 'hf': + if tokenizer_type == 'hf': return HFTokenizer(model_path=tokenizer_path) elif tokenizer_type == 'openai': return OpenAITokenizer(model_path=tokenizer_path) @@ -41,40 +34,6 @@ def select_tokenizer(tokenizer_type, tokenizer_path): raise ValueError(f"Unknown tokenizer_type {tokenizer_type}") -class NeMoTikTokenTokenizer: - """ - Tokenizer from NeMo TiktokenTokenizer - """ - def __init__(self, vocab_file) -> None: - from nemo.collections.common.tokenizers.tiktoken_tokenizer import TiktokenTokenizer - self.tokenizer = TiktokenTokenizer(vocab_file=vocab_file) - - def text_to_tokens(self, text: str) -> List[str]: - tokens = self.tokenizer.text_to_tokens(text) - return tokens - - def tokens_to_text(self, tokens: List[int]) -> str: - text = self.tokenizer.tokens_to_text(tokens) - return text - - -class NeMoSentencePieceTokenizer: - """ - Tokenizer from NeMo SentencePieceTokenizer - """ - def __init__(self, model_path) -> None: - from nemo.collections.common.tokenizers.sentencepiece_tokenizer import SentencePieceTokenizer - self.tokenizer = SentencePieceTokenizer(model_path=model_path) - - def text_to_tokens(self, text: str) -> List[str]: - tokens = self.tokenizer.text_to_tokens(text) - return tokens - - def tokens_to_text(self, tokens: List[int]) -> str: - text = self.tokenizer.tokens_to_text(tokens) - return text - - class HFTokenizer: """ Tokenizer from HF models diff --git a/requirements/main.txt b/requirements/main.txt index 2c2fa36696..a7d7dded31 100644 --- a/requirements/main.txt +++ b/requirements/main.txt @@ -50,3 +50,4 @@ tqdm transformers typer >= 0.13 wandb +editdistance From 8779b7524ee4d11974ed767ab535c5ea07584260 Mon Sep 17 00:00:00 2001 From: Cheng-Ping Hsieh Date: Tue, 13 Jan 2026 15:20:29 +0800 Subject: [PATCH 82/88] fix Signed-off-by: Cheng-Ping Hsieh --- nemo_skills/dataset/ruler2/prepare.py | 349 +++++++++++---------- nemo_skills/dataset/ruler2/prepare_niah.py | 2 +- nemo_skills/dataset/ruler2/tokenizer.py | 2 +- 3 files changed, 183 insertions(+), 170 deletions(-) diff --git a/nemo_skills/dataset/ruler2/prepare.py b/nemo_skills/dataset/ruler2/prepare.py index 26f9825862..c044db6a04 100644 --- a/nemo_skills/dataset/ruler2/prepare.py +++ b/nemo_skills/dataset/ruler2/prepare.py @@ -31,229 +31,241 @@ def prepare_mk_niah_basic(output_folder, tokenizer_type, tokenizer_path, length, dataset_size): subprocess.run( - f"python -m nemo_skills.dataset.ruler2.prepare_niah " - f"--output_folder {output_folder} " - f"--tokenizer_type {tokenizer_type} " - f"--tokenizer_path {tokenizer_path} " - f"--max_seq_length {length} " - f"--num_samples {dataset_size} " - f"--random_seed 42 " - f"--num_needle_k 1 " - f"--num_needle_v 1 " - f"--num_needle_q 1 " - f"--type_haystack needle " - f"--type_needle_k words " - f"--type_needle_v numbers " - f"--num_digits_v 10", - shell=True, + [ + "python", "-m", "nemo_skills.dataset.ruler2.prepare_niah", + "--output_folder", output_folder, + "--tokenizer_type", tokenizer_type, + "--tokenizer_path", tokenizer_path, + "--max_seq_length", length, + "--num_samples", dataset_size, + "--random_seed", "42", + "--num_needle_k", "1", + "--num_needle_v", "1", + "--num_needle_q", "1", + "--type_haystack", "needle", + "--type_needle_k", "words", + "--type_needle_v", "numbers", + "--num_digits_v", "10", + ] check=True, ) def prepare_mk_niah_easy(output_folder, tokenizer_type, tokenizer_path, length, dataset_size): subprocess.run( - f"python -m nemo_skills.dataset.ruler2.prepare_mmlu " - f"--output_folder {output_folder} " - f"--tokenizer_type {tokenizer_type} " - f"--tokenizer_path {tokenizer_path} " - f"--max_seq_length {length} " - f"--num_samples {dataset_size} " - f"--random_seed 42 " - f"--dataset mmlu " - f"--fewshot 0 " - f"--prompt_type instruct " - f"--num_order 0 " - f"--task_type retrieve " - f"--algo_type single", - shell=True, + [ + "python", "-m", "nemo_skills.dataset.ruler2.prepare_mmlu", + "--output_folder", output_folder, + "--tokenizer_type", tokenizer_type, + "--tokenizer_path", tokenizer_path, + "--max_seq_length", length, + "--num_samples", dataset_size, + "--random_seed", "42", + "--dataset", "mmlu", + "--fewshot", "0", + "--prompt_type", "instruct", + "--num_order", "0", + "--task_type", "retrieve", + "--algo_type", "single", + ] check=True, ) def prepare_mk_niah_medium(output_folder, tokenizer_type, tokenizer_path, length, dataset_size): subprocess.run( - f"python -m nemo_skills.dataset.ruler2.prepare_mmlu " - f"--output_folder {output_folder} " - f"--tokenizer_type {tokenizer_type} " - f"--tokenizer_path {tokenizer_path} " - f"--max_seq_length {length} " - f"--num_samples {dataset_size} " - f"--random_seed 42 " - f"--dataset mmlu " - f"--fewshot 5 " - f"--prompt_type instruct " - f"--num_order 0 " - f"--task_type solve " - f"--algo_type 2steps", - shell=True, + [ + "python", "-m", "nemo_skills.dataset.ruler2.prepare_mmlu", + "--output_folder", output_folder, + "--tokenizer_type", tokenizer_type, + "--tokenizer_path", tokenizer_path, + "--max_seq_length", length, + "--num_samples", dataset_size, + "--random_seed", "42", + "--dataset", "mmlu", + "--fewshot", "5", + "--prompt_type", "instruct", + "--num_order", "0", + "--task_type", "solve", + "--algo_type", "2steps", + ] check=True, ) def prepare_mk_niah_hard(output_folder, tokenizer_type, tokenizer_path, length, dataset_size): subprocess.run( - f"python -m nemo_skills.dataset.ruler2.prepare_mmlu " - f"--output_folder {output_folder} " - f"--tokenizer_type {tokenizer_type} " - f"--tokenizer_path {tokenizer_path} " - f"--max_seq_length {length} " - f"--num_samples {dataset_size} " - f"--random_seed 42 " - f"--dataset mmlu " - f"--fewshot 5 " - f"--prompt_type instruct " - f"--num_order 0 " - f"--task_type solve " - f"--algo_type single", - shell=True, + [ + "python", "-m", "nemo_skills.dataset.ruler2.prepare_mmlu", + "--output_folder", output_folder, + "--tokenizer_type", tokenizer_type, + "--tokenizer_path", tokenizer_path, + "--max_seq_length", length, + "--num_samples", dataset_size, + "--random_seed", "42", + "--dataset", "mmlu", + "--fewshot", "5", + "--prompt_type", "instruct", + "--num_order", "0", + "--task_type", "solve", + "--algo_type", "single", + ] check=True, ) def prepare_mv_niah_basic(output_folder, tokenizer_type, tokenizer_path, length, dataset_size): subprocess.run( - f"python -m nemo_skills.dataset.ruler2.prepare_niah " - f"--output_folder {output_folder} " - f"--tokenizer_type {tokenizer_type} " - f"--tokenizer_path {tokenizer_path} " - f"--max_seq_length {length} " - f"--num_samples {dataset_size} " - f"--random_seed 42 " - f"--num_needle_k 1 " - f"--num_needle_v 4 " - f"--num_needle_q 1 " - f"--type_haystack needle " - f"--type_needle_k words " - f"--type_needle_v numbers " - f"--num_digits_v 10", - shell=True, + [ + "python", "-m", "nemo_skills.dataset.ruler2.prepare_niah", + "--output_folder", output_folder, + "--tokenizer_type", tokenizer_type, + "--tokenizer_path", tokenizer_path, + "--max_seq_length", length, + "--num_samples", dataset_size, + "--random_seed", "42", + "--num_needle_k", "1", + "--num_needle_v", "4", + "--num_needle_q", "1", + "--type_haystack", "needle", + "--type_needle_k", "words", + "--type_needle_v", "numbers", + "--num_digits_v", "10", + ] check=True, ) def prepare_mv_niah_easy(output_folder, tokenizer_type, tokenizer_path, length, dataset_size): subprocess.run( - f"python -m nemo_skills.dataset.ruler2.prepare_mmlu " - f"--output_folder {output_folder} " - f"--tokenizer_type {tokenizer_type} " - f"--tokenizer_path {tokenizer_path} " - f"--max_seq_length {length} " - f"--num_samples {dataset_size} " - f"--random_seed 42 " - f"--dataset mmlu " - f"--fewshot 0 " - f"--prompt_type instruct " - f"--num_order 4 " - f"--task_type niah " - f"--algo_type single", - shell=True, + [ + "python", "-m", "nemo_skills.dataset.ruler2.prepare_mmlu", + "--output_folder", output_folder, + "--tokenizer_type", tokenizer_type, + "--tokenizer_path", tokenizer_path, + "--max_seq_length", length, + "--num_samples", dataset_size, + "--random_seed", "42", + "--dataset", "mmlu", + "--fewshot", "0", + "--prompt_type", "instruct", + "--num_order", "4", + "--task_type", "niah", + "--algo_type", "single", + ] check=True, ) def prepare_mv_niah_medium(output_folder, tokenizer_type, tokenizer_path, length, dataset_size): subprocess.run( - f"python -m nemo_skills.dataset.ruler2.prepare_mmlu " - f"--output_folder {output_folder} " - f"--tokenizer_type {tokenizer_type} " - f"--tokenizer_path {tokenizer_path} " - f"--max_seq_length {length} " - f"--num_samples {dataset_size} " - f"--random_seed 42 " - f"--dataset mmlu " - f"--fewshot 0 " - f"--prompt_type instruct " - f"--num_order 4 " - f"--task_type retrieve " - f"--algo_type 2steps", - shell=True, + [ + "python", "-m", "nemo_skills.dataset.ruler2.prepare_mmlu", + "--output_folder", output_folder, + "--tokenizer_type", tokenizer_type, + "--tokenizer_path", tokenizer_path, + "--max_seq_length", length, + "--num_samples", dataset_size, + "--random_seed", "42", + "--dataset", "mmlu", + "--fewshot", "0", + "--prompt_type", "instruct", + "--num_order", "4", + "--task_type", "retrieve", + "--algo_type", "2steps", + ] check=True, - ) + ) def prepare_mv_niah_hard(output_folder, tokenizer_type, tokenizer_path, length, dataset_size): subprocess.run( - f"python -m nemo_skills.dataset.ruler2.prepare_mmlu " - f"--output_folder {output_folder} " - f"--tokenizer_type {tokenizer_type} " - f"--tokenizer_path {tokenizer_path} " - f"--max_seq_length {length} " - f"--num_samples {dataset_size} " - f"--random_seed 42 " - f"--dataset mmlu " - f"--fewshot 0 " - f"--prompt_type instruct " - f"--num_order 4 " - f"--task_type retrieve " - f"--algo_type single", - shell=True, + [ + "python", "-m", "nemo_skills.dataset.ruler2.prepare_mmlu", + "--output_folder", output_folder, + "--tokenizer_type", tokenizer_type, + "--tokenizer_path", tokenizer_path, + "--max_seq_length", length, + "--num_samples", dataset_size, + "--random_seed", "42", + "--dataset", "mmlu", + "--fewshot", "0", + "--prompt_type", "instruct", + "--num_order", "4", + "--task_type", "retrieve", + "--algo_type", "single", + ] check=True, ) def prepare_qa_basic(output_folder, tokenizer_type, tokenizer_path, length, dataset_size): subprocess.run( - f"python -m nemo_skills.dataset.ruler2.prepare_qa " - f"--output_folder {output_folder} " - f"--tokenizer_type {tokenizer_type} " - f"--tokenizer_path {tokenizer_path} " - f"--max_seq_length {length} " - f"--num_samples {dataset_size} " - f"--random_seed 42 " - f"--dataset hotpotqa " - f"--fewshot 0 " - f"--prompt_type instruct " - f"--task_type retrieve " - f"--query_type doc", - shell=True, + [ + "python", "-m", "nemo_skills.dataset.ruler2.prepare_qa", + "--output_folder", output_folder, + "--tokenizer_type", tokenizer_type, + "--tokenizer_path", tokenizer_path, + "--max_seq_length", length, + "--num_samples", dataset_size, + "--random_seed", "42", + "--dataset", "hotpotqa", + "--fewshot", "0", + "--prompt_type", "instruct", + "--task_type", "retrieve", + "--query_type", "doc", + ] check=True, ) def prepare_qa_easy(output_folder, tokenizer_type, tokenizer_path, length, dataset_size): subprocess.run( - f"python -m nemo_skills.dataset.ruler2.prepare_qa " - f"--output_folder {output_folder} " - f"--tokenizer_type {tokenizer_type} " - f"--tokenizer_path {tokenizer_path} " - f"--max_seq_length {length} " - f"--num_samples {dataset_size} " - f"--random_seed 42 " - f"--dataset hotpotqa " - f"--fewshot 0 " - f"--prompt_type instruct " - f"--task_type retrieve " - f"--query_type question", - shell=True, + [ + "python", "-m", "nemo_skills.dataset.ruler2.prepare_qa", + "--output_folder", output_folder, + "--tokenizer_type", tokenizer_type, + "--tokenizer_path", tokenizer_path, + "--max_seq_length", length, + "--num_samples", dataset_size, + "--random_seed", "42", + "--dataset", "hotpotqa", + "--fewshot", "0", + "--prompt_type", "instruct", + "--task_type", "retrieve", + "--query_type", "question", + ] check=True, ) def prepare_qa_medium(output_folder, tokenizer_type, tokenizer_path, length, dataset_size): subprocess.run( - f"python -m nemo_skills.dataset.ruler2.prepare_qa " - f"--output_folder {output_folder} " - f"--tokenizer_type {tokenizer_type} " - f"--tokenizer_path {tokenizer_path} " - f"--max_seq_length {length} " - f"--num_samples {dataset_size} " - f"--random_seed 42 " - f"--dataset hotpotqa " - f"--fewshot 0 " - f"--prompt_type instruct " - f"--task_type solve " - f"--algo_type 2steps", - shell=True, + [ + "python", "-m", "nemo_skills.dataset.ruler2.prepare_qa", + "--output_folder", output_folder, + "--tokenizer_type", tokenizer_type, + "--tokenizer_path", tokenizer_path, + "--max_seq_length", length, + "--num_samples", dataset_size, + "--random_seed", "42", + "--dataset", "hotpotqa", + "--fewshot", "0", + "--prompt_type", "instruct", + "--task_type", "solve", + "--algo_type", "2steps", + ] check=True, ) def prepare_qa_hard(output_folder, tokenizer_type, tokenizer_path, length, dataset_size): subprocess.run( - f"python -m nemo_skills.dataset.ruler2.prepare_qa " - f"--output_folder {output_folder} " - f"--tokenizer_type {tokenizer_type} " - f"--tokenizer_path {tokenizer_path} " - f"--max_seq_length {length} " - f"--num_samples {dataset_size} " - f"--random_seed 42 " - f"--dataset hotpotqa " - f"--fewshot 0 " - f"--prompt_type instruct " - f"--task_type solve " - f"--algo_type single", - shell=True, + [ + "python", "-m", "nemo_skills.dataset.ruler2.prepare_qa", + "--output_folder", output_folder, + "--tokenizer_type", tokenizer_type, + "--tokenizer_path", tokenizer_path, + "--max_seq_length", length, + "--num_samples", dataset_size, + "--random_seed", "42", + "--dataset", "hotpotqa", + "--fewshot", "0", + "--prompt_type", "instruct", + "--task_type", "solve", + "--algo_type", "single", + ] check=True, ) @@ -299,7 +311,8 @@ def prepare_dataset(tasks, setup, max_seq_length, tokenizer_type, tokenizer_path output_folder = Path(__file__).parent / setup # 1. installing necessary packages - subprocess.run(["pip install wonderwords html2text tenacity"], check=True, shell=True) + subprocess.run(["pip", "install", "wonderwords", "html2text", "tenacity"], check=True) + for task in tasks: prepare_task_for_ns(output_folder, task) diff --git a/nemo_skills/dataset/ruler2/prepare_niah.py b/nemo_skills/dataset/ruler2/prepare_niah.py index 7dfa844260..6fab5d3b7e 100644 --- a/nemo_skills/dataset/ruler2/prepare_niah.py +++ b/nemo_skills/dataset/ruler2/prepare_niah.py @@ -120,7 +120,7 @@ def generate_input_output(num_haystack): )) values.append(value) - random.Random(args.random_seed).shuffle(needles) + random.shuffle(needles) # Context if args.num_needle_v == 1: diff --git a/nemo_skills/dataset/ruler2/tokenizer.py b/nemo_skills/dataset/ruler2/tokenizer.py index 4917d2cd58..30b2a8b66f 100644 --- a/nemo_skills/dataset/ruler2/tokenizer.py +++ b/nemo_skills/dataset/ruler2/tokenizer.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From 9490505b50773e920931043eb6259cb029dc2728 Mon Sep 17 00:00:00 2001 From: Cheng-Ping Hsieh Date: Tue, 13 Jan 2026 15:37:35 +0800 Subject: [PATCH 83/88] fix Signed-off-by: Cheng-Ping Hsieh --- nemo_skills/dataset/ruler2/prepare.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/nemo_skills/dataset/ruler2/prepare.py b/nemo_skills/dataset/ruler2/prepare.py index c044db6a04..236169aff4 100644 --- a/nemo_skills/dataset/ruler2/prepare.py +++ b/nemo_skills/dataset/ruler2/prepare.py @@ -46,7 +46,7 @@ def prepare_mk_niah_basic(output_folder, tokenizer_type, tokenizer_path, length, "--type_needle_k", "words", "--type_needle_v", "numbers", "--num_digits_v", "10", - ] + ], check=True, ) @@ -66,7 +66,7 @@ def prepare_mk_niah_easy(output_folder, tokenizer_type, tokenizer_path, length, "--num_order", "0", "--task_type", "retrieve", "--algo_type", "single", - ] + ], check=True, ) @@ -86,7 +86,7 @@ def prepare_mk_niah_medium(output_folder, tokenizer_type, tokenizer_path, length "--num_order", "0", "--task_type", "solve", "--algo_type", "2steps", - ] + ], check=True, ) @@ -106,7 +106,7 @@ def prepare_mk_niah_hard(output_folder, tokenizer_type, tokenizer_path, length, "--num_order", "0", "--task_type", "solve", "--algo_type", "single", - ] + ], check=True, ) @@ -127,7 +127,7 @@ def prepare_mv_niah_basic(output_folder, tokenizer_type, tokenizer_path, length, "--type_needle_k", "words", "--type_needle_v", "numbers", "--num_digits_v", "10", - ] + ], check=True, ) @@ -147,7 +147,7 @@ def prepare_mv_niah_easy(output_folder, tokenizer_type, tokenizer_path, length, "--num_order", "4", "--task_type", "niah", "--algo_type", "single", - ] + ], check=True, ) @@ -167,7 +167,7 @@ def prepare_mv_niah_medium(output_folder, tokenizer_type, tokenizer_path, length "--num_order", "4", "--task_type", "retrieve", "--algo_type", "2steps", - ] + ], check=True, ) @@ -187,7 +187,7 @@ def prepare_mv_niah_hard(output_folder, tokenizer_type, tokenizer_path, length, "--num_order", "4", "--task_type", "retrieve", "--algo_type", "single", - ] + ], check=True, ) @@ -207,7 +207,7 @@ def prepare_qa_basic(output_folder, tokenizer_type, tokenizer_path, length, data "--prompt_type", "instruct", "--task_type", "retrieve", "--query_type", "doc", - ] + ], check=True, ) @@ -226,7 +226,7 @@ def prepare_qa_easy(output_folder, tokenizer_type, tokenizer_path, length, datas "--prompt_type", "instruct", "--task_type", "retrieve", "--query_type", "question", - ] + ], check=True, ) @@ -246,7 +246,7 @@ def prepare_qa_medium(output_folder, tokenizer_type, tokenizer_path, length, dat "--prompt_type", "instruct", "--task_type", "solve", "--algo_type", "2steps", - ] + ], check=True, ) @@ -265,7 +265,7 @@ def prepare_qa_hard(output_folder, tokenizer_type, tokenizer_path, length, datas "--prompt_type", "instruct", "--task_type", "solve", "--algo_type", "single", - ] + ], check=True, ) From 53a7d14df2821363fb20966304a140f8840feb36 Mon Sep 17 00:00:00 2001 From: Cheng-Ping Hsieh Date: Tue, 13 Jan 2026 15:49:24 +0800 Subject: [PATCH 84/88] fix Signed-off-by: Cheng-Ping Hsieh --- nemo_skills/dataset/ruler2/__init__.py | 2 +- nemo_skills/dataset/ruler2/prepare.py | 512 ++++++++++++++------- nemo_skills/dataset/ruler2/prepare_mmlu.py | 325 +++++++------ nemo_skills/dataset/ruler2/prepare_niah.py | 140 +++--- nemo_skills/dataset/ruler2/prepare_qa.py | 234 ++++++---- nemo_skills/dataset/ruler2/ruler2_score.py | 9 +- nemo_skills/dataset/ruler2/tokenizer.py | 19 +- nemo_skills/evaluation/evaluator/ruler.py | 28 +- requirements/main.txt | 2 +- 9 files changed, 798 insertions(+), 473 deletions(-) diff --git a/nemo_skills/dataset/ruler2/__init__.py b/nemo_skills/dataset/ruler2/__init__.py index 8af131119c..56cd62a201 100644 --- a/nemo_skills/dataset/ruler2/__init__.py +++ b/nemo_skills/dataset/ruler2/__init__.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -DATASET_GROUP = "long-context" \ No newline at end of file +DATASET_GROUP = "long-context" diff --git a/nemo_skills/dataset/ruler2/prepare.py b/nemo_skills/dataset/ruler2/prepare.py index 236169aff4..514ed188d4 100644 --- a/nemo_skills/dataset/ruler2/prepare.py +++ b/nemo_skills/dataset/ruler2/prepare.py @@ -15,7 +15,6 @@ import argparse import concurrent.futures -import json import subprocess from pathlib import Path @@ -32,161 +31,282 @@ def prepare_mk_niah_basic(output_folder, tokenizer_type, tokenizer_path, length, dataset_size): subprocess.run( [ - "python", "-m", "nemo_skills.dataset.ruler2.prepare_niah", - "--output_folder", output_folder, - "--tokenizer_type", tokenizer_type, - "--tokenizer_path", tokenizer_path, - "--max_seq_length", length, - "--num_samples", dataset_size, - "--random_seed", "42", - "--num_needle_k", "1", - "--num_needle_v", "1", - "--num_needle_q", "1", - "--type_haystack", "needle", - "--type_needle_k", "words", - "--type_needle_v", "numbers", - "--num_digits_v", "10", + "python", + "-m", + "nemo_skills.dataset.ruler2.prepare_niah", + "--output_folder", + output_folder, + "--tokenizer_type", + tokenizer_type, + "--tokenizer_path", + tokenizer_path, + "--max_seq_length", + length, + "--num_samples", + dataset_size, + "--random_seed", + "42", + "--num_needle_k", + "1", + "--num_needle_v", + "1", + "--num_needle_q", + "1", + "--type_haystack", + "needle", + "--type_needle_k", + "words", + "--type_needle_v", + "numbers", + "--num_digits_v", + "10", ], check=True, ) + def prepare_mk_niah_easy(output_folder, tokenizer_type, tokenizer_path, length, dataset_size): subprocess.run( [ - "python", "-m", "nemo_skills.dataset.ruler2.prepare_mmlu", - "--output_folder", output_folder, - "--tokenizer_type", tokenizer_type, - "--tokenizer_path", tokenizer_path, - "--max_seq_length", length, - "--num_samples", dataset_size, - "--random_seed", "42", - "--dataset", "mmlu", - "--fewshot", "0", - "--prompt_type", "instruct", - "--num_order", "0", - "--task_type", "retrieve", - "--algo_type", "single", + "python", + "-m", + "nemo_skills.dataset.ruler2.prepare_mmlu", + "--output_folder", + output_folder, + "--tokenizer_type", + tokenizer_type, + "--tokenizer_path", + tokenizer_path, + "--max_seq_length", + length, + "--num_samples", + dataset_size, + "--random_seed", + "42", + "--dataset", + "mmlu", + "--fewshot", + "0", + "--prompt_type", + "instruct", + "--num_order", + "0", + "--task_type", + "retrieve", + "--algo_type", + "single", ], check=True, ) + def prepare_mk_niah_medium(output_folder, tokenizer_type, tokenizer_path, length, dataset_size): subprocess.run( [ - "python", "-m", "nemo_skills.dataset.ruler2.prepare_mmlu", - "--output_folder", output_folder, - "--tokenizer_type", tokenizer_type, - "--tokenizer_path", tokenizer_path, - "--max_seq_length", length, - "--num_samples", dataset_size, - "--random_seed", "42", - "--dataset", "mmlu", - "--fewshot", "5", - "--prompt_type", "instruct", - "--num_order", "0", - "--task_type", "solve", - "--algo_type", "2steps", + "python", + "-m", + "nemo_skills.dataset.ruler2.prepare_mmlu", + "--output_folder", + output_folder, + "--tokenizer_type", + tokenizer_type, + "--tokenizer_path", + tokenizer_path, + "--max_seq_length", + length, + "--num_samples", + dataset_size, + "--random_seed", + "42", + "--dataset", + "mmlu", + "--fewshot", + "5", + "--prompt_type", + "instruct", + "--num_order", + "0", + "--task_type", + "solve", + "--algo_type", + "2steps", ], check=True, ) + def prepare_mk_niah_hard(output_folder, tokenizer_type, tokenizer_path, length, dataset_size): subprocess.run( [ - "python", "-m", "nemo_skills.dataset.ruler2.prepare_mmlu", - "--output_folder", output_folder, - "--tokenizer_type", tokenizer_type, - "--tokenizer_path", tokenizer_path, - "--max_seq_length", length, - "--num_samples", dataset_size, - "--random_seed", "42", - "--dataset", "mmlu", - "--fewshot", "5", - "--prompt_type", "instruct", - "--num_order", "0", - "--task_type", "solve", - "--algo_type", "single", + "python", + "-m", + "nemo_skills.dataset.ruler2.prepare_mmlu", + "--output_folder", + output_folder, + "--tokenizer_type", + tokenizer_type, + "--tokenizer_path", + tokenizer_path, + "--max_seq_length", + length, + "--num_samples", + dataset_size, + "--random_seed", + "42", + "--dataset", + "mmlu", + "--fewshot", + "5", + "--prompt_type", + "instruct", + "--num_order", + "0", + "--task_type", + "solve", + "--algo_type", + "single", ], check=True, ) + def prepare_mv_niah_basic(output_folder, tokenizer_type, tokenizer_path, length, dataset_size): subprocess.run( [ - "python", "-m", "nemo_skills.dataset.ruler2.prepare_niah", - "--output_folder", output_folder, - "--tokenizer_type", tokenizer_type, - "--tokenizer_path", tokenizer_path, - "--max_seq_length", length, - "--num_samples", dataset_size, - "--random_seed", "42", - "--num_needle_k", "1", - "--num_needle_v", "4", - "--num_needle_q", "1", - "--type_haystack", "needle", - "--type_needle_k", "words", - "--type_needle_v", "numbers", - "--num_digits_v", "10", + "python", + "-m", + "nemo_skills.dataset.ruler2.prepare_niah", + "--output_folder", + output_folder, + "--tokenizer_type", + tokenizer_type, + "--tokenizer_path", + tokenizer_path, + "--max_seq_length", + length, + "--num_samples", + dataset_size, + "--random_seed", + "42", + "--num_needle_k", + "1", + "--num_needle_v", + "4", + "--num_needle_q", + "1", + "--type_haystack", + "needle", + "--type_needle_k", + "words", + "--type_needle_v", + "numbers", + "--num_digits_v", + "10", ], check=True, ) + def prepare_mv_niah_easy(output_folder, tokenizer_type, tokenizer_path, length, dataset_size): subprocess.run( [ - "python", "-m", "nemo_skills.dataset.ruler2.prepare_mmlu", - "--output_folder", output_folder, - "--tokenizer_type", tokenizer_type, - "--tokenizer_path", tokenizer_path, - "--max_seq_length", length, - "--num_samples", dataset_size, - "--random_seed", "42", - "--dataset", "mmlu", - "--fewshot", "0", - "--prompt_type", "instruct", - "--num_order", "4", - "--task_type", "niah", - "--algo_type", "single", + "python", + "-m", + "nemo_skills.dataset.ruler2.prepare_mmlu", + "--output_folder", + output_folder, + "--tokenizer_type", + tokenizer_type, + "--tokenizer_path", + tokenizer_path, + "--max_seq_length", + length, + "--num_samples", + dataset_size, + "--random_seed", + "42", + "--dataset", + "mmlu", + "--fewshot", + "0", + "--prompt_type", + "instruct", + "--num_order", + "4", + "--task_type", + "niah", + "--algo_type", + "single", ], check=True, ) + def prepare_mv_niah_medium(output_folder, tokenizer_type, tokenizer_path, length, dataset_size): subprocess.run( [ - "python", "-m", "nemo_skills.dataset.ruler2.prepare_mmlu", - "--output_folder", output_folder, - "--tokenizer_type", tokenizer_type, - "--tokenizer_path", tokenizer_path, - "--max_seq_length", length, - "--num_samples", dataset_size, - "--random_seed", "42", - "--dataset", "mmlu", - "--fewshot", "0", - "--prompt_type", "instruct", - "--num_order", "4", - "--task_type", "retrieve", - "--algo_type", "2steps", + "python", + "-m", + "nemo_skills.dataset.ruler2.prepare_mmlu", + "--output_folder", + output_folder, + "--tokenizer_type", + tokenizer_type, + "--tokenizer_path", + tokenizer_path, + "--max_seq_length", + length, + "--num_samples", + dataset_size, + "--random_seed", + "42", + "--dataset", + "mmlu", + "--fewshot", + "0", + "--prompt_type", + "instruct", + "--num_order", + "4", + "--task_type", + "retrieve", + "--algo_type", + "2steps", ], check=True, - ) + ) + def prepare_mv_niah_hard(output_folder, tokenizer_type, tokenizer_path, length, dataset_size): subprocess.run( [ - "python", "-m", "nemo_skills.dataset.ruler2.prepare_mmlu", - "--output_folder", output_folder, - "--tokenizer_type", tokenizer_type, - "--tokenizer_path", tokenizer_path, - "--max_seq_length", length, - "--num_samples", dataset_size, - "--random_seed", "42", - "--dataset", "mmlu", - "--fewshot", "0", - "--prompt_type", "instruct", - "--num_order", "4", - "--task_type", "retrieve", - "--algo_type", "single", + "python", + "-m", + "nemo_skills.dataset.ruler2.prepare_mmlu", + "--output_folder", + output_folder, + "--tokenizer_type", + tokenizer_type, + "--tokenizer_path", + tokenizer_path, + "--max_seq_length", + length, + "--num_samples", + dataset_size, + "--random_seed", + "42", + "--dataset", + "mmlu", + "--fewshot", + "0", + "--prompt_type", + "instruct", + "--num_order", + "4", + "--task_type", + "retrieve", + "--algo_type", + "single", ], check=True, ) @@ -195,37 +315,64 @@ def prepare_mv_niah_hard(output_folder, tokenizer_type, tokenizer_path, length, def prepare_qa_basic(output_folder, tokenizer_type, tokenizer_path, length, dataset_size): subprocess.run( [ - "python", "-m", "nemo_skills.dataset.ruler2.prepare_qa", - "--output_folder", output_folder, - "--tokenizer_type", tokenizer_type, - "--tokenizer_path", tokenizer_path, - "--max_seq_length", length, - "--num_samples", dataset_size, - "--random_seed", "42", - "--dataset", "hotpotqa", - "--fewshot", "0", - "--prompt_type", "instruct", - "--task_type", "retrieve", - "--query_type", "doc", + "python", + "-m", + "nemo_skills.dataset.ruler2.prepare_qa", + "--output_folder", + output_folder, + "--tokenizer_type", + tokenizer_type, + "--tokenizer_path", + tokenizer_path, + "--max_seq_length", + length, + "--num_samples", + dataset_size, + "--random_seed", + "42", + "--dataset", + "hotpotqa", + "--fewshot", + "0", + "--prompt_type", + "instruct", + "--task_type", + "retrieve", + "--query_type", + "doc", ], check=True, ) + def prepare_qa_easy(output_folder, tokenizer_type, tokenizer_path, length, dataset_size): subprocess.run( [ - "python", "-m", "nemo_skills.dataset.ruler2.prepare_qa", - "--output_folder", output_folder, - "--tokenizer_type", tokenizer_type, - "--tokenizer_path", tokenizer_path, - "--max_seq_length", length, - "--num_samples", dataset_size, - "--random_seed", "42", - "--dataset", "hotpotqa", - "--fewshot", "0", - "--prompt_type", "instruct", - "--task_type", "retrieve", - "--query_type", "question", + "python", + "-m", + "nemo_skills.dataset.ruler2.prepare_qa", + "--output_folder", + output_folder, + "--tokenizer_type", + tokenizer_type, + "--tokenizer_path", + tokenizer_path, + "--max_seq_length", + length, + "--num_samples", + dataset_size, + "--random_seed", + "42", + "--dataset", + "hotpotqa", + "--fewshot", + "0", + "--prompt_type", + "instruct", + "--task_type", + "retrieve", + "--query_type", + "question", ], check=True, ) @@ -234,43 +381,69 @@ def prepare_qa_easy(output_folder, tokenizer_type, tokenizer_path, length, datas def prepare_qa_medium(output_folder, tokenizer_type, tokenizer_path, length, dataset_size): subprocess.run( [ - "python", "-m", "nemo_skills.dataset.ruler2.prepare_qa", - "--output_folder", output_folder, - "--tokenizer_type", tokenizer_type, - "--tokenizer_path", tokenizer_path, - "--max_seq_length", length, - "--num_samples", dataset_size, - "--random_seed", "42", - "--dataset", "hotpotqa", - "--fewshot", "0", - "--prompt_type", "instruct", - "--task_type", "solve", - "--algo_type", "2steps", + "python", + "-m", + "nemo_skills.dataset.ruler2.prepare_qa", + "--output_folder", + output_folder, + "--tokenizer_type", + tokenizer_type, + "--tokenizer_path", + tokenizer_path, + "--max_seq_length", + length, + "--num_samples", + dataset_size, + "--random_seed", + "42", + "--dataset", + "hotpotqa", + "--fewshot", + "0", + "--prompt_type", + "instruct", + "--task_type", + "solve", + "--algo_type", + "2steps", ], check=True, ) + def prepare_qa_hard(output_folder, tokenizer_type, tokenizer_path, length, dataset_size): subprocess.run( [ - "python", "-m", "nemo_skills.dataset.ruler2.prepare_qa", - "--output_folder", output_folder, - "--tokenizer_type", tokenizer_type, - "--tokenizer_path", tokenizer_path, - "--max_seq_length", length, - "--num_samples", dataset_size, - "--random_seed", "42", - "--dataset", "hotpotqa", - "--fewshot", "0", - "--prompt_type", "instruct", - "--task_type", "solve", - "--algo_type", "single", + "python", + "-m", + "nemo_skills.dataset.ruler2.prepare_qa", + "--output_folder", + output_folder, + "--tokenizer_type", + tokenizer_type, + "--tokenizer_path", + tokenizer_path, + "--max_seq_length", + length, + "--num_samples", + dataset_size, + "--random_seed", + "42", + "--dataset", + "hotpotqa", + "--fewshot", + "0", + "--prompt_type", + "instruct", + "--task_type", + "solve", + "--algo_type", + "single", ], check=True, ) - def prepare_task_for_ns(output_folder, task): """Adding proper __init__.py""" output_folder = Path(output_folder) / task @@ -291,6 +464,7 @@ def prepare_task_for_ns(output_folder, task): init_file.write(DEFAULT_SETTINGS.format(metrics_type=metrics_type, eval_args=eval_args)) + def prepare_dataset(tasks, setup, max_seq_length, tokenizer_type, tokenizer_path, dataset_size): prepare_task = { "mk_niah_basic": prepare_mk_niah_basic, @@ -307,26 +481,27 @@ def prepare_dataset(tasks, setup, max_seq_length, tokenizer_type, tokenizer_path "qa_hard": prepare_qa_hard, } - output_folder = Path(__file__).parent / setup # 1. installing necessary packages subprocess.run(["pip", "install", "wonderwords", "html2text", "tenacity"], check=True) - for task in tasks: prepare_task_for_ns(output_folder, task) # preparing the datasets based on user options, in parallel with concurrent.futures.ThreadPoolExecutor() as executor: - futures = [executor.submit( - prepare_task[task], - str(output_folder / task), - tokenizer_type, - tokenizer_path, - max_seq_length, - dataset_size - ) for task in tasks] + futures = [ + executor.submit( + prepare_task[task], + str(output_folder / task), + tokenizer_type, + tokenizer_path, + max_seq_length, + dataset_size, + ) + for task in tasks + ] for future in concurrent.futures.as_completed(futures): future.result() # Will raise exception if any subprocess fails @@ -336,6 +511,7 @@ def prepare_dataset(tasks, setup, max_seq_length, tokenizer_type, tokenizer_path benchmarks = ", ".join(f"'ruler2.{setup}.{task}': {{}}" for task in tasks) init_file.write(f"BENCHMARKS = {{{benchmarks}}}\n") + if __name__ == "__main__": parser = argparse.ArgumentParser(description="Prepare RULER2 dataset.") parser.add_argument( @@ -398,4 +574,4 @@ def prepare_dataset(tasks, setup, max_seq_length, tokenizer_type, tokenizer_path args.tokenizer_path, args.dataset_size, ) - print("RULER2 dataset preparation completed.") \ No newline at end of file + print("RULER2 dataset preparation completed.") diff --git a/nemo_skills/dataset/ruler2/prepare_mmlu.py b/nemo_skills/dataset/ruler2/prepare_mmlu.py index 963a54a641..cf6195dd36 100644 --- a/nemo_skills/dataset/ruler2/prepare_mmlu.py +++ b/nemo_skills/dataset/ruler2/prepare_mmlu.py @@ -12,22 +12,22 @@ # See the License for the specific language governing permissions and # limitations under the License -import re -import os -import subprocess -import json import argparse +import json +import logging +import math import random -import numpy as np +import re +from collections import defaultdict from pathlib import Path -from tqdm import tqdm + +import inflect +import numpy as np from datasets import load_dataset +from tqdm import tqdm + from .tokenizer import select_tokenizer -import logging -from collections import defaultdict -import math -import inflect convert = inflect.engine() logging.basicConfig(level=logging.INFO) @@ -35,18 +35,30 @@ parser = argparse.ArgumentParser() # Basic Configurations -parser.add_argument("--output_folder", type=str) -parser.add_argument("--tokenizer_type", type=str, default='hf', help='[Options] nemo, hf, openai.') -parser.add_argument("--tokenizer_path", type=str, required=True, help='path to the tokenizer model') -parser.add_argument("--max_seq_length", type=int, required=True, help='max sequence length including all input tokens and generated tokens.') +parser.add_argument("--output_folder", type=str) +parser.add_argument("--tokenizer_type", type=str, default="hf", help="[Options] nemo, hf, openai.") +parser.add_argument("--tokenizer_path", type=str, required=True, help="path to the tokenizer model") +parser.add_argument( + "--max_seq_length", + type=int, + required=True, + help="max sequence length including all input tokens and generated tokens.", +) parser.add_argument("--random_seed", type=int, default=42) -parser.add_argument("--insert_position", type=float, default=-1, help='insert position of the true context in the context.') -parser.add_argument("--num_samples", type=int, default=None, help='number of samples to generate') +parser.add_argument( + "--insert_position", type=float, default=-1, help="insert position of the true context in the context." +) +parser.add_argument("--num_samples", type=int, default=None, help="number of samples to generate") parser.add_argument("--dataset", type=str, default="gsm8k") parser.add_argument("--fewshot", type=int, default=0) parser.add_argument("--prompt_type", type=str, default="chat") parser.add_argument("--num_order", type=int, default=0) -parser.add_argument("--algo_type", type=str, default="single", choices=["single", "attention","2steps","3steps", "size_2steps", "size_single"]) +parser.add_argument( + "--algo_type", + type=str, + default="single", + choices=["single", "attention", "2steps", "3steps", "size_2steps", "size_single"], +) parser.add_argument("--task_type", type=str, default="retrieve", choices=["retrieve", "solve", "niah"]) args = parser.parse_args() @@ -66,15 +78,25 @@ PROBLEM_PROMPT = "Please first pay attention to all the Question {i} from the context and then only copy the {order}Question {i} in your response. Do not output any other questions." elif args.algo_type == "2steps": # PROBLEM_PROMPT = "Please first find all the Question {i} from the context and then copy the {order}Question {i} at the end." - PROBLEM_PROMPT = "Please first copy all the Question {i} from the context and then copy the {order}Question {i} at the end." + PROBLEM_PROMPT = ( + "Please first copy all the Question {i} from the context and then copy the {order}Question {i} at the end." + ) # PROBLEM_PROMPT = "Please first copy all instances of Question {i} from the context in the order in which they appear, and then copy the {order}Question {i} (1-indexed) at the end." elif args.algo_type == "3steps": PROBLEM_PROMPT = "Please first find how many Question {i} from the context, list them in order, and then copy the {order}Question {i} at the end." elif args.algo_type == "size_2steps": - PROBLEM_PROMPT = "Please first find all the" + str(args.num_order) + " Question {i} from the context and then copy the {order}Question {i} at the end." + PROBLEM_PROMPT = ( + "Please first find all the" + + str(args.num_order) + + " Question {i} from the context and then copy the {order}Question {i} at the end." + ) elif args.algo_type == "size_single": - PROBLEM_PROMPT = "There are " + str(args.num_order) + " Question {i} in the context. Please copy the {order}Question {i} from the context." - + PROBLEM_PROMPT = ( + "There are " + + str(args.num_order) + + " Question {i} in the context. Please copy the {order}Question {i} from the context." + ) + if args.fewshot > 0: EXAMPLE_PROMPT = PROBLEM_PROMPT + "\nQuestion {i}: {question}" @@ -101,13 +123,15 @@ if args.algo_type == "single": PROBLEM_PROMPT = "Please solve the Question {i} from the context with an answer from A, B, C, D." elif args.algo_type == "2steps": - PROBLEM_PROMPT = "Please copy the Question {i} from the context and then solve it with an answer from A, B, C, D." + PROBLEM_PROMPT = ( + "Please copy the Question {i} from the context and then solve it with an answer from A, B, C, D." + ) elif args.dataset == "mbpp": if args.algo_type == "single": PROBLEM_PROMPT = "Please solve the Question {i} from the context by generating or completing code.\nYour answer should be in the following format:\n```python\n# Your code here\n```" elif args.algo_type == "2steps": PROBLEM_PROMPT = "Please copy the Question {i} from the context and then solve it by generating or completing code.\nYour answer should be in the following format:\n```python\n# Your code here\n```" - + if args.fewshot > 0: if args.algo_type == "single": EXAMPLE_PROMPT = PROBLEM_PROMPT + "\nSolution:{solution}" @@ -122,129 +146,156 @@ haystack, needle = [], [] if args.dataset == "gsm8k": test_dataset = load_dataset("openai/gsm8k", "main") - for d in test_dataset['train']: - solution, answer = d['answer'].split("#### ") - haystack.append({ - "Question": d['question'], - "Solution": " " + solution + f"So the answer is \\boxed{{{answer}}}.", - "Answer": answer, - }) - for d in test_dataset['test']: - solution, answer = d['answer'].split("#### ") - needle.append({ - "Question": d['question'], - "Solution": " " + solution + f"So the answer is \\boxed{{{answer}}}.", - "Answer": answer, - }) + for d in test_dataset["train"]: + solution, answer = d["answer"].split("#### ") + haystack.append( + { + "Question": d["question"], + "Solution": " " + solution + f"So the answer is \\boxed{{{answer}}}.", + "Answer": answer, + } + ) + for d in test_dataset["test"]: + solution, answer = d["answer"].split("#### ") + needle.append( + { + "Question": d["question"], + "Solution": " " + solution + f"So the answer is \\boxed{{{answer}}}.", + "Answer": answer, + } + ) elif args.dataset == "math500": questions = set() test_dataset = load_dataset("HuggingFaceH4/MATH-500") - for d in test_dataset['test']: - needle.append({ - "Question": d['problem'], - "Solution": " " + d['solution'], - "Answer": d['answer'], - }) - questions.add(d['problem']) + for d in test_dataset["test"]: + needle.append( + { + "Question": d["problem"], + "Solution": " " + d["solution"], + "Answer": d["answer"], + } + ) + questions.add(d["problem"]) from math_verify import parse - for subject in ['algebra', 'counting_and_probability', 'geometry', 'intermediate_algebra', 'number_theory', 'prealgebra', 'precalculus']: + + for subject in [ + "algebra", + "counting_and_probability", + "geometry", + "intermediate_algebra", + "number_theory", + "prealgebra", + "precalculus", + ]: train_dataset = load_dataset("EleutherAI/hendrycks_math", subject) - for index, d in enumerate(train_dataset['test']): - if d['problem'] not in questions: - haystack.append({ - "Question": d['problem'], - "Solution": " " + d['solution'], - "Answer": parse(d["solution"])[-1], - }) + for index, d in enumerate(train_dataset["test"]): + if d["problem"] not in questions: + haystack.append( + { + "Question": d["problem"], + "Solution": " " + d["solution"], + "Answer": parse(d["solution"])[-1], + } + ) elif args.dataset == "mmlu": test_dataset = load_dataset("cais/mmlu", "all") - options = ['A', 'B', 'C', 'D'] + options = ["A", "B", "C", "D"] haystack = [] needle = [] - for d in test_dataset['test']: + for d in test_dataset["test"]: choices = d["choices"] item = { - "Question": d['question'] + f'\nA. {choices[0]}\nB. {choices[1]}\nC. {choices[2]}\nD. {choices[3]}', - "Solution": " " + f'\\boxed{{{options[d["answer"]]}}}', - "Answer": options[d['answer']], + "Question": d["question"] + f"\nA. {choices[0]}\nB. {choices[1]}\nC. {choices[2]}\nD. {choices[3]}", + "Solution": " " + f"\\boxed{{{options[d['answer']]}}}", + "Answer": options[d["answer"]], } needle.append(item) - for d in test_dataset['auxiliary_train']: + for d in test_dataset["auxiliary_train"]: choices = d["choices"] item = { - "Question": d['question'] + f'\nA. {choices[0]}\nB. {choices[1]}\nC. {choices[2]}\nD. {choices[3]}', - "Solution": " " + f'\\boxed{{{options[d["answer"]]}}}', - "Answer": options[d['answer']], + "Question": d["question"] + f"\nA. {choices[0]}\nB. {choices[1]}\nC. {choices[2]}\nD. {choices[3]}", + "Solution": " " + f"\\boxed{{{options[d['answer']]}}}", + "Answer": options[d["answer"]], } haystack.append(item) elif args.dataset == "mbpp": test_dataset = load_dataset("evalplus/mbppplus") - for d in test_dataset['test']: - prompt = d['prompt'].replace(' ', '\t').strip() - assertion = d['test_list'][0] - needle.append({ - "task_id": f'Mbpp/{d["task_id"]}', - "Question": f"{prompt}\n{assertion}", - "Solution": f"\n```python\n{d['code'].strip()}\n```", - "canonical_solution": f"\n{d['code'].strip()}\n", - "assertion": "\n".join(d['test_list']), - }) + for d in test_dataset["test"]: + prompt = d["prompt"].replace(" ", "\t").strip() + assertion = d["test_list"][0] + needle.append( + { + "task_id": f"Mbpp/{d['task_id']}", + "Question": f"{prompt}\n{assertion}", + "Solution": f"\n```python\n{d['code'].strip()}\n```", + "canonical_solution": f"\n{d['code'].strip()}\n", + "assertion": "\n".join(d["test_list"]), + } + ) train_dataset = load_dataset("google-research-datasets/mbpp", "full") - for d in train_dataset['train']: - prompt = d['text'].replace(' ', '\t').strip() - assertion = d['test_list'][0] - haystack.append({ - "Question": f"{prompt}\n{assertion}", - "Solution": f"\n```python\n{d['code'].strip()}\n```", - "canonical_solution": f"\n{d['code'].strip()}\n", - "assertion": "\n".join(d['test_list']), - }) - for d in train_dataset['validation']: - prompt = d['text'].replace(' ', '\t').strip() - assertion = d['test_list'][0] - haystack.append({ - "Question": f"{prompt}\n{assertion}", - "Solution": f"\n```python\n{d['code'].strip()}\n```", - "canonical_solution": f"\n{d['code'].strip()}\n", - "assertion": "\n".join(d['test_list']), - }) - for d in train_dataset['test']: - prompt = d['text'].replace(' ', '\t').strip() - assertion = d['test_list'][0] - haystack.append({ - "Question": f"{prompt}\n{assertion}", - "Solution": f"\n```python\n{d['code'].strip()}\n```", - "canonical_solution": f"\n{d['code'].strip()}\n", - "assertion": "\n".join(d['test_list']), - }) + for d in train_dataset["train"]: + prompt = d["text"].replace(" ", "\t").strip() + assertion = d["test_list"][0] + haystack.append( + { + "Question": f"{prompt}\n{assertion}", + "Solution": f"\n```python\n{d['code'].strip()}\n```", + "canonical_solution": f"\n{d['code'].strip()}\n", + "assertion": "\n".join(d["test_list"]), + } + ) + for d in train_dataset["validation"]: + prompt = d["text"].replace(" ", "\t").strip() + assertion = d["test_list"][0] + haystack.append( + { + "Question": f"{prompt}\n{assertion}", + "Solution": f"\n```python\n{d['code'].strip()}\n```", + "canonical_solution": f"\n{d['code'].strip()}\n", + "assertion": "\n".join(d["test_list"]), + } + ) + for d in train_dataset["test"]: + prompt = d["text"].replace(" ", "\t").strip() + assertion = d["test_list"][0] + haystack.append( + { + "Question": f"{prompt}\n{assertion}", + "Solution": f"\n```python\n{d['code'].strip()}\n```", + "canonical_solution": f"\n{d['code'].strip()}\n", + "assertion": "\n".join(d["test_list"]), + } + ) else: raise ValueError(f"Dataset {args.dataset} is not supported.") for item in needle: - item["Question"] = re.sub(r'\s+', ' ', item["Question"]) + item["Question"] = re.sub(r"\s+", " ", item["Question"]) for item in haystack: - item["Question"] = re.sub(r'\s+', ' ', item["Question"]) + item["Question"] = re.sub(r"\s+", " ", item["Question"]) + +logger.info(f"Dataset size: {len(needle)}") -logger.info(f'Dataset size: {len(needle)}') def generate_random_number(num_digits=7): - lower_bound = 10**(num_digits - 1) + lower_bound = 10 ** (num_digits - 1) upper_bound = 10**num_digits - 1 return str(random.randint(lower_bound, upper_bound)) + def generate_input_output(index, num_qs): if num_qs > len(haystack): repeats = (num_qs + len(haystack) - 1) // len(haystack) # Ceiling division else: repeats = 1 - + curr_context = [dict(item) for item in random.sample([item for item in haystack for _ in range(repeats)], num_qs)] if args.num_order > 0: @@ -254,8 +305,8 @@ def generate_input_output(index, num_qs): random_numbers = [generate_random_number() for _ in range(num_qs + 1)] random.shuffle(random_numbers) - random_numbers = random_numbers[:num_qs+1] - for i,q in enumerate(curr_context): + random_numbers = random_numbers[: num_qs + 1] + for i, q in enumerate(curr_context): q["random_index"] = random_numbers[i] random.shuffle(curr_context) @@ -267,17 +318,19 @@ def generate_input_output(index, num_qs): insert_position = random.randint(0, len(curr_context)) else: insert_position = int(args.insert_position * len(curr_context)) - curr_context.insert(insert_position,true_context) + curr_context.insert(insert_position, true_context) counts = defaultdict(int) - for i,q in enumerate(curr_context): + for i, q in enumerate(curr_context): counts[q["random_index"]] += 1 if args.num_order > 0: q["order"] = convert.ordinal(counts[q["random_index"]]) + " (1 indexed) " else: q["order"] = "" - needles = '\n\n'.join([NEEDLE_PROMPT.format(i=q["random_index"], question=q["Question"]) for i, q in enumerate(curr_context)]) + needles = "\n\n".join( + [NEEDLE_PROMPT.format(i=q["random_index"], question=q["Question"]) for i, q in enumerate(curr_context)] + ) if args.task_type == "niah": problem = PROBLEM_PROMPT.format(i=true_context["random_index"]) else: @@ -285,12 +338,24 @@ def generate_input_output(index, num_qs): if args.fewshot > 0: if args.task_type == "retrieve": - example = '\n\n'.join([EXAMPLE_PROMPT.format(i=q["random_index"], question=q["Question"], order=q["order"]) for q in examples]) + example = "\n\n".join( + [ + EXAMPLE_PROMPT.format(i=q["random_index"], question=q["Question"], order=q["order"]) + for q in examples + ] + ) elif args.task_type == "solve": if args.algo_type == "single": - example = '\n\n'.join([EXAMPLE_PROMPT.format(i=q["random_index"], solution=q["Solution"]) for q in examples]) + example = "\n\n".join( + [EXAMPLE_PROMPT.format(i=q["random_index"], solution=q["Solution"]) for q in examples] + ) elif args.algo_type == "2steps": - example = '\n\n'.join([EXAMPLE_PROMPT.format(i=q["random_index"], question=q["Question"], solution=q["Solution"]) for q in examples]) + example = "\n\n".join( + [ + EXAMPLE_PROMPT.format(i=q["random_index"], question=q["Question"], solution=q["Solution"]) + for q in examples + ] + ) if args.prompt_type == "base": example = f"{example}\n\n" @@ -299,21 +364,20 @@ def generate_input_output(index, num_qs): else: example = "" - context = CONTEXT_PROMPT.format(needles=needles) input_text = TOTAL_PROMPT.format( context=context, problem=problem, example=example, ) - + if args.task_type == "retrieve": - expected_answer = { - "expected_answer" : [true_context["Question"]] - } + expected_answer = {"expected_answer": [true_context["Question"]]} elif args.task_type == "niah": expected_answer = { - "expected_answer" : [c["Question"] for c in curr_context if c["random_index"] == true_context["random_index"]] + "expected_answer": [ + c["Question"] for c in curr_context if c["random_index"] == true_context["random_index"] + ] } elif args.task_type == "solve": if args.dataset == "mbpp": @@ -323,11 +387,9 @@ def generate_input_output(index, num_qs): "canonical_solution": true_context["canonical_solution"], } else: - expected_answer = { - "expected_answer" : true_context["Answer"] - } + expected_answer = {"expected_answer": true_context["Answer"]} - save_dict = { + save_dict = { "index": index, "question": f"{context}\n\n{example}{problem}", **expected_answer, @@ -336,7 +398,6 @@ def generate_input_output(index, num_qs): def generate_samples(max_seq_length: int, incremental: int = 10): - write_jsons = [] # Estimate tokens per question to determine reasonable upper bound @@ -373,7 +434,7 @@ def generate_samples(max_seq_length: int, incremental: int = 10): upper_bound = mid - 1 num_qs = optimal_num_qs if optimal_num_qs is not None else incremental - logger.info(f'Final optimal haystack size (number of questions): {num_qs}') + logger.info(f"Final optimal haystack size (number of questions): {num_qs}") if args.num_samples is not None: needle_sample = random.sample(list(range(len(needle))), min(len(needle), args.num_samples)) @@ -389,11 +450,11 @@ def generate_samples(max_seq_length: int, incremental: int = 10): length = len(TOKENIZER.text_to_tokens(input_text)) assert length <= max_seq_length, f"{length} exceeds max_seq_length." break -+ except AssertionError: -+ if used_qs > incremental: -+ used_qs -= incremental -+ else: -+ raise + except AssertionError: + if used_qs > incremental: + used_qs -= incremental + else: + raise save_dict["length"] = length formatted_output = save_dict @@ -405,14 +466,12 @@ def generate_samples(max_seq_length: int, incremental: int = 10): def main(): output_file = Path(args.output_folder) / "test.jsonl" - write_jsons = generate_samples( - max_seq_length=args.max_seq_length, - incremental=max(10, args.fewshot) - ) + write_jsons = generate_samples(max_seq_length=args.max_seq_length, incremental=max(10, args.fewshot)) with open(output_file, "wt", encoding="utf-8") as fout: for entry in write_jsons: fout.write(json.dumps(entry) + "\n") -if __name__=="__main__": + +if __name__ == "__main__": main() diff --git a/nemo_skills/dataset/ruler2/prepare_niah.py b/nemo_skills/dataset/ruler2/prepare_niah.py index 6fab5d3b7e..8d12a33ce6 100644 --- a/nemo_skills/dataset/ruler2/prepare_niah.py +++ b/nemo_skills/dataset/ruler2/prepare_niah.py @@ -12,46 +12,51 @@ # See the License for the specific language governing permissions and # limitations under the License -import os -import re -import json -import uuid import argparse +import json +import math import random +import uuid +from pathlib import Path + import nltk -import math import numpy as np import wonderwords -from pathlib import Path from tqdm import tqdm + from .tokenizer import select_tokenizer -from nltk.tokenize import sent_tokenize + try: - nltk.data.find('tokenizers/punkt') - nltk.data.find('tokenizers/punkt_tab') + nltk.data.find("tokenizers/punkt") + nltk.data.find("tokenizers/punkt_tab") except LookupError: - nltk.download('punkt') - nltk.download('punkt_tab') + nltk.download("punkt") + nltk.download("punkt_tab") import logging logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) parser = argparse.ArgumentParser() -parser.add_argument("--output_folder", type=str) -parser.add_argument("--tokenizer_type", type=str, default='hf', help='[Options] nemo, hf, openai.') -parser.add_argument("--tokenizer_path", type=str, required=True, help='path to the tokenizer model') -parser.add_argument("--max_seq_length", type=int, required=True, help='max sequence length including all input tokens and generated tokens.') -parser.add_argument("--num_samples", type=int, required=True, help='number of samples to generate') +parser.add_argument("--output_folder", type=str) +parser.add_argument("--tokenizer_type", type=str, default="hf", help="[Options] nemo, hf, openai.") +parser.add_argument("--tokenizer_path", type=str, required=True, help="path to the tokenizer model") +parser.add_argument( + "--max_seq_length", + type=int, + required=True, + help="max sequence length including all input tokens and generated tokens.", +) +parser.add_argument("--num_samples", type=int, required=True, help="number of samples to generate") parser.add_argument("--random_seed", type=int, default=42) # Complexity Configurations parser.add_argument("--num_needle_k", type=int, default=1) parser.add_argument("--num_needle_v", type=int, default=1) parser.add_argument("--num_needle_q", type=int, default=1) -parser.add_argument("--type_haystack", type=str, default='needle', help='[Options] needle.') -parser.add_argument("--type_needle_k", type=str, default='words', help='[Options] numbers, words, uuids.') -parser.add_argument("--type_needle_v", type=str, default='numbers', help='[Options] numbers, words, uuids.') +parser.add_argument("--type_haystack", type=str, default="needle", help="[Options] needle.") +parser.add_argument("--type_needle_k", type=str, default="words", help="[Options] numbers, words, uuids.") +parser.add_argument("--type_needle_v", type=str, default="numbers", help="[Options] numbers, words, uuids.") parser.add_argument("--num_digits_k", type=int, default=7) parser.add_argument("--num_digits_v", type=int, default=7) @@ -66,12 +71,12 @@ TEMPLATE_SINGLE = """A special magic {type_needle_v} is hidden within the following text. Make sure to memorize it. I will quiz you about the {type_needle_v} afterwards.\n{context}\nWhat is the special magic {type_needle_v} for {query} mentioned in the provided text? The special magic {type_needle_v} for {query} mentioned in the provided text is""" TEMPLATE_MULTIPLE = """Some special magic {type_needle_v} are hidden within the following text. Make sure to memorize them. I will quiz you about the {type_needle_v} afterwards.\n{context}\nWhat are all the special magic {type_needle_v} for {query} mentioned in the provided text? The special magic {type_needle_v} for {query} mentioned in the provided text are""" -# Define Needle/Haystack Format +# Define Needle/Haystack Format needle = "One of the special magic {type_needle_v} for {key} is: {value}." -if args.type_haystack == 'needle': +if args.type_haystack == "needle": haystack = needle else: - raise NotImplementedError(f'{args.type_haystack} is not implemented.') + raise NotImplementedError(f"{args.type_haystack} is not implemented.") # Words nouns = wonderwords.random_word._get_words_from_text_file("nounlist.txt") @@ -82,29 +87,34 @@ # Positions DEPTHS = list(np.round(np.linspace(0, 100, num=40, endpoint=True)).astype(int)) + def generate_random_number(num_digits=7): - lower_bound = 10**(num_digits - 1) + lower_bound = 10 ** (num_digits - 1) upper_bound = 10**num_digits - 1 return str(random.randint(lower_bound, upper_bound)) + def generate_random_word(): word = random.choice(words) return word + def generate_random_uuid(): return str(uuid.UUID(int=random.getrandbits(128), version=4)) + def generate_random(type_needle: str, digits: int | None = None): - if type_needle == 'numbers': + if type_needle == "numbers": if digits is None: raise ValueError("digits must be provided when type_needle='numbers'") return generate_random_number(digits) - elif type_needle == 'words': + elif type_needle == "words": return generate_random_word() - elif type_needle == 'uuids': + elif type_needle == "uuids": return generate_random_uuid() else: - raise NotImplementedError(f'{type_needle} is not implemented.') + raise NotImplementedError(f"{type_needle} is not implemented.") + def generate_input_output(num_haystack): keys, values, needles = [], [], [] @@ -113,30 +123,44 @@ def generate_input_output(num_haystack): value = [] for _ in range(args.num_needle_v): value.append(generate_random(args.type_needle_v, args.num_digits_v)) - needles.append(needle.format( - type_needle_v=args.type_needle_v, - key=keys[-1], - value=value[-1], - )) + needles.append( + needle.format( + type_needle_v=args.type_needle_v, + key=keys[-1], + value=value[-1], + ) + ) values.append(value) - + random.shuffle(needles) - + # Context if args.num_needle_v == 1: - sentences = [haystack.format( - type_needle_v=args.type_needle_v, - key=generate_random(args.type_needle_k, args.num_digits_k), - value=generate_random(args.type_needle_v, args.num_digits_v), - ) for _ in range(num_haystack)] + sentences = [ + haystack.format( + type_needle_v=args.type_needle_v, + key=generate_random(args.type_needle_k, args.num_digits_k), + value=generate_random(args.type_needle_v, args.num_digits_v), + ) + for _ in range(num_haystack) + ] else: haystack_values = [generate_random(args.type_needle_v, args.num_digits_v) for _ in range(num_haystack)] - haystack_keys = ([generate_random(args.type_needle_k, args.num_digits_k) for _ in range(math.ceil(num_haystack / args.num_needle_v))] * args.num_needle_v)[:num_haystack] - sentences = [haystack.format( - type_needle_v=args.type_needle_v, - key=haystack_keys[i], - value=haystack_values[i], - ) for i in range(num_haystack)] + haystack_keys = ( + [ + generate_random(args.type_needle_k, args.num_digits_k) + for _ in range(math.ceil(num_haystack / args.num_needle_v)) + ] + * args.num_needle_v + )[:num_haystack] + sentences = [ + haystack.format( + type_needle_v=args.type_needle_v, + key=haystack_keys[i], + value=haystack_values[i], + ) + for i in range(num_haystack) + ] random.shuffle(sentences) indexes = sorted(random.sample(range(num_haystack), len(needles)), reverse=True) @@ -144,20 +168,19 @@ def generate_input_output(num_haystack): sentences.insert(index, element) context = "\n".join(sentences) - ## Query and Answer indices = random.sample(range(args.num_needle_k), args.num_needle_q) queries = [keys[i] for i in indices] answers = [a for i in indices for a in values[i]] - query = ', '.join(queries[:-1]) + ', and ' + queries[-1] if len(queries) > 1 else queries[0] - + query = ", ".join(queries[:-1]) + ", and " + queries[-1] if len(queries) > 1 else queries[0] + if args.num_needle_q * args.num_needle_v == 1: template = TEMPLATE_SINGLE - type_needle_v = args.type_needle_v[:-1] # remove "s" + type_needle_v = args.type_needle_v[:-1] # remove "s" else: template = TEMPLATE_MULTIPLE type_needle_v = args.type_needle_v - + input_text = template.format( type_needle_v=type_needle_v, context=context, @@ -170,9 +193,9 @@ def generate_input_output(num_haystack): def generate_samples(num_samples: int, max_seq_length: int, incremental: int = 500): write_jsons = [] - if args.type_haystack == 'needle': + if args.type_haystack == "needle": incremental = max(5, args.num_needle_v * args.num_needle_k) - + if args.max_seq_length < 4096: incremental = 5 @@ -209,14 +232,14 @@ def generate_samples(num_samples: int, max_seq_length: int, incremental: int = 5 upper_bound = mid - 1 num_haystack = optimal_num_haystack if optimal_num_haystack is not None else incremental - logger.info(f'Final optimal haystack size (number of haystack): {num_haystack}') - + logger.info(f"Final optimal haystack size (number of haystack): {num_haystack}") + # Generate samples for index in tqdm(range(num_samples)): used_haystack = num_haystack while True: try: - input_text, answer = generate_input_output(used_haystack) + input_text, answer = generate_input_output(used_haystack) length = len(TOKENIZER.text_to_tokens(input_text)) assert length <= max_seq_length, f"{length} exceeds max_seq_length." break @@ -227,7 +250,7 @@ def generate_samples(num_samples: int, max_seq_length: int, incremental: int = 5 raise formatted_output = { - 'index': index, + "index": index, "question": input_text, "expected_answer": answer, "length": length, @@ -241,12 +264,13 @@ def main(): output_file = Path(args.output_folder) / "test.jsonl" write_jsons = generate_samples( - num_samples=args.num_samples, + num_samples=args.num_samples, max_seq_length=args.max_seq_length, ) with open(output_file, "wt", encoding="utf-8") as fout: for entry in write_jsons: fout.write(json.dumps(entry) + "\n") + if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/nemo_skills/dataset/ruler2/prepare_qa.py b/nemo_skills/dataset/ruler2/prepare_qa.py index d1096a615c..32289e1bca 100644 --- a/nemo_skills/dataset/ruler2/prepare_qa.py +++ b/nemo_skills/dataset/ruler2/prepare_qa.py @@ -12,32 +12,36 @@ # See the License for the specific language governing permissions and # limitations under the License -import os -import re -import json import argparse -from pathlib import Path -from tqdm import tqdm +import json +import logging import random -import numpy as np -import subprocess -from datasets import load_dataset from collections import defaultdict +from pathlib import Path + +import numpy as np +from datasets import load_dataset +from tqdm import tqdm + from .tokenizer import select_tokenizer -import logging logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) parser = argparse.ArgumentParser() # Basic Configurations -parser.add_argument("--output_folder", type=str) -parser.add_argument("--tokenizer_type", type=str, default='hf', help='[Options] nemo, hf, openai.') -parser.add_argument("--tokenizer_path", type=str, required=True, help='path to the tokenizer model') -parser.add_argument("--max_seq_length", type=int, required=True, help='max sequence length including all input tokens and generated tokens.') +parser.add_argument("--output_folder", type=str) +parser.add_argument("--tokenizer_type", type=str, default="hf", help="[Options] nemo, hf, openai.") +parser.add_argument("--tokenizer_path", type=str, required=True, help="path to the tokenizer model") +parser.add_argument( + "--max_seq_length", + type=int, + required=True, + help="max sequence length including all input tokens and generated tokens.", +) parser.add_argument("--random_seed", type=int, default=42) parser.add_argument("--num_samples", type=int, default=500) -parser.add_argument("--dataset", type=str, required=True, help='dataset file') +parser.add_argument("--dataset", type=str, required=True, help="dataset file") parser.add_argument("--fewshot", type=int, default=0) parser.add_argument("--prompt_type", type=str, default="chat") parser.add_argument("--query_type", type=str, default="id", choices=["id", "doc", "question"]) @@ -71,7 +75,9 @@ # CONTEXT_PROMPT = """Below are some documents. I will give you a text at the end. Please find the document index most relevant to the text. Only give me the index without any document contents.\n\n{needles}""" # PROBLEM_PROMPT = "Text: {question}\nMost relevant document index:" CONTEXT_PROMPT = """Below are some documents. I will give you a question at the end. Please find the index of the most relevant document that can help answer the question. Only give me the index without any document contents.\n\n{needles}""" - PROBLEM_PROMPT = "Question: {question}\nIndex of the most relevant document that can help answer the question:" + PROBLEM_PROMPT = ( + "Question: {question}\nIndex of the most relevant document that can help answer the question:" + ) if args.fewshot > 0: EXAMPLE_PROMPT = PROBLEM_PROMPT + " {i}" elif args.task_type == "solve": @@ -90,87 +96,103 @@ # Read SQuAD QA dataset def read_squad(): data = load_dataset("squad_v2")["train"] - haystack = [d['context'] for d in data] + haystack = [d["context"] for d in data] haystack = list(set(haystack)) - haystack = [{ - "text": d - } for d in haystack] - + haystack = [{"text": d} for d in haystack] data = load_dataset("squad_v2")["validation"] title2context = defaultdict(set) for d in data: - title2context[d['title']].add(d['context']) - - needle = [{ - "question": d['question'], - "answer": d['answers']['text'], - "context": [{"text": d['context']}], - "distractor": [{"text": t} for t in title2context[d['title']] if t != d['context']] - } for d in data] + title2context[d["title"]].add(d["context"]) + + needle = [ + { + "question": d["question"], + "answer": d["answers"]["text"], + "context": [{"text": d["context"]}], + "distractor": [{"text": t} for t in title2context[d["title"]] if t != d["context"]], + } + for d in data + ] needle = [n for n in needle if len(n["answer"]) > 0] - + return haystack, needle + # Read Hotpot QA dataset def read_hotpotqa(): - data = load_dataset(f"hotpotqa/hotpot_qa", "distractor")["train"] - haystack = [f"{t}\n{''.join(s)}" for d in data for t, s in zip(d['context']['title'], d['context']['sentences'])] + data = load_dataset("hotpotqa/hotpot_qa", "distractor")["train"] + haystack = [f"{t}\n{''.join(s)}" for d in data for t, s in zip(d["context"]["title"], d["context"]["sentences"])] haystack = list(set(haystack)) - haystack = [{ - "text": d - } for d in haystack] - - data = load_dataset(f"hotpotqa/hotpot_qa", "distractor")["validation"] - needle = [{ - "question": d['question'], - "answer": [d['answer']], - "context": [{"text": f"{t}\n{''.join(s)}"} for t, s in zip(d['context']['title'], d['context']['sentences']) if t in d['supporting_facts']['title']], - "distractor": [{"text": f"{t}\n{''.join(s)}"} for t, s in zip(d['context']['title'], d['context']['sentences']) if t not in d['supporting_facts']['title']] - } for d in data] + haystack = [{"text": d} for d in haystack] + + data = load_dataset("hotpotqa/hotpot_qa", "distractor")["validation"] + needle = [ + { + "question": d["question"], + "answer": [d["answer"]], + "context": [ + {"text": f"{t}\n{''.join(s)}"} + for t, s in zip(d["context"]["title"], d["context"]["sentences"]) + if t in d["supporting_facts"]["title"] + ], + "distractor": [ + {"text": f"{t}\n{''.join(s)}"} + for t, s in zip(d["context"]["title"], d["context"]["sentences"]) + if t not in d["supporting_facts"]["title"] + ], + } + for d in data + ] needle = [n for n in needle if len(n["answer"]) > 0] - + return haystack, needle def read_musique(): data = load_dataset("dgslibisey/MuSiQue")["train"] - haystack = [f"{p['title']}\n{p['paragraph_text']}" for d in data for p in d['paragraphs']] + haystack = [f"{p['title']}\n{p['paragraph_text']}" for d in data for p in d["paragraphs"]] haystack = list(set(haystack)) - haystack = [{ - "text": d - } for d in haystack] + haystack = [{"text": d} for d in haystack] data = load_dataset("dgslibisey/MuSiQue")["validation"] - needle = [{ - "question": d['question'], - "answer": [d['answer']] + d['answer_aliases'], - "context": [{"text": f"{p['title']}\n{p['paragraph_text']}"} for p in d['paragraphs'] if p['is_supporting']], - "distractor": [{"text": f"{p['title']}\n{p['paragraph_text']}"} for p in d['paragraphs'] if not p['is_supporting']] - } for d in data if d['answerable']] + needle = [ + { + "question": d["question"], + "answer": [d["answer"]] + d["answer_aliases"], + "context": [ + {"text": f"{p['title']}\n{p['paragraph_text']}"} for p in d["paragraphs"] if p["is_supporting"] + ], + "distractor": [ + {"text": f"{p['title']}\n{p['paragraph_text']}"} for p in d["paragraphs"] if not p["is_supporting"] + ], + } + for d in data + if d["answerable"] + ] needle = [n for n in needle if len(n["answer"]) > 0] return haystack, needle + # Download dataset -if args.dataset == 'squad': +if args.dataset == "squad": haystack, needle = read_squad() -elif args.dataset == 'hotpotqa': +elif args.dataset == "hotpotqa": haystack, needle = read_hotpotqa() -elif args.dataset == 'musique': +elif args.dataset == "musique": haystack, needle = read_musique() else: - raise NotImplementedError(f'{args.dataset} is not implemented.') + raise NotImplementedError(f"{args.dataset} is not implemented.") def generate_random_number(num_digits=7): - lower_bound = 10**(num_digits - 1) + lower_bound = 10 ** (num_digits - 1) upper_bound = 10**num_digits - 1 return str(random.randint(lower_bound, upper_bound)) def generate_input_output(index, num_docs): - curr_needle = dict(needle[index]) curr_needle["context"] = [{**c, "random_index": generate_random_number()} for c in curr_needle["context"]] curr_needle["distractor"] = [{**c, "random_index": generate_random_number()} for c in curr_needle["distractor"]] @@ -179,12 +201,17 @@ def generate_input_output(index, num_docs): fewshot_examples = random.sample([i for i in range(len(needle)) if i != index], args.fewshot) fewshot_examples = [dict(needle[i]) for i in fewshot_examples] for e in fewshot_examples: - e["context"] = [{**c, "random_index": generate_random_number()} for c in e['context']] - e["distractor"] = [{**c, "random_index": generate_random_number()} for c in e['distractor']] + e["context"] = [{**c, "random_index": generate_random_number()} for c in e["context"]] + e["distractor"] = [{**c, "random_index": generate_random_number()} for c in e["distractor"]] else: fewshot_examples = [] - remaining_haystack_size = len(haystack) - len(set([c["text"] for c in (curr_needle["context"] + curr_needle["distractor"])] + [f["text"] for e in fewshot_examples for f in (e["context"] + e["distractor"])])) + remaining_haystack_size = len(haystack) - len( + set( + [c["text"] for c in (curr_needle["context"] + curr_needle["distractor"])] + + [f["text"] for e in fewshot_examples for f in (e["context"] + e["distractor"])] + ) + ) if remaining_haystack_size <= 0: raise ValueError("No remaining haystack documents available after exclusions.") @@ -195,12 +222,18 @@ def generate_input_output(index, num_docs): curr_context = random.sample([i for i in range(len(haystack)) for _ in range(repeats)], num_docs) curr_context = [{**haystack[i], "random_index": generate_random_number()} for i in curr_context] - curr_context = curr_context + curr_needle["context"] + [item for example in fewshot_examples for item in example["context"]] + curr_context = ( + curr_context + curr_needle["context"] + [item for example in fewshot_examples for item in example["context"]] + ) if num_docs > 0: - curr_context = curr_context + curr_needle["distractor"] + [item for example in fewshot_examples for item in example["distractor"]] + curr_context = ( + curr_context + + curr_needle["distractor"] + + [item for example in fewshot_examples for item in example["distractor"]] + ) random.shuffle(curr_context) - needles = '\n\n'.join([NEEDLE_PROMPT.format(i=c["random_index"], text=c["text"]) for c in curr_context]) + needles = "\n\n".join([NEEDLE_PROMPT.format(i=c["random_index"], text=c["text"]) for c in curr_context]) if args.task_type == "retrieve": if args.query_type == "id": problem = PROBLEM_PROMPT.format(i=curr_needle["context"][0]["random_index"]) @@ -211,51 +244,59 @@ def generate_input_output(index, num_docs): elif args.task_type == "solve": problem = PROBLEM_PROMPT.format(question=curr_needle["question"]) - if args.fewshot > 0: if args.task_type == "retrieve": if args.query_type == "id": - example = '\n\n'.join([EXAMPLE_PROMPT.format(i=e["context"][0]["random_index"], text=e["context"][0]["text"]) for e in fewshot_examples]) + example = "\n\n".join( + [ + EXAMPLE_PROMPT.format(i=e["context"][0]["random_index"], text=e["context"][0]["text"]) + for e in fewshot_examples + ] + ) elif args.query_type == "doc": - example = '\n\n'.join([EXAMPLE_PROMPT.format(i=e["context"][0]["random_index"], text=e["context"][0]["text"]) for e in fewshot_examples]) + example = "\n\n".join( + [ + EXAMPLE_PROMPT.format(i=e["context"][0]["random_index"], text=e["context"][0]["text"]) + for e in fewshot_examples + ] + ) elif args.query_type == "question": - example = '\n\n'.join([EXAMPLE_PROMPT.format(i=random.sample(e["context"], 1)[0]["random_index"], question=e["question"]) for e in fewshot_examples]) + example = "\n\n".join( + [ + EXAMPLE_PROMPT.format( + i=random.sample(e["context"], 1)[0]["random_index"], question=e["question"] + ) + for e in fewshot_examples + ] + ) elif args.task_type == "solve": - example = '\n\n'.join([EXAMPLE_PROMPT.format(answer=random.sample(e["answer"], 1)[0], question=e["question"]) for e in fewshot_examples]) + example = "\n\n".join( + [ + EXAMPLE_PROMPT.format(answer=random.sample(e["answer"], 1)[0], question=e["question"]) + for e in fewshot_examples + ] + ) if args.prompt_type == "base": example = f"{example}\n\n" else: example = f"Here are some examples to help you understand the task:\n\n{example}\n\nHere is the actual task you need to solve:\n\n" - + else: example = "" - context = CONTEXT_PROMPT.format(needles=needles) - input_text = TOTAL_PROMPT.format( - context=context, - problem=problem, - example=example - ) + input_text = TOTAL_PROMPT.format(context=context, problem=problem, example=example) if args.task_type == "retrieve": if args.query_type == "id": - expected_answer = { - "expected_answer" : [curr_needle["context"][0]["text"]] - } + expected_answer = {"expected_answer": [curr_needle["context"][0]["text"]]} elif args.query_type == "doc": - expected_answer = { - "expected_answer" : [curr_needle["context"][0]["random_index"]] - } + expected_answer = {"expected_answer": [curr_needle["context"][0]["random_index"]]} elif args.query_type == "question": - expected_answer = { - "expected_answer" : [c["random_index"] for c in curr_needle["context"]] - } + expected_answer = {"expected_answer": [c["random_index"] for c in curr_needle["context"]]} elif args.task_type == "solve": - expected_answer = { - "expected_answer" : curr_needle["answer"] - } + expected_answer = {"expected_answer": curr_needle["answer"]} save_dict = { "index": index, @@ -264,10 +305,10 @@ def generate_input_output(index, num_docs): } return input_text, save_dict -def generate_samples(num_samples: int, max_seq_length: int, incremental: int = 5): - + +def generate_samples(num_samples: int, max_seq_length: int, incremental: int = 5): write_jsons = [] - + # Estimate tokens per question to determine reasonable upper bound sample_input_text, _ = generate_input_output(0, incremental) sample_tokens = len(TOKENIZER.text_to_tokens(sample_input_text)) @@ -303,7 +344,7 @@ def generate_samples(num_samples: int, max_seq_length: int, incremental: int = 5 upper_bound = mid - 1 num_docs = optimal_num_docs if optimal_num_docs is not None else incremental - logger.info(f'Final optimal haystack size (number of docs): {num_docs}') + logger.info(f"Final optimal haystack size (number of docs): {num_docs}") else: num_docs = 0 @@ -334,12 +375,13 @@ def main(): output_file = Path(args.output_folder) / "test.jsonl" write_jsons = generate_samples( - num_samples=args.num_samples, - max_seq_length=args.max_seq_length, + num_samples=args.num_samples, + max_seq_length=args.max_seq_length, ) with open(output_file, "wt", encoding="utf-8") as fout: for entry in write_jsons: fout.write(json.dumps(entry) + "\n") -if __name__=="__main__": + +if __name__ == "__main__": main() diff --git a/nemo_skills/dataset/ruler2/ruler2_score.py b/nemo_skills/dataset/ruler2/ruler2_score.py index 14d4ed6d97..7bc01c1dc1 100644 --- a/nemo_skills/dataset/ruler2/ruler2_score.py +++ b/nemo_skills/dataset/ruler2/ruler2_score.py @@ -29,14 +29,19 @@ def compute_score(metrics: dict): "qa_easy", "qa_medium", "qa_hard", - ] setup = list(metrics.keys())[0].rsplit(".", 1)[0] metrics[setup] = {} for aggregation in metrics[f"{setup}.mk_niah_basic"]: metrics[setup][aggregation] = { - "accuracy": sum(metrics[f"{setup}.{task}"][aggregation].get("accuracy", (metrics[f"{setup}.{task}"][aggregation].get("symbolic_correct", 0))) for task in tasks) / len(tasks) + "accuracy": sum( + metrics[f"{setup}.{task}"][aggregation].get( + "accuracy", (metrics[f"{setup}.{task}"][aggregation].get("symbolic_correct", 0)) + ) + for task in tasks + ) + / len(tasks) } return metrics diff --git a/nemo_skills/dataset/ruler2/tokenizer.py b/nemo_skills/dataset/ruler2/tokenizer.py index 30b2a8b66f..f339c47964 100644 --- a/nemo_skills/dataset/ruler2/tokenizer.py +++ b/nemo_skills/dataset/ruler2/tokenizer.py @@ -15,20 +15,21 @@ import os from typing import List + from tenacity import ( retry, stop_after_attempt, wait_fixed, wait_random, -) +) def select_tokenizer(tokenizer_type, tokenizer_path): - if tokenizer_type == 'hf': + if tokenizer_type == "hf": return HFTokenizer(model_path=tokenizer_path) - elif tokenizer_type == 'openai': + elif tokenizer_type == "openai": return OpenAITokenizer(model_path=tokenizer_path) - elif tokenizer_type == 'gemini': + elif tokenizer_type == "gemini": return GeminiTokenizer(model_path=tokenizer_path) else: raise ValueError(f"Unknown tokenizer_type {tokenizer_type}") @@ -38,10 +39,12 @@ class HFTokenizer: """ Tokenizer from HF models """ + def __init__(self, model_path) -> None: from transformers import AutoTokenizer + self.tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True) - + def text_to_tokens(self, text: str) -> List[str]: tokens = self.tokenizer.tokenize(text) return tokens @@ -55,8 +58,10 @@ class OpenAITokenizer: """ Tokenizer from tiktoken """ + def __init__(self, model_path="cl100k_base") -> None: import tiktoken + self.tokenizer = tiktoken.get_encoding(model_path) def text_to_tokens(self, text: str) -> List[int]: @@ -72,11 +77,13 @@ class GeminiTokenizer: """ Tokenizer from gemini """ + def __init__(self, model_path="gemini-1.5-pro-latest") -> None: import google.generativeai as genai + genai.configure(api_key=os.environ["GEMINI_API_KEY"]) self.model = genai.GenerativeModel(model_path) - + @retry(wait=wait_fixed(60) + wait_random(0, 10), stop=stop_after_attempt(3)) def text_to_tokens(self, text: str) -> List[int]: tokens = list(range(self.model.count_tokens(text).total_tokens)) diff --git a/nemo_skills/evaluation/evaluator/ruler.py b/nemo_skills/evaluation/evaluator/ruler.py index e3e9e1173b..d89ff1958d 100644 --- a/nemo_skills/evaluation/evaluator/ruler.py +++ b/nemo_skills/evaluation/evaluator/ruler.py @@ -16,8 +16,8 @@ import logging import os import re -import editdistance +import editdistance from tqdm import tqdm from nemo_skills.evaluation.evaluator.base import BaseEvaluatorConfig @@ -95,7 +95,6 @@ def default_parse(prediction): def post_process_preds(preds): return preds - def wer(hypotheses: list[str], references: list[str]) -> float: scores = 0 words = 0 @@ -113,7 +112,7 @@ def wer(hypotheses: list[str], references: list[str]) -> float: if words != 0: wer = 1.0 * scores / words else: - wer = float('inf') + wer = float("inf") return wer def string_match_all_single(preds, refs): @@ -122,7 +121,9 @@ def string_match_all_single(preds, refs): preds = [preds] refs = [refs] score = [ - sum([max(1.0 if r.lower() in pred.lower() else 0.0, 1 - wer([pred.lower()], [r.lower()])) for r in ref]) / len(ref) for pred, ref in zip(preds, refs) + sum([max(1.0 if r.lower() in pred.lower() else 0.0, 1 - wer([pred.lower()], [r.lower()])) for r in ref]) + / len(ref) + for pred, ref in zip(preds, refs) ][0] return score @@ -132,18 +133,30 @@ def string_match_2steps_single(preds, refs): preds = [preds] refs = [refs] score = [ - sum([max(1.0 if r.lower() in pred.lower() else 0.0, 1 - wer([pred.lower()], [r.lower()])) for r in ref]) / len(ref) for pred, ref in zip(preds, refs) + sum([max(1.0 if r.lower() in pred.lower() else 0.0, 1 - wer([pred.lower()], [r.lower()])) for r in ref]) + / len(ref) + for pred, ref in zip(preds, refs) ][0] return score def string_match_part_single(preds, refs): preds = post_process_preds(preds) - preds = re.sub(r'Document \d+:(?:.*\n)+?\n', '', preds) + preds = re.sub(r"Document \d+:(?:.*\n)+?\n", "", preds) preds = [preds] refs = [refs] score = [ - sum([max([max(1.0 if r.lower() in pred.lower() else 0.0, 1 - wer([pred.lower()], [r.lower()])) for r in ref]) for pred, ref in zip(preds, refs)]) + sum( + [ + max( + [ + max(1.0 if r.lower() in pred.lower() else 0.0, 1 - wer([pred.lower()], [r.lower()])) + for r in ref + ] + ) + for pred, ref in zip(preds, refs) + ] + ) ][0] return score @@ -158,7 +171,6 @@ def string_match_part_single(preds, refs): "2steps": string_match_2steps_single, } - jsonl_file = eval_config.input_file with open(jsonl_file, "rt", encoding="utf-8") as fin: data = [json.loads(line) for line in fin] diff --git a/requirements/main.txt b/requirements/main.txt index a7d7dded31..f3ac3405c6 100644 --- a/requirements/main.txt +++ b/requirements/main.txt @@ -16,6 +16,7 @@ bs4 click < 8.2.0 # https://github.com/ai-dynamo/dynamo/issues/1039 compute-eval @ git+https://github.com/NVIDIA/compute-eval.git@2d14770 datasets<4 # lcb problem with datasets 4.0.0 +editdistance # ddgs # Needed for BFCLv4 - currently cannot be installed directly due to lepton obsolete httpx version in use evalplus @ git+https://github.com/evalplus/evalplus@c91370f faiss-cpu # Needed for BFCLv4 @@ -50,4 +51,3 @@ tqdm transformers typer >= 0.13 wandb -editdistance From 733844d4937962065cbc95f7a8783466759b98bb Mon Sep 17 00:00:00 2001 From: Cheng-Ping Hsieh Date: Tue, 13 Jan 2026 16:01:12 +0800 Subject: [PATCH 85/88] fix Signed-off-by: Cheng-Ping Hsieh --- nemo_skills/dataset/ruler/prepare.py | 2 +- nemo_skills/dataset/ruler2/prepare.py | 48 +++++++++++++-------------- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/nemo_skills/dataset/ruler/prepare.py b/nemo_skills/dataset/ruler/prepare.py index 71e4a5872a..2a1ffe6df1 100644 --- a/nemo_skills/dataset/ruler/prepare.py +++ b/nemo_skills/dataset/ruler/prepare.py @@ -209,4 +209,4 @@ def prepare_task(task): ruler_prepare_args, tmp_data_dir=args.tmp_data_dir, ) - print("RULER dataset preparation completed.") \ No newline at end of file + print("RULER dataset preparation completed.") diff --git a/nemo_skills/dataset/ruler2/prepare.py b/nemo_skills/dataset/ruler2/prepare.py index 514ed188d4..724c1ff8d9 100644 --- a/nemo_skills/dataset/ruler2/prepare.py +++ b/nemo_skills/dataset/ruler2/prepare.py @@ -41,9 +41,9 @@ def prepare_mk_niah_basic(output_folder, tokenizer_type, tokenizer_path, length, "--tokenizer_path", tokenizer_path, "--max_seq_length", - length, + str(length), "--num_samples", - dataset_size, + str(dataset_size), "--random_seed", "42", "--num_needle_k", @@ -78,9 +78,9 @@ def prepare_mk_niah_easy(output_folder, tokenizer_type, tokenizer_path, length, "--tokenizer_path", tokenizer_path, "--max_seq_length", - length, + str(length), "--num_samples", - dataset_size, + str(dataset_size), "--random_seed", "42", "--dataset", @@ -113,9 +113,9 @@ def prepare_mk_niah_medium(output_folder, tokenizer_type, tokenizer_path, length "--tokenizer_path", tokenizer_path, "--max_seq_length", - length, + str(length), "--num_samples", - dataset_size, + str(dataset_size), "--random_seed", "42", "--dataset", @@ -148,9 +148,9 @@ def prepare_mk_niah_hard(output_folder, tokenizer_type, tokenizer_path, length, "--tokenizer_path", tokenizer_path, "--max_seq_length", - length, + str(length), "--num_samples", - dataset_size, + str(dataset_size), "--random_seed", "42", "--dataset", @@ -183,9 +183,9 @@ def prepare_mv_niah_basic(output_folder, tokenizer_type, tokenizer_path, length, "--tokenizer_path", tokenizer_path, "--max_seq_length", - length, + str(length), "--num_samples", - dataset_size, + str(dataset_size), "--random_seed", "42", "--num_needle_k", @@ -220,9 +220,9 @@ def prepare_mv_niah_easy(output_folder, tokenizer_type, tokenizer_path, length, "--tokenizer_path", tokenizer_path, "--max_seq_length", - length, + str(length), "--num_samples", - dataset_size, + str(dataset_size), "--random_seed", "42", "--dataset", @@ -255,9 +255,9 @@ def prepare_mv_niah_medium(output_folder, tokenizer_type, tokenizer_path, length "--tokenizer_path", tokenizer_path, "--max_seq_length", - length, + str(length), "--num_samples", - dataset_size, + str(dataset_size), "--random_seed", "42", "--dataset", @@ -290,9 +290,9 @@ def prepare_mv_niah_hard(output_folder, tokenizer_type, tokenizer_path, length, "--tokenizer_path", tokenizer_path, "--max_seq_length", - length, + str(length), "--num_samples", - dataset_size, + str(dataset_size), "--random_seed", "42", "--dataset", @@ -325,9 +325,9 @@ def prepare_qa_basic(output_folder, tokenizer_type, tokenizer_path, length, data "--tokenizer_path", tokenizer_path, "--max_seq_length", - length, + str(length), "--num_samples", - dataset_size, + str(dataset_size), "--random_seed", "42", "--dataset", @@ -358,9 +358,9 @@ def prepare_qa_easy(output_folder, tokenizer_type, tokenizer_path, length, datas "--tokenizer_path", tokenizer_path, "--max_seq_length", - length, + str(length), "--num_samples", - dataset_size, + str(dataset_size), "--random_seed", "42", "--dataset", @@ -391,9 +391,9 @@ def prepare_qa_medium(output_folder, tokenizer_type, tokenizer_path, length, dat "--tokenizer_path", tokenizer_path, "--max_seq_length", - length, + str(length), "--num_samples", - dataset_size, + str(dataset_size), "--random_seed", "42", "--dataset", @@ -424,9 +424,9 @@ def prepare_qa_hard(output_folder, tokenizer_type, tokenizer_path, length, datas "--tokenizer_path", tokenizer_path, "--max_seq_length", - length, + str(length), "--num_samples", - dataset_size, + str(dataset_size), "--random_seed", "42", "--dataset", From 258106418a93061778d4c0ecd931301082a7903f Mon Sep 17 00:00:00 2001 From: Cheng-Ping Hsieh Date: Wed, 14 Jan 2026 12:48:09 +0800 Subject: [PATCH 86/88] resolve comment Signed-off-by: Cheng-Ping Hsieh --- docs/evaluation/long-context.md | 14 ++++++++++++++ nemo_skills/dataset/ruler2/prepare.py | 2 +- nemo_skills/dataset/ruler2/ruler2_score.py | 2 +- tests/gpu-tests/test_eval.py | 1 + 4 files changed, 17 insertions(+), 2 deletions(-) diff --git a/docs/evaluation/long-context.md b/docs/evaluation/long-context.md index 2ce8327e59..1709ca68c6 100644 --- a/docs/evaluation/long-context.md +++ b/docs/evaluation/long-context.md @@ -9,6 +9,20 @@ More details are coming soon! - Benchmark is defined in [`nemo_skills/dataset/ruler/__init__.py`](https://github.com/NVIDIA-NeMo/Skills/blob/main/nemo_skills/dataset/ruler/__init__.py) - Original benchmark source is [here](https://github.com/NVIDIA/RULER). + +### ruler2 + +- Benchmark is defined in [`nemo_skills/dataset/ruler2/__init__.py`](https://github.com/NVIDIA-NeMo/Skills/blob/main/nemo_skills/dataset/ruler2/__init__.py) +- Please follow this [setup](https://github.com/NVIDIA/RULER/blob/rulerv2-ns/README.md) to run evaluation. +- Example scores +| Model | Avg | 8192 | 16384 | 32768 | 65536 | 131072 | 262144 | 524288 | 1000000 | +|-----------------------------------------|------|------|-------|-------|-------|--------|--------|--------|---------| +| Gemini 2.5 Flash Think On | 91.4 | 94.3 | 93.7 | 91.4 | 88.4 | 89.0 | - | - | - | +| Gemini 2.5 Flash Think Off | 88.0 | 91.3 | 89.0 | 88.8 | 85.5 | 85.5 | 82.5 | 79.1 | 77.0 | +| GPT 4.1 | 89.2 | 91.2 | 90.8 | 89.8 | 87.7 | 86.5 | 80.6 | 74.5 | 75.2 | +| Qwen3-235B-A22B-Thinking-2507 | 85.2 | 92.9 | 91.3 | 85.3 | 80.6 | 75.7 | - | - | - | +| Qwen3-235B-A22B-Instruct-2507 | 83.7 | 87.3 | 85.8 | 84.5 | 82.5 | 78.2 | 65.3 | 53.0 | 36.1 | + ### mrcr - Benchmark is defined in [`nemo_skills/dataset/mrcr/__init__.py`](https://github.com/NVIDIA-NeMo/Skills/blob/main/nemo_skills/dataset/mrcr/__init__.py) diff --git a/nemo_skills/dataset/ruler2/prepare.py b/nemo_skills/dataset/ruler2/prepare.py index 724c1ff8d9..1152ff3bcd 100644 --- a/nemo_skills/dataset/ruler2/prepare.py +++ b/nemo_skills/dataset/ruler2/prepare.py @@ -484,7 +484,7 @@ def prepare_dataset(tasks, setup, max_seq_length, tokenizer_type, tokenizer_path output_folder = Path(__file__).parent / setup # 1. installing necessary packages - subprocess.run(["pip", "install", "wonderwords", "html2text", "tenacity"], check=True) + # subprocess.run(["pip", "install", "wonderwords", "html2text", "tenacity"], check=True) for task in tasks: prepare_task_for_ns(output_folder, task) diff --git a/nemo_skills/dataset/ruler2/ruler2_score.py b/nemo_skills/dataset/ruler2/ruler2_score.py index 7bc01c1dc1..68e4e34542 100644 --- a/nemo_skills/dataset/ruler2/ruler2_score.py +++ b/nemo_skills/dataset/ruler2/ruler2_score.py @@ -37,7 +37,7 @@ def compute_score(metrics: dict): metrics[setup][aggregation] = { "accuracy": sum( metrics[f"{setup}.{task}"][aggregation].get( - "accuracy", (metrics[f"{setup}.{task}"][aggregation].get("symbolic_correct", 0)) + "accuracy", (metrics[f"{setup}.{task}"][aggregation]["symbolic_correct"]) ) for task in tasks ) diff --git a/tests/gpu-tests/test_eval.py b/tests/gpu-tests/test_eval.py index ee908a9108..91abbaf288 100644 --- a/tests/gpu-tests/test_eval.py +++ b/tests/gpu-tests/test_eval.py @@ -28,6 +28,7 @@ EXCLUDED_DATASETS = { "__pycache__", "ruler", + "ruler2", "bigcodebench", "livecodebench", "livebench_coding", From 91cdf7d87c14652808e56e5854da241a99103e8e Mon Sep 17 00:00:00 2001 From: Igor Gitman Date: Fri, 16 Jan 2026 15:29:19 -0800 Subject: [PATCH 87/88] Update docs Signed-off-by: Igor Gitman --- docs/evaluation/long-context.md | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/docs/evaluation/long-context.md b/docs/evaluation/long-context.md index 22449a5120..c41c5fadae 100644 --- a/docs/evaluation/long-context.md +++ b/docs/evaluation/long-context.md @@ -28,8 +28,35 @@ Other supported options ### ruler2 - Benchmark is defined in [`nemo_skills/dataset/ruler2/__init__.py`](https://github.com/NVIDIA-NeMo/Skills/blob/main/nemo_skills/dataset/ruler2/__init__.py) -- Please follow this [setup](https://github.com/NVIDIA/RULER/blob/rulerv2-ns/README.md) to run evaluation. -- Example scores + +It's recommended to use [data_dir parameter](../evaluation/index.md#using-data-on-cluster) when running evaluation. +Ruler2 also requires `setup`, `tokenizer_path` and `max_seq_length` to be specified. Example command to prepare data + +```bash +ns prepare_data ruler2 \ + --cluster= \ + --data_dir= \ + --setup= \ + --tokenizer_path= \ + --max_seq_length= +``` + +Example evaluation command + +```bash +ns eval \ + --cluster= \ + --data_dir= \ + --output_dir= \ + --benchmarks=ruler2. \ + --model= \ + --server_nodes=1 \ + --server_gpus=8 \ + --server_type=vllm +``` + +Example scores + | Model | Avg | 8192 | 16384 | 32768 | 65536 | 131072 | 262144 | 524288 | 1000000 | |-----------------------------------------|------|------|-------|-------|-------|--------|--------|--------|---------| | Gemini 2.5 Flash Think On | 91.4 | 94.3 | 93.7 | 91.4 | 88.4 | 89.0 | - | - | - | From d9eed2ac80d269882992d9a79f566bd0beb91c03 Mon Sep 17 00:00:00 2001 From: Igor Gitman Date: Fri, 16 Jan 2026 15:32:08 -0800 Subject: [PATCH 88/88] Add link Signed-off-by: Igor Gitman --- docs/evaluation/long-context.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/evaluation/long-context.md b/docs/evaluation/long-context.md index c41c5fadae..92815edca9 100644 --- a/docs/evaluation/long-context.md +++ b/docs/evaluation/long-context.md @@ -65,6 +65,8 @@ Example scores | Qwen3-235B-A22B-Thinking-2507 | 85.2 | 92.9 | 91.3 | 85.3 | 80.6 | 75.7 | - | - | - | | Qwen3-235B-A22B-Instruct-2507 | 83.7 | 87.3 | 85.8 | 84.5 | 82.5 | 78.2 | 65.3 | 53.0 | 36.1 | +For more details see [https://github.com/NVIDIA/RULER/blob/rulerv2-ns](https://github.com/NVIDIA/RULER/blob/rulerv2-ns/) + ### mrcr - Benchmark is defined in [`nemo_skills/dataset/mrcr/__init__.py`](https://github.com/NVIDIA-NeMo/Skills/blob/main/nemo_skills/dataset/mrcr/__init__.py)